diff --git a/.gitignore b/.gitignore index 26bf11f2..b65c164f 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,7 @@ e2e/tests/ui/features/**/auto-generated.step.ts # Test results test-results/ + +# Reports +*.csv + diff --git a/e2e/README.md b/e2e/README.md index 73ee0233..eaad3611 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -25,16 +25,23 @@ npm run test ``` +- Run a set of tests marked with a tag: + + ``` + npm run test --grep performance + ``` + - For other methods and operating systems, see [Developing tests](DEVELOPING.md) ## Environment Variables General: -| Variable | Default Value | Description | -|----------------|---------------|-------------------------------------------------| -| LOG_LEVEL | info | Possible values: debug, info, warn, error, none | -| SKIP_INGESTION | false | If to skip initial data ingestion/cleanup | +| Variable | Default Value | Description | +|----------------|-----------------|-------------------------------------------------------------------------| +| LOG_LEVEL | info | Possible values: debug, info, warn, error, none | +| SKIP_INGESTION | false | If to skip initial data ingestion/cleanup | +| REPORT_DIR | "test-results/" | Where additional reports should be stored (e.g. from performance tests) | For UI tests: diff --git a/e2e/tests/api/dependencies/global.setup.ts b/e2e/tests/api/dependencies/global.setup.ts index d4f64afa..55bdd101 100644 --- a/e2e/tests/api/dependencies/global.setup.ts +++ b/e2e/tests/api/dependencies/global.setup.ts @@ -1,8 +1,3 @@ -import fs from "node:fs"; -import path from "node:path"; - -import type { AxiosInstance } from "axios"; - import { ADVISORY_FILES, logger, @@ -10,6 +5,7 @@ import { SETUP_TIMEOUT, } from "../../common/constants"; import { test as setup } from "../fixtures"; +import { uploadAdvisories, uploadSboms } from "../helpers/upload"; setup.describe("Ingest initial data", () => { setup.skip( @@ -21,36 +17,8 @@ setup.describe("Ingest initial data", () => { setup.setTimeout(SETUP_TIMEOUT); logger.info("Setup: start uploading assets"); - await uploadSboms(axios, SBOM_FILES); - await uploadAdvisories(axios, ADVISORY_FILES); + await uploadSboms(axios, "../../common/assets/sbom/", SBOM_FILES); + await uploadAdvisories(axios, "../../common/assets/csaf/", ADVISORY_FILES); logger.info("Setup: upload finished successfully"); }); }); - -const uploadSboms = async (axios: AxiosInstance, files: string[]) => { - const uploads = files.map((e) => { - const filePath = path.join(__dirname, `../../common/assets/sbom/${e}`); - fs.statSync(filePath); // Verify file exists - - const fileStream = fs.createReadStream(filePath); - return axios.post("/api/v2/sbom", fileStream, { - headers: { "Content-Type": "application/json+bzip2" }, - }); - }); - - await Promise.all(uploads); -}; - -const uploadAdvisories = async (axios: AxiosInstance, files: string[]) => { - const uploads = files.map((e) => { - const filePath = path.join(__dirname, `../../common/assets/csaf/${e}`); - fs.statSync(filePath); // Verify file exists - - const fileStream = fs.createReadStream(filePath); - return axios.post("/api/v2/advisory", fileStream, { - headers: { "Content-Type": "application/json+bzip2" }, - }); - }); - - await Promise.all(uploads); -}; diff --git a/e2e/tests/api/features/assets/performance/delete/1.46.0-26.el9_4-product.json.bz2 b/e2e/tests/api/features/assets/performance/delete/1.46.0-26.el9_4-product.json.bz2 new file mode 100644 index 00000000..c93c20a7 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/1.46.0-26.el9_4-product.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/1.46.0-26.el9_4-release.json.bz2 b/e2e/tests/api/features/assets/performance/delete/1.46.0-26.el9_4-release.json.bz2 new file mode 100644 index 00000000..6526a913 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/1.46.0-26.el9_4-release.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/1.46.0-27.el9_4-product.json.bz2 b/e2e/tests/api/features/assets/performance/delete/1.46.0-27.el9_4-product.json.bz2 new file mode 100644 index 00000000..bf53e3c9 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/1.46.0-27.el9_4-product.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/1.46.0-27.el9_4-release.json.bz2 b/e2e/tests/api/features/assets/performance/delete/1.46.0-27.el9_4-release.json.bz2 new file mode 100644 index 00000000..9927d74f Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/1.46.0-27.el9_4-release.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/1_devspaces_pluginregistry-rhel8.json.bz2 b/e2e/tests/api/features/assets/performance/delete/1_devspaces_pluginregistry-rhel8.json.bz2 new file mode 100644 index 00000000..ce4d5646 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/1_devspaces_pluginregistry-rhel8.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/1_devspaces_server-rhel8.json.bz2 b/e2e/tests/api/features/assets/performance/delete/1_devspaces_server-rhel8.json.bz2 new file mode 100644 index 00000000..f61e5841 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/1_devspaces_server-rhel8.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.10.Final-redhat-00002.json.bz2 b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.10.Final-redhat-00002.json.bz2 new file mode 100644 index 00000000..6a7cd80a Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.10.Final-redhat-00002.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.11.Final-redhat-00001.json.bz2 b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.11.Final-redhat-00001.json.bz2 new file mode 100644 index 00000000..63629e72 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.11.Final-redhat-00001.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.12.Final-redhat-00002.json.bz2 b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.12.Final-redhat-00002.json.bz2 new file mode 100644 index 00000000..8491d3ef Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.12.Final-redhat-00002.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.6.Final-redhat-00002.json.bz2 b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.6.Final-redhat-00002.json.bz2 new file mode 100644 index 00000000..b1aa06f7 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.6.Final-redhat-00002.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.9.Final-redhat-00003.json.bz2 b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.9.Final-redhat-00003.json.bz2 new file mode 100644 index 00000000..326c907e Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/3_quarkus-bom-3.2.9.Final-redhat-00003.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/4_RHEL-9-FAST-DATAPATH.json.bz2 b/e2e/tests/api/features/assets/performance/delete/4_RHEL-9-FAST-DATAPATH.json.bz2 new file mode 100644 index 00000000..3f19857a Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/4_RHEL-9-FAST-DATAPATH.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/jboss-eap-7_eap74-openjdk11-openshift-rhel8.json.bz2 b/e2e/tests/api/features/assets/performance/delete/jboss-eap-7_eap74-openjdk11-openshift-rhel8.json.bz2 new file mode 100644 index 00000000..838fd3da Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/jboss-eap-7_eap74-openjdk11-openshift-rhel8.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/jboss-eap-7_eap74-openjdk8-openshift-rhel8.json.bz2 b/e2e/tests/api/features/assets/performance/delete/jboss-eap-7_eap74-openjdk8-openshift-rhel8.json.bz2 new file mode 100644 index 00000000..0e336545 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/jboss-eap-7_eap74-openjdk8-openshift-rhel8.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-amd64.json.bz2 b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-amd64.json.bz2 new file mode 100644 index 00000000..573bfc6d Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-amd64.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-image-index.json.bz2 b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-image-index.json.bz2 new file mode 100644 index 00000000..7c33554b Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-image-index.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-product.json.bz2 b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-product.json.bz2 new file mode 100644 index 00000000..ac4e931d Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel-8-product.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel8-v3.14.0-4-binary.json.bz2 b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel8-v3.14.0-4-binary.json.bz2 new file mode 100644 index 00000000..28f47ecc Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel8-v3.14.0-4-binary.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel8-v3.14.0-4-index.json.bz2 b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel8-v3.14.0-4-index.json.bz2 new file mode 100644 index 00000000..93b79b5f Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/quay-builder-qemu-rhcos-rhel8-v3.14.0-4-index.json.bz2 differ diff --git a/e2e/tests/api/features/assets/performance/delete/quay-v3.14.0-product.json.bz2 b/e2e/tests/api/features/assets/performance/delete/quay-v3.14.0-product.json.bz2 new file mode 100644 index 00000000..61bbaa52 Binary files /dev/null and b/e2e/tests/api/features/assets/performance/delete/quay-v3.14.0-product.json.bz2 differ diff --git a/e2e/tests/api/features/performance-delete.ts b/e2e/tests/api/features/performance-delete.ts new file mode 100644 index 00000000..36e6ad92 --- /dev/null +++ b/e2e/tests/api/features/performance-delete.ts @@ -0,0 +1,127 @@ +import { logger } from "../../common/constants"; +import { test } from "../fixtures"; +import { deleteSboms } from "../helpers/delete"; +import { writeRequestDurationToFile } from "../helpers/report"; +import { uploadSboms } from "../helpers/upload"; + +test.describe.configure({ mode: "serial" }); + +const SBOM_DIR = "../features/assets/performance/delete"; // The path is relative to the helpers/upload.ts file. +const SBOM_FILES = [ + "1_devspaces_pluginregistry-rhel8.json.bz2", + "1_devspaces_server-rhel8.json.bz2", + "1.46.0-26.el9_4-product.json.bz2", + "1.46.0-26.el9_4-release.json.bz2", + "1.46.0-27.el9_4-product.json.bz2", + "1.46.0-27.el9_4-release.json.bz2", + "3_quarkus-bom-3.2.6.Final-redhat-00002.json.bz2", + "3_quarkus-bom-3.2.9.Final-redhat-00003.json.bz2", + "3_quarkus-bom-3.2.10.Final-redhat-00002.json.bz2", + "3_quarkus-bom-3.2.11.Final-redhat-00001.json.bz2", + "3_quarkus-bom-3.2.12.Final-redhat-00002.json.bz2", + "4_RHEL-9-FAST-DATAPATH.json.bz2", + "jboss-eap-7_eap74-openjdk8-openshift-rhel8.json.bz2", + "jboss-eap-7_eap74-openjdk11-openshift-rhel8.json.bz2", + "quay-builder-qemu-rhcos-rhel-8-amd64.json.bz2", + "quay-builder-qemu-rhcos-rhel-8-image-index.json.bz2", + "quay-builder-qemu-rhcos-rhel-8-product.json.bz2", + "quay-builder-qemu-rhcos-rhel8-v3.14.0-4-binary.json.bz2", + "quay-builder-qemu-rhcos-rhel8-v3.14.0-4-index.json.bz2", + "quay-v3.14.0-product.json.bz2", +]; + +let sbomIds: string[] = []; + +const REPORT_FILE_PREFIX = "report-perf-delete-"; + +test.describe("Performance / Deletion", { tag: "@performance" }, () => { + test.beforeEach(async ({ axios }) => { + logger.info("Uploading SBOMs before deletion performance tests."); + + const uploadResponses = await uploadSboms(axios, SBOM_DIR, SBOM_FILES); + + uploadResponses.forEach((response) => { + sbomIds.push(response.data.id); + }); + + sbomIds.forEach((id) => { + logger.info(id); + }); + + logger.info(`Uploaded ${sbomIds.length} SBOMs.`); + }); + + test("SBOMs / Sequential", async ({ axios }) => { + const currentTimeStamp = Date.now(); + const reportFile = `${REPORT_FILE_PREFIX}sequential-${currentTimeStamp}.csv`; + var index = 1; + + var duration = ""; + + writeRequestDurationToFile(reportFile, "No.", "SBOM ID", "Duration [ms]"); + + for (const sbomId of sbomIds) { + try { + await axios.delete(`/api/v2/sbom/${sbomId}`).then((response) => { + duration = String(response.duration); + }); + } catch (error) { + logger.error(`SBOM with ID ${sbomId} could not be deleted.`, error); + duration = "n/a"; + } + + writeRequestDurationToFile(reportFile, String(index), sbomId, duration); + duration = ""; + index++; + } + }); + + test("SBOMs / Parallel", async ({ axios }) => { + const currentTimeStamp = Date.now(); + const reportFile = `${REPORT_FILE_PREFIX}parallel-${currentTimeStamp}.csv`; + + writeRequestDurationToFile(reportFile, "No.", "SBOM ID", "Duration [ms]"); + + const deletionPromises = sbomIds.map(async (sbomId) => { + const deletePromise = axios + .delete(`/api/v2/sbom/${sbomId}`) + .then((response) => + writeRequestDurationToFile( + reportFile, + "n/a", + response.data.id, + String(response.duration), + ), + ) + .catch((error) => { + logger.error(`SBOM with ID ${sbomId} could not be deleted.`, error); + }); + + return deletePromise; + }); + + await Promise.all(deletionPromises); + }); + + // Re-try deletion of all SBOMs in case some of the SBOMs didn't get deleted during the tests. + test.afterEach(async ({ axios }) => { + logger.info("Cleaning up SBOMs after deletion performance tests."); + + const deleteResponses = await deleteSboms(axios, sbomIds); + + if ( + deleteResponses.every( + (result) => + result.status === "fulfilled" && result.value?.status === 200, + ) + ) { + logger.info("All SBOMS have been deleted successfully."); + } else { + logger.warn( + "Some SBOM deletions were unsuccessful. Check the logs and/or consider deleting the SBOMs manually.", + ); + } + + sbomIds = []; + }); +}); diff --git a/e2e/tests/api/fixtures.ts b/e2e/tests/api/fixtures.ts index d18e9575..e4bbf12c 100644 --- a/e2e/tests/api/fixtures.ts +++ b/e2e/tests/api/fixtures.ts @@ -87,56 +87,87 @@ export const discoverTokenEndpoint = async ( return envInfo.OIDC_SERVER_URL ?? null; }; +declare module "axios" { + export interface AxiosRequestConfig { + startTime?: number; + } + export interface AxiosResponse { + endTime?: number; + duration?: number; // in milliseconds + } +} + const initAxiosInstance = async ( axiosInstance: AxiosInstance, baseURL?: string, ) => { - const { data: tokenResponse } = await getToken(baseURL); - access_token = tokenResponse.access_token; - - // Intercept Requests - axiosInstance.interceptors.request.use( - (config) => { - config.headers.Authorization = `Bearer ${access_token}`; - logger.debug(config); - return config; - }, - (error) => { - return Promise.reject(error); - }, - ); - - // Intercept Responses - axiosInstance.interceptors.response.use( - (response) => { - return response; - }, - async (error) => { - if (error.response && error.response.status === 401) { - const { data: refreshedTokenResponse } = await getToken(baseURL); - access_token = refreshedTokenResponse.access_token; - - const retryCounter = error.config.retryCounter || 1; - const retryConfig = { - ...error.config, - headers: { - ...error.config.headers, - Authorization: `Bearer ${access_token}`, - }, - }; - - // Retry limited times - if (retryCounter < 2) { - return axios({ - ...retryConfig, - retryCounter: retryCounter + 1, - }); + if (AUTH_REQUIRED === "true") { + logger.info("Auth enabled. Getting token."); + + const { data: tokenResponse } = await getToken(baseURL); + access_token = tokenResponse.access_token; + + // Add access token + axiosInstance.interceptors.request.use( + (config) => { + config.headers.Authorization = `Bearer ${access_token}`; + logger.debug(config); + return config; + }, + (error) => { + return Promise.reject(error); + }, + ); + + // Retry + axiosInstance.interceptors.response.use( + (response) => { + return response; + }, + async (error) => { + if (error.response && error.response.status === 401) { + const { data: refreshedTokenResponse } = await getToken(baseURL); + access_token = refreshedTokenResponse.access_token; + + const retryCounter = error.config.retryCounter || 1; + const retryConfig = { + ...error.config, + headers: { + ...error.config.headers, + Authorization: `Bearer ${access_token}`, + }, + }; + + // Retry limited times + if (retryCounter < 2) { + return axios({ + ...retryConfig, + retryCounter: retryCounter + 1, + }); + } } - } - return Promise.reject(error); - }, - ); + return Promise.reject(error); + }, + ); + } + + // Measure request start time + axiosInstance.interceptors.request.use((config) => { + config.startTime = Date.now(); + return config; + }); + + // Measure response reception time + axiosInstance.interceptors.response.use((response) => { + if (response.config.startTime != null) { + response.endTime = Date.now(); + response.duration = response.endTime - response.config.startTime; + } else { + response.duration = 0; + } + return response; + }); }; // Declare the types of your fixtures. @@ -157,10 +188,8 @@ export const test = base.extend({ : undefined, }); - if (AUTH_REQUIRED === "true") { - logger.info("Auth enabled. Initializing configuration"); - await initAxiosInstance(axiosInstance, TRUSTIFY_API_URL); - } + logger.info("Initializing configuration."); + await initAxiosInstance(axiosInstance, TRUSTIFY_API_URL); await use(axiosInstance); }, diff --git a/e2e/tests/api/helpers/delete.ts b/e2e/tests/api/helpers/delete.ts new file mode 100644 index 00000000..2bc8e8e7 --- /dev/null +++ b/e2e/tests/api/helpers/delete.ts @@ -0,0 +1,24 @@ +import type { AxiosInstance } from "axios"; + +import { logger } from "../../common/constants"; + +export async function deleteSboms(axios: AxiosInstance, sbomIds: string[]) { + var existingSbomIds = []; + + for (const sbomId of sbomIds) { + try { + await axios.get(`/api/v2/sbom/${sbomId}`); + existingSbomIds.push(sbomId); + } catch (_error) { + logger.info(`SBOM with ID ${sbomId} does not exist anymore. Skipping.`); + } + } + + const deletionPromises = existingSbomIds.map((sbomId) => + axios.delete(`/api/v2/sbom/${sbomId}`), + ); + + const responses = await Promise.allSettled(deletionPromises); + + return responses; +} diff --git a/e2e/tests/api/helpers/report.ts b/e2e/tests/api/helpers/report.ts new file mode 100644 index 00000000..27f374a6 --- /dev/null +++ b/e2e/tests/api/helpers/report.ts @@ -0,0 +1,19 @@ +import fs from "node:fs"; +import { logger, REPORT_DIR } from "../../common/constants"; + +export function writeRequestDurationToFile( + fileName: string, + sbomNumber: string, + sbomId: string, + duration: string, +) { + const line = `${sbomNumber},${sbomId},${duration}`; + + try { + fs.appendFileSync(`${REPORT_DIR}${fileName}`, `${line}\n`); + logger.debug(line); + } catch (error) { + logger.error(`Error writing the request duration to file: ${error}`); + throw error; + } +} diff --git a/e2e/tests/api/helpers/upload.ts b/e2e/tests/api/helpers/upload.ts new file mode 100644 index 00000000..2c578884 --- /dev/null +++ b/e2e/tests/api/helpers/upload.ts @@ -0,0 +1,42 @@ +import fs from "node:fs"; +import path from "node:path"; + +import type { AxiosInstance } from "axios"; + +export async function uploadSboms( + axios: AxiosInstance, + sbomDirPath: string, + files: string[], +) { + const uploads = files.map((e) => { + const filePath = path.join(__dirname, `${sbomDirPath}/${e}`); + fs.statSync(filePath); // Verify file exists + + const fileStream = fs.createReadStream(filePath); + return axios.post("/api/v2/sbom", fileStream, { + headers: { "Content-Type": "application/json+bzip2" }, + }); + }); + + const responses = await Promise.all(uploads); + return responses; +} + +export async function uploadAdvisories( + axios: AxiosInstance, + advisoryDirPath: string, + files: string[], +) { + const uploads = files.map((e) => { + const filePath = path.join(__dirname, `${advisoryDirPath}/${e}`); + fs.statSync(filePath); // Verify file exists + + const fileStream = fs.createReadStream(filePath); + return axios.post("/api/v2/advisory", fileStream, { + headers: { "Content-Type": "application/json+bzip2" }, + }); + }); + + const responses = await Promise.all(uploads); + return responses; +} diff --git a/e2e/tests/common/constants.ts b/e2e/tests/common/constants.ts index 8d203bf3..a2b60e07 100644 --- a/e2e/tests/common/constants.ts +++ b/e2e/tests/common/constants.ts @@ -40,6 +40,7 @@ export const logger = { CURRENT_LOG_LEVEL >= LOG_LEVELS.error && console.error("[ERROR]", ...args); }, }; +export const REPORT_DIR = process.env.REPORT_DIR ?? "test-results/"; export const SETUP_TIMEOUT = 240_000;