SBOM Deletion Performance Tests

.gitignore

@@ -48,3 +48,7 @@ e2e/tests/ui/features/**/auto-generated.step.ts
 
 # Test results
 test-results/
+
+# Reports
+*.csv
+

e2e/README.md

@@ -25,16 +25,23 @@
   npm run test
   ```
 
+- Run a set of tests marked with a tag:
+
+  ```
+  npm run test --grep performance
+  ```
+
 - For other methods and operating systems, see [Developing tests](DEVELOPING.md)
 
 ## Environment Variables
 
 General:
 
-| Variable       | Default Value | Description                                     |
-|----------------|---------------|-------------------------------------------------|
-| LOG_LEVEL      | info          | Possible values: debug, info, warn, error, none |
-| SKIP_INGESTION | false         | If to skip initial data ingestion/cleanup       |
+| Variable       | Default Value   | Description                                                             |
+|----------------|-----------------|-------------------------------------------------------------------------|
+| LOG_LEVEL      | info            | Possible values: debug, info, warn, error, none                         |
+| SKIP_INGESTION | false           | If to skip initial data ingestion/cleanup                               | 
+| REPORT_DIR     | "test-results/" | Where additional reports should be stored (e.g. from performance tests) |
 
 For UI tests:
 

e2e/tests/api/dependencies/global.setup.ts

@@ -1,15 +1,11 @@
-import fs from "node:fs";
-import path from "node:path";
-
-import type { AxiosInstance } from "axios";
-
 import {
   ADVISORY_FILES,
   logger,
   SBOM_FILES,
   SETUP_TIMEOUT,
 } from "../../common/constants";
 import { test as setup } from "../fixtures";
+import { uploadAdvisories, uploadSboms } from "../helpers/upload";
 
 setup.describe("Ingest initial data", () => {
   setup.skip(
@@ -21,36 +17,8 @@ setup.describe("Ingest initial data", () => {
     setup.setTimeout(SETUP_TIMEOUT);
 
     logger.info("Setup: start uploading assets");
-    await uploadSboms(axios, SBOM_FILES);
-    await uploadAdvisories(axios, ADVISORY_FILES);
+    await uploadSboms(axios, "../../common/assets/sbom/", SBOM_FILES);
+    await uploadAdvisories(axios, "../../common/assets/csaf/", ADVISORY_FILES);
     logger.info("Setup: upload finished successfully");
   });
 });
-
-const uploadSboms = async (axios: AxiosInstance, files: string[]) => {
-  const uploads = files.map((e) => {
-    const filePath = path.join(__dirname, `../../common/assets/sbom/${e}`);
-    fs.statSync(filePath); // Verify file exists
-
-    const fileStream = fs.createReadStream(filePath);
-    return axios.post("/api/v2/sbom", fileStream, {
-      headers: { "Content-Type": "application/json+bzip2" },
-    });
-  });
-
-  await Promise.all(uploads);
-};
-
-const uploadAdvisories = async (axios: AxiosInstance, files: string[]) => {
-  const uploads = files.map((e) => {
-    const filePath = path.join(__dirname, `../../common/assets/csaf/${e}`);
-    fs.statSync(filePath); // Verify file exists
-
-    const fileStream = fs.createReadStream(filePath);
-    return axios.post("/api/v2/advisory", fileStream, {
-      headers: { "Content-Type": "application/json+bzip2" },
-    });
-  });
-
-  await Promise.all(uploads);
-};

e2e/tests/api/features/performance-delete.ts

@@ -0,0 +1,127 @@
+import { logger } from "../../common/constants";
+import { test } from "../fixtures";
+import { deleteSboms } from "../helpers/delete";
+import { writeRequestDurationToFile } from "../helpers/report";
+import { uploadSboms } from "../helpers/upload";
+
+test.describe.configure({ mode: "serial" });
+
+const SBOM_DIR = "../features/assets/performance/delete"; // The path is relative to the helpers/upload.ts file.
+const SBOM_FILES = [
+  "1_devspaces_pluginregistry-rhel8.json.bz2",
+  "1_devspaces_server-rhel8.json.bz2",
+  "1.46.0-26.el9_4-product.json.bz2",
+  "1.46.0-26.el9_4-release.json.bz2",
+  "1.46.0-27.el9_4-product.json.bz2",
+  "1.46.0-27.el9_4-release.json.bz2",
+  "3_quarkus-bom-3.2.6.Final-redhat-00002.json.bz2",
+  "3_quarkus-bom-3.2.9.Final-redhat-00003.json.bz2",
+  "3_quarkus-bom-3.2.10.Final-redhat-00002.json.bz2",
+  "3_quarkus-bom-3.2.11.Final-redhat-00001.json.bz2",
+  "3_quarkus-bom-3.2.12.Final-redhat-00002.json.bz2",
+  "4_RHEL-9-FAST-DATAPATH.json.bz2",
+  "jboss-eap-7_eap74-openjdk8-openshift-rhel8.json.bz2",
+  "jboss-eap-7_eap74-openjdk11-openshift-rhel8.json.bz2",
+  "quay-builder-qemu-rhcos-rhel-8-amd64.json.bz2",
+  "quay-builder-qemu-rhcos-rhel-8-image-index.json.bz2",
+  "quay-builder-qemu-rhcos-rhel-8-product.json.bz2",
+  "quay-builder-qemu-rhcos-rhel8-v3.14.0-4-binary.json.bz2",
+  "quay-builder-qemu-rhcos-rhel8-v3.14.0-4-index.json.bz2",
+  "quay-v3.14.0-product.json.bz2",
+];
+
+let sbomIds: string[] = [];
+
+const REPORT_FILE_PREFIX = "report-perf-delete-";
+
+test.describe("Performance / Deletion", { tag: "@performance" }, () => {
+  test.beforeEach(async ({ axios }) => {
+    logger.info("Uploading SBOMs before deletion performance tests.");
+
+    const uploadResponses = await uploadSboms(axios, SBOM_DIR, SBOM_FILES);
+
+    uploadResponses.forEach((response) => {
+      sbomIds.push(response.data.id);
+    });
+
+    sbomIds.forEach((id) => {
+      logger.info(id);
+    });
+
+    logger.info(`Uploaded ${sbomIds.length} SBOMs.`);
+  });
+
+  test("SBOMs / Sequential", async ({ axios }) => {
+    const currentTimeStamp = Date.now();
+    const reportFile = `${REPORT_FILE_PREFIX}sequential-${currentTimeStamp}.csv`;
+    var index = 1;
+
+    var duration = "";
+
+    writeRequestDurationToFile(reportFile, "No.", "SBOM ID", "Duration [ms]");
+
+    for (const sbomId of sbomIds) {
+      try {
+        await axios.delete(`/api/v2/sbom/${sbomId}`).then((response) => {
+          duration = String(response.duration);
+        });
+      } catch (error) {
+        logger.error(`SBOM with ID ${sbomId} could not be deleted.`, error);
+        duration = "n/a";
+      }
+
+      writeRequestDurationToFile(reportFile, String(index), sbomId, duration);
+      duration = "";
+      index++;
+    }
+  });
+
+  test("SBOMs / Parallel", async ({ axios }) => {
+    const currentTimeStamp = Date.now();
+    const reportFile = `${REPORT_FILE_PREFIX}parallel-${currentTimeStamp}.csv`;
+
+    writeRequestDurationToFile(reportFile, "No.", "SBOM ID", "Duration [ms]");
+
+    const deletionPromises = sbomIds.map(async (sbomId) => {
+      const deletePromise = axios
+        .delete(`/api/v2/sbom/${sbomId}`)
+        .then((response) =>
+          writeRequestDurationToFile(
+            reportFile,
+            "n/a",
+            response.data.id,
+            String(response.duration),
+          ),
+        )
+        .catch((error) => {
+          logger.error(`SBOM with ID ${sbomId} could not be deleted.`, error);
+        });
+
+      return deletePromise;
+    });
+
+    await Promise.all(deletionPromises);
+  });
+
+  // Re-try deletion of all SBOMs in case some of the SBOMs didn't get deleted during the tests.
+  test.afterEach(async ({ axios }) => {
+    logger.info("Cleaning up SBOMs after deletion performance tests.");
+
+    const deleteResponses = await deleteSboms(axios, sbomIds);
+
+    if (
+      deleteResponses.every(
+        (result) =>
+          result.status === "fulfilled" && result.value?.status === 200,
+      )
+    ) {
+      logger.info("All SBOMS have been deleted successfully.");
+    } else {
+      logger.warn(
+        "Some SBOM deletions were unsuccessful. Check the logs and/or consider deleting the SBOMs manually.",
+      );
+    }
+
+    sbomIds = [];
+  });
+});

e2e/tests/api/fixtures.ts

@@ -87,56 +87,87 @@ export const discoverTokenEndpoint = async (
   return envInfo.OIDC_SERVER_URL ?? null;
 };
 
+declare module "axios" {
+  export interface AxiosRequestConfig {
+    startTime?: number;
+  }
+  export interface AxiosResponse {
+    endTime?: number;
+    duration?: number; // in milliseconds
+  }
+}
+
 const initAxiosInstance = async (
   axiosInstance: AxiosInstance,
   baseURL?: string,
 ) => {
-  const { data: tokenResponse } = await getToken(baseURL);
-  access_token = tokenResponse.access_token;
-
-  // Intercept Requests
-  axiosInstance.interceptors.request.use(
-    (config) => {
-      config.headers.Authorization = `Bearer ${access_token}`;
-      logger.debug(config);
-      return config;
-    },
-    (error) => {
-      return Promise.reject(error);
-    },
-  );
-
-  // Intercept Responses
-  axiosInstance.interceptors.response.use(
-    (response) => {
-      return response;
-    },
-    async (error) => {
-      if (error.response && error.response.status === 401) {
-        const { data: refreshedTokenResponse } = await getToken(baseURL);
-        access_token = refreshedTokenResponse.access_token;
-
-        const retryCounter = error.config.retryCounter || 1;
-        const retryConfig = {
-          ...error.config,
-          headers: {
-            ...error.config.headers,
-            Authorization: `Bearer ${access_token}`,
-          },
-        };
-
-        // Retry limited times
-        if (retryCounter < 2) {
-          return axios({
-            ...retryConfig,
-            retryCounter: retryCounter + 1,
-          });
+  if (AUTH_REQUIRED === "true") {
+    logger.info("Auth enabled. Getting token.");
+
+    const { data: tokenResponse } = await getToken(baseURL);
+    access_token = tokenResponse.access_token;
+
+    // Add access token
+    axiosInstance.interceptors.request.use(
+      (config) => {
+        config.headers.Authorization = `Bearer ${access_token}`;
+        logger.debug(config);
+        return config;
+      },
+      (error) => {
+        return Promise.reject(error);
+      },
+    );
+
+    // Retry
+    axiosInstance.interceptors.response.use(
+      (response) => {
+        return response;
+      },
+      async (error) => {
+        if (error.response && error.response.status === 401) {
+          const { data: refreshedTokenResponse } = await getToken(baseURL);
+          access_token = refreshedTokenResponse.access_token;
+
+          const retryCounter = error.config.retryCounter || 1;
+          const retryConfig = {
+            ...error.config,
+            headers: {
+              ...error.config.headers,
+              Authorization: `Bearer ${access_token}`,
+            },
+          };
+
+          // Retry limited times
+          if (retryCounter < 2) {
+            return axios({
+              ...retryConfig,
+              retryCounter: retryCounter + 1,
+            });
+          }
         }
-      }
 
-      return Promise.reject(error);
-    },
-  );
+        return Promise.reject(error);
+      },
+    );
+  }
+
+  // Measure request start time
+  axiosInstance.interceptors.request.use((config) => {
+    config.startTime = Date.now();
+    return config;
+  });
+
+  // Measure response reception time
+  axiosInstance.interceptors.response.use((response) => {
+    if (response.config.startTime != null) {
+      response.endTime = Date.now();
+      response.duration = response.endTime - response.config.startTime;
+    } else {
+      response.duration = 0;
+    }
+    return response;
+  });
 };
 
 // Declare the types of your fixtures.
@@ -157,10 +188,8 @@ export const test = base.extend<ApiClientFixture>({
         : undefined,
     });
 
-    if (AUTH_REQUIRED === "true") {
-      logger.info("Auth enabled. Initializing configuration");
-      await initAxiosInstance(axiosInstance, TRUSTIFY_API_URL);
-    }
+    logger.info("Initializing configuration.");
+    await initAxiosInstance(axiosInstance, TRUSTIFY_API_URL);
 
     await use(axiosInstance);
   },

e2e/tests/api/helpers/delete.ts

@@ -0,0 +1,24 @@
+import type { AxiosInstance } from "axios";
+
+import { logger } from "../../common/constants";
+
+export async function deleteSboms(axios: AxiosInstance, sbomIds: string[]) {
+  var existingSbomIds = [];
+
+  for (const sbomId of sbomIds) {
+    try {
+      await axios.get(`/api/v2/sbom/${sbomId}`);
+      existingSbomIds.push(sbomId);
+    } catch (_error) {
+      logger.info(`SBOM with ID ${sbomId} does not exist anymore. Skipping.`);
+    }
+  }
+
+  const deletionPromises = existingSbomIds.map((sbomId) =>
+    axios.delete(`/api/v2/sbom/${sbomId}`),
+  );
+
+  const responses = await Promise.allSettled(deletionPromises);
+
+  return responses;
+}