Merge pull request #20 from Splinter0/devel

Devel

packages/backend/src/services/analysis.ts

@@ -65,59 +65,68 @@ export const runAnalysis = async (sdk: SDK<never, BackendEvents>) => {
   );
 
   for (const template of templates) {
-    for (const user of users) {
-      const analysisRequest = await sendRequest(sdk, template, user);
-      if (analysisRequest) {
-        analysisStore.addRequest(analysisRequest);
-        sdk.api.send("results:created", analysisRequest);
+    // Run each template async
+    (async () => {
+      for (const user of users) {
+        if (analysisStore.resultExists(template.id, user.id)) {
+          continue;
+        }
+        const analysisRequest = await sendRequest(sdk, template, user);
+        if (analysisRequest) {
+          analysisStore.addRequest(analysisRequest);
+          sdk.api.send("results:created", analysisRequest);
+        }
       }
-    }
-  }
 
-  const roles = roleStore.getRoles();
-  for (const template of templates) {
-    const newRules: TemplateDTO["rules"] = [];
-
-    // Generate role rule statuses in parallel
-    const rolePromises = roles.map(async (role) => {
-      const currentRule = template.rules.find(
-        (rule) => rule.type === "RoleRule" && rule.roleId === role.id,
-      ) ?? {
-        type: "RoleRule",
-        roleId: role.id,
-        hasAccess: false,
-        status: "Untested",
-      };
-
-      const status = await generateRoleRuleStatus(sdk, template, role.id);
-      return { ...currentRule, status };
-    });
-
-    // Generate user rule statuses in parallel
-    const userPromises = users.map(async (user) => {
-      const currentRule = template.rules.find(
-        (rule) => rule.type === "UserRule" && rule.userId === user.id,
-      ) ?? {
-        type: "UserRule",
-        userId: user.id,
-        hasAccess: false,
-        status: "Untested",
-      };
-
-      const status = await generateUserRuleStatus(sdk, template, user);
-      return { ...currentRule, status };
-    });
-
-    // Await all role and user statuses
-    const roleResults = await Promise.all(rolePromises);
-    const userResults = await Promise.all(userPromises);
-
-    // Combine results
-    newRules.push(...roleResults, ...userResults);
-
-    template.rules = newRules;
-    templateStore.updateTemplate(template.id, template);
-    sdk.api.send("templates:updated", template);
+      const newRules: TemplateDTO["rules"] = [];
+      const roles = roleStore.getRoles();
+      const rolePromises = roles.map(async (role) => {
+        const currentRule = template.rules.find(
+          (rule) => rule.type === "RoleRule" && rule.roleId === role.id,
+        ) ?? {
+          type: "RoleRule",
+          roleId: role.id,
+          hasAccess: false,
+          status: "Untested",
+        };
+
+
+        if (currentRule.status !== "Untested") {
+          return currentRule;
+        }
+
+        const status = await generateRoleRuleStatus(sdk, template, role.id);
+        return { ...currentRule, status };
+      });
+
+      const userPromises = users.map(async (user) => {
+        const currentRule = template.rules.find(
+          (rule) => rule.type === "UserRule" && rule.userId === user.id,
+        ) ?? {
+          type: "UserRule",
+          userId: user.id,
+          hasAccess: false,
+          status: "Untested",
+        };
+
+        if (currentRule.status !== "Untested") {
+          return currentRule;
+        }
+
+        const status = await generateUserRuleStatus(sdk, template, user);
+        return { ...currentRule, status };
+      });
+
+      const roleResults = await Promise.all(rolePromises);
+      const userResults = await Promise.all(userPromises);
+
+      // Combine results
+      newRules.push(...roleResults, ...userResults);
+
+      template.rules = newRules;
+      templateStore.updateTemplate(template.id, template);
+      sdk.api.send("templates:updated", template);
+    })()
   }
 };
 

packages/backend/src/services/templates.ts

@@ -3,7 +3,7 @@ import type { Request, Response } from "caido:utils";
 import type { TemplateDTO } from "shared";
 
 import { TemplateStore } from "../stores/templates";
-import { generateID } from "../utils";
+import { generateID, sha256Hash } from "../utils";
 
 import { SettingsStore } from "../stores/settings";
 import type { BackendEvents } from "../types";
@@ -89,18 +89,25 @@ export const onInterceptResponse = async (
   const settings = settingsStore.getSettings();
   const store = TemplateStore.get();
 
+  if (settings.autoCaptureRequests == "off") {
+    return;
+  }
+
+  const templateId = generateTemplateId(request, settings.deDuplicateHeaders);
+  if (store.templateExists(templateId)) {
+    return
+  }
+
   switch (settings.autoCaptureRequests) {
-    case "off":
-      return;
     case "all": {
-      const template = toTemplate(request, response);
+      const template = toTemplate(request, response, templateId);
       store.addTemplate(template);
       sdk.api.send("templates:created", template);
       break;
     }
     case "inScope": {
       if (sdk.requests.inScope(request)) {
-        const template = toTemplate(request, response);
+        const template = toTemplate(request, response, templateId);
         store.addTemplate(template);
         sdk.api.send("templates:created", template);
       }
@@ -116,9 +123,23 @@ export const registerTemplateEvents = (sdk: SDK) => {
   sdk.events.onInterceptResponse(onInterceptResponse);
 };
 
-const toTemplate = (request: Request, response: Response): TemplateDTO => {
+
+const generateTemplateId = (request: Request, dedupeHeaders: string[] = []): string => {
+  let body = request.getBody()?.toText();
+  if (!body) {
+    body = "";
+  }
+  const bodyHash = sha256Hash(body);
+  let dedupe = `${request.getMethod}~${request.getUrl()}~${bodyHash}`;
+  dedupeHeaders.forEach((h) => {
+    dedupe += `~${request.getHeader(h)?.join("~")}`
+  })
+  return sha256Hash(dedupe)
+}
+
+const toTemplate = (request: Request, response: Response, templateId: string = generateTemplateId(request)): TemplateDTO => {
   return {
-    id: generateID(),
+    id: templateId,
     requestId: request.getId(),
     authSuccessRegex: `HTTP/1[.]1 ${response.getCode()}`,
     rules: [],

packages/backend/src/stores/analysis.ts

@@ -5,8 +5,11 @@ export class AnalysisStore {
 
   private requests: Map<string, AnalysisRequestDTO>;
 
+  private analysisLookup: Map<string, boolean>;
+
   private constructor() {
     this.requests = new Map();
+    this.analysisLookup = new Map();
   }
 
   static get(): AnalysisStore {
@@ -21,15 +24,38 @@ export class AnalysisStore {
     return [...this.requests.values()];
   }
 
+  getResultHash(templateId: string, userId: string): string {
+    return `${templateId}-${userId}`;
+  }
+
+  resultExists(templateId: string, userId: string): boolean {
+    return this.resultExistsByResultHash(this.getResultHash(templateId, userId));
+  }
+
+  resultExistsByResultHash(resultHash: string): boolean {
+    return this.analysisLookup.get(resultHash) === true;
+  }
+
   addRequest(result: AnalysisRequestDTO) {
+    let resultHash = this.getResultHash(result.templateId, result.userId);
+    if (this.resultExistsByResultHash(resultHash)) {
+      return;
+    }
     this.requests.set(result.id, result);
+    this.analysisLookup.set(resultHash, true);
   }
 
   deleteRequest(requestId: string) {
+    let result = this.requests.get(requestId);
+    if (!result) {
+      return;
+    }
+    this.analysisLookup.delete(this.getResultHash(result.templateId, result.userId));
     this.requests.delete(requestId);
   }
 
   clearRequests() {
     this.requests.clear();
+    this.analysisLookup.clear();
   }
 }

packages/backend/src/stores/settings.ts

@@ -9,6 +9,7 @@ export class SettingsStore {
     this.settings = {
       autoCaptureRequests: "off",
       autoRunAnalysis: true,
+      deDuplicateHeaders: [],
     };
   }
 

packages/backend/src/stores/templates.ts

@@ -1,4 +1,4 @@
-import type { TemplateDTO } from "shared";
+import type { RoleRuleDTO, TemplateDTO, UserRuleDTO } from "shared";
 
 export class TemplateStore {
   private static _store?: TemplateStore;
@@ -21,6 +21,10 @@ export class TemplateStore {
     return [...this.templates.values()];
   }
 
+  templateExists(id: string): boolean {
+    return this.templates.get(id) !== undefined;
+  }
+
   addTemplate(template: TemplateDTO) {
     this.templates.set(template.id, template);
   }
@@ -45,7 +49,7 @@ export class TemplateStore {
       });
 
       if (currRule) {
-        currRule.hasAccess = !currRule.hasAccess;
+        this.toggleRule(currRule);
       } else {
         template.rules.push({
           type: "RoleRule",
@@ -59,6 +63,15 @@ export class TemplateStore {
     return template;
   }
 
+  toggleRule(currRule: RoleRuleDTO | UserRuleDTO) {
+    currRule.hasAccess = !currRule.hasAccess;
+    if (currRule.status === "Bypassed" && currRule.hasAccess) {
+      currRule.status = "Enforced"
+    } else if (currRule.status === "Enforced" && !currRule.hasAccess) {
+      currRule.status = "Bypassed"
+    }
+  }
+
   toggleTemplateUser(templateId: string, userId: string) {
     const template = this.templates.get(templateId);
     if (template) {
@@ -67,7 +80,7 @@ export class TemplateStore {
       });
 
       if (currRule) {
-        currRule.hasAccess = !currRule.hasAccess;
+        this.toggleRule(currRule);
       } else {
         template.rules.push({
           type: "UserRule",

packages/backend/src/utils.ts

@@ -1,3 +1,9 @@
+import { createHash } from 'crypto';
+
+export function sha256Hash(text: string): string {
+    return createHash('sha256').update(text).digest('hex');
+}
+
 export const generateID = () => {
   return (
     Date.now().toString(36) +

packages/shared/src/types.ts

@@ -43,6 +43,7 @@ export type RoleDTO = {
 export type SettingsDTO = {
   autoCaptureRequests: "off" | "all" | "inScope";
   autoRunAnalysis: boolean;
+  deDuplicateHeaders: string[];
 };
 
 export type UserAttributeDTO = {