From 289ad44b9084323b937df3763e43ce3865d6b869 Mon Sep 17 00:00:00 2001 From: splinter Date: Sat, 26 Oct 2024 19:09:20 +0200 Subject: [PATCH 1/3] Add some simple de-duplication logic --- packages/backend/src/services/analysis.ts | 3 +++ packages/backend/src/services/templates.ts | 27 ++++++++++++++++------ packages/backend/src/stores/analysis.ts | 26 +++++++++++++++++++++ packages/backend/src/stores/templates.ts | 4 ++++ packages/backend/src/utils.ts | 6 +++++ 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/services/analysis.ts b/packages/backend/src/services/analysis.ts index 8dcbb70..6242a68 100644 --- a/packages/backend/src/services/analysis.ts +++ b/packages/backend/src/services/analysis.ts @@ -66,6 +66,9 @@ export const runAnalysis = async (sdk: SDK) => { for (const template of templates) { for (const user of users) { + if (analysisStore.resultExists(template.id, user.id)) { + continue; + } const analysisRequest = await sendRequest(sdk, template, user); if (analysisRequest) { analysisStore.addRequest(analysisRequest); diff --git a/packages/backend/src/services/templates.ts b/packages/backend/src/services/templates.ts index bbdf798..f4fa807 100644 --- a/packages/backend/src/services/templates.ts +++ b/packages/backend/src/services/templates.ts @@ -3,7 +3,7 @@ import type { Request, Response } from "caido:utils"; import type { TemplateDTO } from "shared"; import { TemplateStore } from "../stores/templates"; -import { generateID } from "../utils"; +import { generateID, sha256Hash } from "../utils"; import { SettingsStore } from "../stores/settings"; import type { BackendEvents } from "../types"; @@ -89,18 +89,25 @@ export const onInterceptResponse = async ( const settings = settingsStore.getSettings(); const store = TemplateStore.get(); + if (settings.autoCaptureRequests == "off") { + return; + } + + const templateId = generateTemplateId(request); + if (store.templateExists(templateId)) { + return + } + switch (settings.autoCaptureRequests) { - case "off": - return; case "all": { - const template = toTemplate(request, response); + const template = toTemplate(request, response, templateId); store.addTemplate(template); sdk.api.send("templates:created", template); break; } case "inScope": { if (sdk.requests.inScope(request)) { - const template = toTemplate(request, response); + const template = toTemplate(request, response, templateId); store.addTemplate(template); sdk.api.send("templates:created", template); } @@ -116,9 +123,15 @@ export const registerTemplateEvents = (sdk: SDK) => { sdk.events.onInterceptResponse(onInterceptResponse); }; -const toTemplate = (request: Request, response: Response): TemplateDTO => { + +const generateTemplateId = (request: Request): string => { + // Should replace to perhaps exclude and include different parts of the request + return sha256Hash(request.getRaw().toText()) +} + +const toTemplate = (request: Request, response: Response, templateId: string = generateTemplateId(request)): TemplateDTO => { return { - id: generateID(), + id: templateId, requestId: request.getId(), authSuccessRegex: `HTTP/1[.]1 ${response.getCode()}`, rules: [], diff --git a/packages/backend/src/stores/analysis.ts b/packages/backend/src/stores/analysis.ts index 2ae2412..1f7f265 100644 --- a/packages/backend/src/stores/analysis.ts +++ b/packages/backend/src/stores/analysis.ts @@ -5,8 +5,11 @@ export class AnalysisStore { private requests: Map; + private analysisLookup: Map; + private constructor() { this.requests = new Map(); + this.analysisLookup = new Map(); } static get(): AnalysisStore { @@ -21,15 +24,38 @@ export class AnalysisStore { return [...this.requests.values()]; } + getResultHash(templateId: string, userId: string): string { + return `${templateId}-${userId}`; + } + + resultExists(templateId: string, userId: string): boolean { + return this.resultExistsByResultHash(this.getResultHash(templateId, userId)); + } + + resultExistsByResultHash(resultHash: string): boolean { + return this.analysisLookup.get(resultHash) === true; + } + addRequest(result: AnalysisRequestDTO) { + let resultHash = this.getResultHash(result.templateId, result.userId); + if (this.resultExistsByResultHash(resultHash)) { + return; + } this.requests.set(result.id, result); + this.analysisLookup.set(resultHash, true); } deleteRequest(requestId: string) { + let result = this.requests.get(requestId); + if (!result) { + return; + } + this.analysisLookup.delete(this.getResultHash(result.templateId, result.userId)); this.requests.delete(requestId); } clearRequests() { this.requests.clear(); + this.analysisLookup.clear(); } } diff --git a/packages/backend/src/stores/templates.ts b/packages/backend/src/stores/templates.ts index fc6f549..7049517 100644 --- a/packages/backend/src/stores/templates.ts +++ b/packages/backend/src/stores/templates.ts @@ -21,6 +21,10 @@ export class TemplateStore { return [...this.templates.values()]; } + templateExists(id: string): boolean { + return this.templates.get(id) !== undefined; + } + addTemplate(template: TemplateDTO) { this.templates.set(template.id, template); } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index e4b64da..e16ef48 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,3 +1,9 @@ +import { createHash } from 'crypto'; + +export function sha256Hash(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + export const generateID = () => { return ( Date.now().toString(36) + From 8e11dd669d4684749d6908f3b0fbafd7b7a7095a Mon Sep 17 00:00:00 2001 From: splinter Date: Sun, 27 Oct 2024 15:52:19 +0100 Subject: [PATCH 2/3] Improve dedupe logic and add option for dedupe by headers from settings --- packages/backend/src/services/templates.ts | 16 ++++++++++++---- packages/backend/src/stores/settings.ts | 1 + packages/shared/src/types.ts | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/services/templates.ts b/packages/backend/src/services/templates.ts index f4fa807..ac9d643 100644 --- a/packages/backend/src/services/templates.ts +++ b/packages/backend/src/services/templates.ts @@ -93,7 +93,7 @@ export const onInterceptResponse = async ( return; } - const templateId = generateTemplateId(request); + const templateId = generateTemplateId(request, settings.deDuplicateHeaders); if (store.templateExists(templateId)) { return } @@ -124,9 +124,17 @@ export const registerTemplateEvents = (sdk: SDK) => { }; -const generateTemplateId = (request: Request): string => { - // Should replace to perhaps exclude and include different parts of the request - return sha256Hash(request.getRaw().toText()) +const generateTemplateId = (request: Request, dedupeHeaders: string[] = []): string => { + let body = request.getBody()?.toText(); + if (!body) { + body = ""; + } + const bodyHash = sha256Hash(body); + let dedupe = `${request.getMethod}~${request.getUrl()}~${bodyHash}`; + dedupeHeaders.forEach((h) => { + dedupe += `~${request.getHeader(h)?.join("~")}` + }) + return sha256Hash(dedupe) } const toTemplate = (request: Request, response: Response, templateId: string = generateTemplateId(request)): TemplateDTO => { diff --git a/packages/backend/src/stores/settings.ts b/packages/backend/src/stores/settings.ts index 4070058..cd368cf 100644 --- a/packages/backend/src/stores/settings.ts +++ b/packages/backend/src/stores/settings.ts @@ -9,6 +9,7 @@ export class SettingsStore { this.settings = { autoCaptureRequests: "off", autoRunAnalysis: true, + deDuplicateHeaders: [], }; } diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 5c732bb..6a5bd88 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -43,6 +43,7 @@ export type RoleDTO = { export type SettingsDTO = { autoCaptureRequests: "off" | "all" | "inScope"; autoRunAnalysis: boolean; + deDuplicateHeaders: string[]; }; export type UserAttributeDTO = { From b2645a2801f64b9c4ff4338bea5b44952940dec4 Mon Sep 17 00:00:00 2001 From: splinter Date: Sun, 27 Oct 2024 17:32:41 +0100 Subject: [PATCH 3/3] Performance optimization and fix status toggling --- packages/backend/src/services/analysis.ts | 114 ++++++++++++---------- packages/backend/src/stores/templates.ts | 15 ++- 2 files changed, 72 insertions(+), 57 deletions(-) diff --git a/packages/backend/src/services/analysis.ts b/packages/backend/src/services/analysis.ts index 6242a68..7e8c2ff 100644 --- a/packages/backend/src/services/analysis.ts +++ b/packages/backend/src/services/analysis.ts @@ -65,62 +65,68 @@ export const runAnalysis = async (sdk: SDK) => { ); for (const template of templates) { - for (const user of users) { - if (analysisStore.resultExists(template.id, user.id)) { - continue; - } - const analysisRequest = await sendRequest(sdk, template, user); - if (analysisRequest) { - analysisStore.addRequest(analysisRequest); - sdk.api.send("results:created", analysisRequest); + // Run each template async + (async () => { + for (const user of users) { + if (analysisStore.resultExists(template.id, user.id)) { + continue; + } + const analysisRequest = await sendRequest(sdk, template, user); + if (analysisRequest) { + analysisStore.addRequest(analysisRequest); + sdk.api.send("results:created", analysisRequest); + } } - } - } - const roles = roleStore.getRoles(); - for (const template of templates) { - const newRules: TemplateDTO["rules"] = []; - - // Generate role rule statuses in parallel - const rolePromises = roles.map(async (role) => { - const currentRule = template.rules.find( - (rule) => rule.type === "RoleRule" && rule.roleId === role.id, - ) ?? { - type: "RoleRule", - roleId: role.id, - hasAccess: false, - status: "Untested", - }; - - const status = await generateRoleRuleStatus(sdk, template, role.id); - return { ...currentRule, status }; - }); - - // Generate user rule statuses in parallel - const userPromises = users.map(async (user) => { - const currentRule = template.rules.find( - (rule) => rule.type === "UserRule" && rule.userId === user.id, - ) ?? { - type: "UserRule", - userId: user.id, - hasAccess: false, - status: "Untested", - }; - - const status = await generateUserRuleStatus(sdk, template, user); - return { ...currentRule, status }; - }); - - // Await all role and user statuses - const roleResults = await Promise.all(rolePromises); - const userResults = await Promise.all(userPromises); - - // Combine results - newRules.push(...roleResults, ...userResults); - - template.rules = newRules; - templateStore.updateTemplate(template.id, template); - sdk.api.send("templates:updated", template); + const newRules: TemplateDTO["rules"] = []; + const roles = roleStore.getRoles(); + const rolePromises = roles.map(async (role) => { + const currentRule = template.rules.find( + (rule) => rule.type === "RoleRule" && rule.roleId === role.id, + ) ?? { + type: "RoleRule", + roleId: role.id, + hasAccess: false, + status: "Untested", + }; + + + if (currentRule.status !== "Untested") { + return currentRule; + } + + const status = await generateRoleRuleStatus(sdk, template, role.id); + return { ...currentRule, status }; + }); + + const userPromises = users.map(async (user) => { + const currentRule = template.rules.find( + (rule) => rule.type === "UserRule" && rule.userId === user.id, + ) ?? { + type: "UserRule", + userId: user.id, + hasAccess: false, + status: "Untested", + }; + + if (currentRule.status !== "Untested") { + return currentRule; + } + + const status = await generateUserRuleStatus(sdk, template, user); + return { ...currentRule, status }; + }); + + const roleResults = await Promise.all(rolePromises); + const userResults = await Promise.all(userPromises); + + // Combine results + newRules.push(...roleResults, ...userResults); + + template.rules = newRules; + templateStore.updateTemplate(template.id, template); + sdk.api.send("templates:updated", template); + })() } }; diff --git a/packages/backend/src/stores/templates.ts b/packages/backend/src/stores/templates.ts index 7049517..54bea4c 100644 --- a/packages/backend/src/stores/templates.ts +++ b/packages/backend/src/stores/templates.ts @@ -1,4 +1,4 @@ -import type { TemplateDTO } from "shared"; +import type { RoleRuleDTO, TemplateDTO, UserRuleDTO } from "shared"; export class TemplateStore { private static _store?: TemplateStore; @@ -49,7 +49,7 @@ export class TemplateStore { }); if (currRule) { - currRule.hasAccess = !currRule.hasAccess; + this.toggleRule(currRule); } else { template.rules.push({ type: "RoleRule", @@ -63,6 +63,15 @@ export class TemplateStore { return template; } + toggleRule(currRule: RoleRuleDTO | UserRuleDTO) { + currRule.hasAccess = !currRule.hasAccess; + if (currRule.status === "Bypassed" && currRule.hasAccess) { + currRule.status = "Enforced" + } else if (currRule.status === "Enforced" && !currRule.hasAccess) { + currRule.status = "Bypassed" + } + } + toggleTemplateUser(templateId: string, userId: string) { const template = this.templates.get(templateId); if (template) { @@ -71,7 +80,7 @@ export class TemplateStore { }); if (currRule) { - currRule.hasAccess = !currRule.hasAccess; + this.toggleRule(currRule); } else { template.rules.push({ type: "UserRule",