From 6228ef54353534baf58860cc21fc01366f756999 Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Thu, 6 Mar 2025 20:02:49 +0100 Subject: [PATCH 1/7] refactoring of testcase generation --- .../tests/accessControlTesting.ts | 5 +- src/api/index.ts | 20 ++ src/core/tests/runner/node-test-runner.ts | 39 ++-- src/core/tests/test-executor.ts | 2 +- src/core/tests/test-reporter.ts | 4 +- src/core/tests/testcase-generator.ts | 179 +++++++++++++++++- 6 files changed, 215 insertions(+), 34 deletions(-) diff --git a/demo-application/tests/accessControlTesting.ts b/demo-application/tests/accessControlTesting.ts index bee60f7..2485b2c 100644 --- a/demo-application/tests/accessControlTesting.ts +++ b/demo-application/tests/accessControlTesting.ts @@ -1,5 +1,6 @@ // todo: replace this with from 'access-control-testing' later import { Act, Resource, User } from '../../src/api/index.ts' +import { NodeTestRunner } from '../../src/core/tests/runner/node-test-runner.js' const test = async () => { // todo: seed the database with testing data @@ -24,7 +25,9 @@ const test = async () => { users, resources, }) - await act.scan() + + const testCases = await act.generateTestCases() + NodeTestRunner.run(testCases) } test() diff --git a/src/api/index.ts b/src/api/index.ts index 714a250..c76e708 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,3 +1,4 @@ +import { OpenAPIParser } from "../core/parsers/openapi-parser.js"; import { Resource } from "../core/policy/entities/resource.ts"; import { User } from "../core/policy/entities/user.ts"; import { @@ -5,6 +6,7 @@ import { type TestRunnerIdentifier, } from "../core/tests/runner/test-runner.ts"; import { TestExecutor } from "../core/tests/test-executor.ts"; +import { TestcaseGenerator } from "../core/tests/testcase-generator.js"; type ActOptions = { apiBaseUrl: string; @@ -18,6 +20,7 @@ class Act { // todo: check if both are valid urls } + // todo: deprecate this public async scan() { console.log("Scanning..."); @@ -34,6 +37,23 @@ class Act { this.options.resources, ); } + + public async generateTestCases() { + console.log("Generating testcases..."); + const { openApiUrl, apiBaseUrl, users, resources } = this.options; + + const openAPIParser = await OpenAPIParser.create(openApiUrl, apiBaseUrl); + openAPIParser.validateCustomFields(resources); + + const testCaseGenerator = new TestcaseGenerator(openAPIParser, users); + + return testCaseGenerator.generateTestCases({ + openApiUrl, + apiBaseUrl, + users, + resources, + }); + } } export { Act, User, Resource }; diff --git a/src/core/tests/runner/node-test-runner.ts b/src/core/tests/runner/node-test-runner.ts index 6c0b78e..51e5c12 100644 --- a/src/core/tests/runner/node-test-runner.ts +++ b/src/core/tests/runner/node-test-runner.ts @@ -1,25 +1,20 @@ import assert from "node:assert"; -import { describe, it } from "node:test"; -import { Expectation, TestRunner } from "./test-runner.ts"; +import { test } from "node:test"; +import { TestCase } from "../testcase-generator.ts"; -export class NodeTestRunner implements TestRunner { - group(name: string, callback: () => void) { - describe(name, callback); - } - - test( - name: string, - callback: (t: { skip: (reason?: string) => void }) => Promise | void, - ) { - it(name, async (t) => - callback({ skip: (reason?: string) => t.skip(reason) }), - ); - } - - expect(actual: any): Expectation { - return { - toBe: (expected) => assert.strictEqual(actual, expected), - notToBe: (expected) => assert.notStrictEqual(actual, expected), +export const NodeTestRunner = { + run: (testCases: Array) => { + const expectation = (actual: any) => { + return { + toBe: (expected) => assert.strictEqual(actual, expected), + notToBe: (expected) => assert.notStrictEqual(actual, expected), + }; }; - } -} + + testCases.forEach((testCase) => { + test(testCase.name, () => { + testCase.test(expectation); + }); + }); + }, +}; diff --git a/src/core/tests/test-executor.ts b/src/core/tests/test-executor.ts index 9fb0ce6..81a0b3f 100644 --- a/src/core/tests/test-executor.ts +++ b/src/core/tests/test-executor.ts @@ -43,7 +43,7 @@ export class TestExecutor { openAPIParser.validateCustomFields(resources); const testController = new TestcaseGenerator(openAPIParser, users); - const testCases = testController.generateTestCases(); //.bind(testController); + const testCases = testController.generateTestCombinations(); //.bind(testController); const results: Array = []; const blockedUserIdentifiers: Array = []; diff --git a/src/core/tests/test-reporter.ts b/src/core/tests/test-reporter.ts index e652dd2..7f371dd 100644 --- a/src/core/tests/test-reporter.ts +++ b/src/core/tests/test-reporter.ts @@ -1,5 +1,5 @@ import { BaseReporter } from "@japa/runner/core"; -import type { TestCase } from "./testcase-generator.ts"; +import type { TestCombination } from "./testcase-generator.ts"; // todo: clarify how test report will work export class TestReporter extends BaseReporter { @@ -9,7 +9,7 @@ export class TestReporter extends BaseReporter { const { hasError: testFailed } = testPayload; const testStateRepresentation = testFailed ? "❌" : "✅"; - const testDataset: TestCase = testPayload.dataset.row; + const testDataset: TestCombination = testPayload.dataset.row; const { route, user, expectedRequestToBeAllowed } = testDataset; const { url, method } = route; diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index da343ae..bf1affd 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -1,18 +1,43 @@ import ObjectSet from "object-set-type"; +import { + HTTP_FORBIDDEN_STATUS_CODE, + HTTP_UNAUTHORIZED_STATUS_CODE, +} from "../constants.ts"; import { OpenAPIParser } from "../parsers/openapi-parser.ts"; import { Resource } from "../policy/entities/resource.ts"; import { User } from "../policy/entities/user.ts"; import { PolicyDecisionPoint } from "../policy/policy-decision-point.ts"; import { Action, ResourceIdentifier, ResourceName } from "../policy/types.ts"; import { removeObjectDuplicatesFromArray } from "../utils.ts"; -import { Route } from "./test-utils.ts"; +import { performRequest, Route } from "./test-utils.ts"; -export type TestCase = { +export type TestCombination = { user: User | null; // alternatively: AnonymousUser (extends User) route: Route; expectedRequestToBeAllowed: boolean; }; -export type TestCases = Array; + +type AccessControlResult = "allowed" | "forbidden"; + +export type Expectation = (actual: any) => { + toBe: (expected: any) => void; + notToBe: (expected: any) => void; +}; + +export type TestCase = { + name: string; + test: (expect: Expectation) => void; +}; + +type TestResult = { + user: User | null; + route: Route; + expected: AccessControlResult; + actual?: AccessControlResult; + testResult?: "✅" | "❌" | "⏭️"; +}; + +export type TestCombinations = Array; /* user1.canView(userResource); // can view all Users -> /admin/users & /admin/users/:id @@ -33,7 +58,7 @@ export class TestcaseGenerator { ) {} // todo: this shouldn't be async, solve async in source (OpenAPI parser) - public generateTestCases(): TestCases { + public generateTestCombinations(): TestCombinations { // todo: generate full-formed URLs with parameters // todo: for now only query parameters and path parameters are supported, maybe add support for other types of parameters // resource params mapping @@ -42,8 +67,8 @@ export class TestcaseGenerator { // each url resource mapping has " " pairs with info where to find resource param const resourceUserCombinations = this.generateResourceUserCombinations(); - const testcases: TestCases = pathResourceMappings.flatMap( - (pathResourceMapping) => { + const testcases: TestCombinations = + pathResourceMappings.flatMap((pathResourceMapping) => { // todo: create Route object for url & method to use instead const { path, method, isPublicPath, resources } = pathResourceMapping; @@ -148,8 +173,7 @@ export class TestcaseGenerator { expectedRequestToBeAllowed, }; }); - }, - ); + }); return removeObjectDuplicatesFromArray(testcases); } @@ -199,4 +223,143 @@ export class TestcaseGenerator { return Array.from(resourceUserCombinations); } + + public async generateTestCases({ + openApiUrl, + apiBaseUrl, + users, + resources, + }: { + openApiUrl: string; + apiBaseUrl: string; + users: Array; + resources: Array; + }): Promise> { + const openAPIParser = await OpenAPIParser.create(openApiUrl, apiBaseUrl); + openAPIParser.validateCustomFields(resources); + + const testController = new TestcaseGenerator(openAPIParser, users); + const testCombinations = testController.generateTestCombinations(); //.bind(testController); + + const results: Array = []; + const blockedUserIdentifiers: Array = []; // todo: still working? + + return testCombinations.map((testCombination) => { + const { user, route, expectedRequestToBeAllowed } = testCombination; + + return { + name: `${route} from the perspective of user '${user ?? "anonymous"}'`, + test: async (expect) => { + const expected: AccessControlResult = expectedRequestToBeAllowed + ? "allowed" + : "forbidden"; // todo: make enum for this? + + const testResult: TestResult = { + user, + route, + expected, + testResult: "❌", + }; + results.push(testResult); + + const userHasBeenBlocked = + user !== null && + blockedUserIdentifiers.includes(user.getCredentials().identifier); + if (userHasBeenBlocked) { + testResult.testResult = "⏭️"; + /* t.skip( + `User '${user}' has been blocked since a previous attempt to authenticate failed.`, + );*/ + return; + } + + const isAnonymousUser = user === null; + const credentials = isAnonymousUser ? null : user.getCredentials(); + + const authenticator = openAPIParser.getAuthenticatorByRoute(route); + + let response; + try { + response = await performRequest(route, authenticator, credentials); + } catch (e: unknown) { + // todo: create two Error instances + if (e instanceof Error) { + console.error(e.message); + + console.warn( + `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. + This test will be skipped and further testcases for user '${user}' will be cancelled. + Please check whether you provided correct credentials and the authentication setup is properly configured.`, + ); + + if (user !== null) { + blockedUserIdentifiers.push(user.getCredentials().identifier); + } + + testResult.testResult = "⏭️"; + //t.skip(e.message); + } + + return; + } + + const isUnauthorized = + response.statusCode === HTTP_UNAUTHORIZED_STATUS_CODE; + + /* if (isUnauthorized && !isAnonymousUser) { + // todo: make route toString() + const { retryCount } = response; + const recurringAuthenticationProblem = retryCount > 0; + + console.warn( + `Although the user ${user} has been authenticated using the authentication endpoint, the server responded with status code ${HTTP_UNAUTHORIZED_STATUS_CODE} when trying to access route ${route.method} ${route.url}. + ${recurringAuthenticationProblem ? `The server kept responding with status code ${response.statusCode} after ${retryCount} retries have been made.` : `The server responded with status code ${response.statusCode}.`} + This testcase will be skipped. + Please check whether the credentials are correct and the authentication setup is properly configured.`, + ); + + t.skip( + recurringAuthenticationProblem + ? "Recurring authentication problem" + : "Authentication problem", + ); // todo: new state for skipped? currently only pass/fail + return; + }*/ + + // todo: make it configurable what is considered as forbidden + // for now, forbidden is when the corresponding status code has been sent + const { statusCode } = response; + console.debug("STATUSCODE " + statusCode); + + let actual: AccessControlResult = + statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "forbidden" : "allowed"; + + if (expectedRequestToBeAllowed) { + // can be one of 2XX codes but could also be an error that occurred due to wrong syntax of request + + // todo: what about anonymous users? for them it should not be forbidden and also not unauthorized + expect(statusCode).notToBe(HTTP_FORBIDDEN_STATUS_CODE); + } else { + // as anonymous user, unauthorized or forbidden is okay + if (isAnonymousUser) { + const requestForbidden = [ + HTTP_FORBIDDEN_STATUS_CODE, // todo: is forbidden really expected for users without authentication details or should it only be Unauthorized? + HTTP_UNAUTHORIZED_STATUS_CODE, + ].includes(statusCode); + + expect(requestForbidden).toBe(true); + + actual = requestForbidden ? "forbidden" : "allowed"; // todo: maybe rename to rejected (is either forbidden/unauthorized) + } else { + expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); + } + } + + testResult.actual = actual; + //testResult.authenticator = authenticator; + testResult.testResult = actual === expected ? "✅" : "❌"; + }, + }; + }); + } } From 108a51cc2107e587317a704a31f7b72af532d64e Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Thu, 6 Mar 2025 21:26:24 +0100 Subject: [PATCH 2/7] deprecate scan() method, removed TestExecutor & renamed allowed/forbidden to permitted/denied --- src/api/index.ts | 35 +---- src/core/policy/policy-decision-point.ts | 4 +- src/core/tests/test-executor.ts | 186 ----------------------- src/core/tests/test-reporter.ts | 2 +- src/core/tests/testcase-generator.ts | 59 +++---- tests/mock-server.ts | 6 +- tests/unit/policy-decision-point.spec.ts | 2 +- tests/unit/test-executor.spec.ts | 11 -- 8 files changed, 28 insertions(+), 277 deletions(-) delete mode 100644 src/core/tests/test-executor.ts delete mode 100644 tests/unit/test-executor.spec.ts diff --git a/src/api/index.ts b/src/api/index.ts index c76e708..91ee48c 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,12 +1,8 @@ import { OpenAPIParser } from "../core/parsers/openapi-parser.js"; import { Resource } from "../core/policy/entities/resource.ts"; import { User } from "../core/policy/entities/user.ts"; -import { - TestRunnerFactory, - type TestRunnerIdentifier, -} from "../core/tests/runner/test-runner.ts"; -import { TestExecutor } from "../core/tests/test-executor.ts"; -import { TestcaseGenerator } from "../core/tests/testcase-generator.js"; +import { type TestRunnerIdentifier } from "../core/tests/runner/test-runner.ts"; +import { TestcaseGenerator } from "../core/tests/testcase-generator.ts"; type ActOptions = { apiBaseUrl: string; @@ -15,29 +11,12 @@ type ActOptions = { resources: Array; testRunner?: TestRunnerIdentifier; }; + class Act { constructor(private readonly options: ActOptions) { // todo: check if both are valid urls } - // todo: deprecate this - public async scan() { - console.log("Scanning..."); - - const testRunner = this.options.testRunner - ? TestRunnerFactory.createTestRunner(this.options.testRunner) - : TestRunnerFactory.createTestRunner(); - - const testExecutor = new TestExecutor(); - await testExecutor.runTests( - testRunner, - this.options.openApiUrl, - this.options.apiBaseUrl, - this.options.users, - this.options.resources, - ); - } - public async generateTestCases() { console.log("Generating testcases..."); const { openApiUrl, apiBaseUrl, users, resources } = this.options; @@ -46,13 +25,7 @@ class Act { openAPIParser.validateCustomFields(resources); const testCaseGenerator = new TestcaseGenerator(openAPIParser, users); - - return testCaseGenerator.generateTestCases({ - openApiUrl, - apiBaseUrl, - users, - resources, - }); + return testCaseGenerator.generateTestCases(); } } diff --git a/src/core/policy/policy-decision-point.ts b/src/core/policy/policy-decision-point.ts index b917692..41755e7 100644 --- a/src/core/policy/policy-decision-point.ts +++ b/src/core/policy/policy-decision-point.ts @@ -7,12 +7,12 @@ export class PolicyDecisionPoint { // todo: rename resource with ResourceType, since its not a concrete instance of a Resource? // some resources could be public for all (-> make that configurable) /** - * Decides if a user is allowed to perform an action on a resource based on the user's privileges that are derived from the user's relationships to resources. + * Decides if a user is permitted to perform an action on a resource based on the user's privileges that are derived from the user's relationships to resources. * @param user The user that wants to perform the action * @param action The action the user wants to perform * @param resource The resource object describing the resource the user wants to perform the action on * @param resourceIdentifier The identifier of the specific resource the user wants to perform the action on - * @returns true if the user is allowed to perform the action on the resource, false otherwise + * @returns true if the user is permitted to perform the action on the resource, false otherwise */ public static isAllowed( user: User, diff --git a/src/core/tests/test-executor.ts b/src/core/tests/test-executor.ts deleted file mode 100644 index 81a0b3f..0000000 --- a/src/core/tests/test-executor.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { - HTTP_FORBIDDEN_STATUS_CODE, - HTTP_UNAUTHORIZED_STATUS_CODE, -} from "../constants.ts"; -import { OpenAPIParser } from "../parsers/openapi-parser.ts"; -import { Resource } from "../policy/entities/resource.ts"; -import { User } from "../policy/entities/user.js"; -import { TestRunner } from "./runner/test-runner.ts"; -import { performRequest, Route } from "./test-utils.ts"; -import { TestcaseGenerator } from "./testcase-generator.ts"; - -type AccessControlResult = "allowed" | "forbidden"; - -type TestResult = { - user: User | null; - route: Route; - expected: AccessControlResult; - actual?: AccessControlResult; - testResult?: "✅" | "❌" | "⏭️"; -}; - -export class TestExecutor { - /* private async prepareTestDataset() { - const configurationParser = new ConfigurationParser(); - // todo: no more Configuration -> constructor options? but how to get params into this file? - const { openApiUrl } = await configurationParser.parse(); - - const openAPIParser = await OpenAPIParser.create(openApiUrl); - - const testController = new TestcaseGenerator(openAPIParser); - const dataset: TestCases = testController.generateTestCases(); //.bind(testController); - return dataset; - }*/ - - public async runTests( - testRunner: TestRunner, - openApiUrl: string, - apiBaseUrl: string, - users: Array, - resources: Array, - ) { - const openAPIParser = await OpenAPIParser.create(openApiUrl, apiBaseUrl); - openAPIParser.validateCustomFields(resources); - - const testController = new TestcaseGenerator(openAPIParser, users); - const testCases = testController.generateTestCombinations(); //.bind(testController); - - const results: Array = []; - const blockedUserIdentifiers: Array = []; - - for (const testCase of testCases) { - const { user, route, expectedRequestToBeAllowed } = testCase; - - testRunner.test( - `${route} from the perspective of user '${user ?? "anonymous"}'`, - async (t) => { - const expected: AccessControlResult = expectedRequestToBeAllowed - ? "allowed" - : "forbidden"; // todo: make enum for this? - - const testResult: TestResult = { - user, - route, - expected, - testResult: "❌", - }; - results.push(testResult); - - const userHasBeenBlocked = - user !== null && - blockedUserIdentifiers.includes(user.getCredentials().identifier); - if (userHasBeenBlocked) { - testResult.testResult = "⏭️"; - t.skip( - `User '${user}' has been blocked since a previous attempt to authenticate failed.`, - ); - return; - } - - const isAnonymousUser = user === null; - const credentials = isAnonymousUser ? null : user.getCredentials(); - - const authenticator = openAPIParser.getAuthenticatorByRoute(route); - - let response; - try { - response = await performRequest(route, authenticator, credentials); - } catch (e: unknown) { - // todo: create two Error instances - if (e instanceof Error) { - console.error(e.message); - - console.warn( - `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. - This test will be skipped and further testcases for user '${user}' will be cancelled. - Please check whether you provided correct credentials and the authentication setup is properly configured.`, - ); - - if (user !== null) { - blockedUserIdentifiers.push(user.getCredentials().identifier); - } - - testResult.testResult = "⏭️"; - t.skip(e.message); - } - - return; - } - - const isUnauthorized = - response.statusCode === HTTP_UNAUTHORIZED_STATUS_CODE; - - /* if (isUnauthorized && !isAnonymousUser) { - // todo: make route toString() - const { retryCount } = response; - const recurringAuthenticationProblem = retryCount > 0; - - console.warn( - `Although the user ${user} has been authenticated using the authentication endpoint, the server responded with status code ${HTTP_UNAUTHORIZED_STATUS_CODE} when trying to access route ${route.method} ${route.url}. - ${recurringAuthenticationProblem ? `The server kept responding with status code ${response.statusCode} after ${retryCount} retries have been made.` : `The server responded with status code ${response.statusCode}.`} - This testcase will be skipped. - Please check whether the credentials are correct and the authentication setup is properly configured.`, - ); - - t.skip( - recurringAuthenticationProblem - ? "Recurring authentication problem" - : "Authentication problem", - ); // todo: new state for skipped? currently only pass/fail - return; - }*/ - - // todo: make it configurable what is considered as forbidden - // for now, forbidden is when the corresponding status code has been sent - const { statusCode } = response; - console.debug("STATUSCODE " + statusCode); - - let actual: AccessControlResult = - statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "forbidden" : "allowed"; - - if (expectedRequestToBeAllowed) { - // can be one of 2XX codes but could also be an error that occurred due to wrong syntax of request - - // todo: what about anonymous users? for them it should not be forbidden and also not unauthorized - testRunner.expect(statusCode).notToBe(HTTP_FORBIDDEN_STATUS_CODE); - } else { - // as anonymous user, unauthorized or forbidden is okay - if (isAnonymousUser) { - const requestForbidden = [ - HTTP_FORBIDDEN_STATUS_CODE, // todo: is forbidden really expected for users without authentication details or should it only be Unauthorized? - HTTP_UNAUTHORIZED_STATUS_CODE, - ].includes(statusCode); - - testRunner.expect(requestForbidden).toBe(true); - - actual = requestForbidden ? "forbidden" : "allowed"; // todo: maybe rename to rejected (is either forbidden/unauthorized) - } else { - testRunner.expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); - } - } - - testResult.actual = actual; - //testResult.authenticator = authenticator; - testResult.testResult = actual === expected ? "✅" : "❌"; - }, - ); - } - - TestExecutor.printResults(results); - } - - private static printResults(results: Array) { - process.on("beforeExit", () => { - console.log("\nTest Results:"); - const transformedResults = results.map((result) => ({ - ...result, - user: result.user?.toString() ?? "anonymous", - route: result.route.toString(), - })); - - console.table(transformedResults); - - // todo: enhance this with a detailed report containing all the routes that failed - }); - } -} diff --git a/src/core/tests/test-reporter.ts b/src/core/tests/test-reporter.ts index 7f371dd..de9482e 100644 --- a/src/core/tests/test-reporter.ts +++ b/src/core/tests/test-reporter.ts @@ -29,7 +29,7 @@ export class TestReporter extends BaseReporter { Test Status: ${testStateRepresentation} Request: ${requestRepresentation} User: ${user} -Expection: ${expectedRequestToBeAllowed ? "Should have been allowed" : "Should have been denied"} +Expection: ${expectedRequestToBeAllowed ? "Should have been permitted" : "Should have been denied"} ------------------------------------------- Actual status: ${actual} Expected status: ${expected} diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index bf1affd..51cb5fa 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -12,12 +12,12 @@ import { removeObjectDuplicatesFromArray } from "../utils.ts"; import { performRequest, Route } from "./test-utils.ts"; export type TestCombination = { - user: User | null; // alternatively: AnonymousUser (extends User) + user: User | null; route: Route; expectedRequestToBeAllowed: boolean; }; -type AccessControlResult = "allowed" | "forbidden"; +type AccessControlResult = "permitted" | "denied"; export type Expectation = (actual: any) => { toBe: (expected: any) => void; @@ -39,25 +39,12 @@ type TestResult = { export type TestCombinations = Array; -/* -user1.canView(userResource); // can view all Users -> /admin/users & /admin/users/:id -console.log( - "user1canview", - PolicyDecisionPoint.isAllowed(user1, "read", userResource, 1), // todo: read vs. view? -);*/ -// todo: unit test for scenarios: -// canView(userResource): -// isAllowed(user1, "read", userResource, 1) -> true && isAllowed(user1, "read", userResource) -> true -// canView(userResource, 1): -// isAllowed(user1, "read", userResource, 1) -> true && isAllowed(user1, "read", userResource) -> false - export class TestcaseGenerator { constructor( private readonly openApiParser: OpenAPIParser, private readonly users: Array, ) {} - // todo: this shouldn't be async, solve async in source (OpenAPI parser) public generateTestCombinations(): TestCombinations { // todo: generate full-formed URLs with parameters // todo: for now only query parameters and path parameters are supported, maybe add support for other types of parameters @@ -86,7 +73,7 @@ export class TestcaseGenerator { // no resources available -> default: deny (except for "public" routes) // todo: mark routes as public (for anonymous users) or available for all logged-in users - // routes that are public for anonymous users are expected to be allowed for logged-in users too + // routes that are public for anonymous users are expected to be permitted for logged-in users too if (!routeHasResources) { if (isPublicPath) { @@ -186,6 +173,7 @@ export class TestcaseGenerator { resourceIdentifier?: ResourceIdentifier; }> = new ObjectSet(); + // todo: move this explanation to JSDoc // generate combinations between users, actions, resources and resource ids // for that, go through relations of each user with a resource, // create a test case with expected result of true (for current user) and false (for other users) @@ -218,28 +206,14 @@ export class TestcaseGenerator { // todo: knowledge of resources based on openapi definitions // this is only applicable for non-parameterized resources (e.g. GET /users) because no valid resourceIdentifiers are available // there will be no guessing of valid resourceIdentifiers in that case - // adds additional test cases only when resource/access combination is not already covered by someone who is allowed to access it + // adds additional test cases only when resource/access combination is not already covered by someone who is permitted to access it } return Array.from(resourceUserCombinations); } - public async generateTestCases({ - openApiUrl, - apiBaseUrl, - users, - resources, - }: { - openApiUrl: string; - apiBaseUrl: string; - users: Array; - resources: Array; - }): Promise> { - const openAPIParser = await OpenAPIParser.create(openApiUrl, apiBaseUrl); - openAPIParser.validateCustomFields(resources); - - const testController = new TestcaseGenerator(openAPIParser, users); - const testCombinations = testController.generateTestCombinations(); //.bind(testController); + public async generateTestCases(): Promise> { + const testCombinations = this.generateTestCombinations(); const results: Array = []; const blockedUserIdentifiers: Array = []; // todo: still working? @@ -251,8 +225,8 @@ export class TestcaseGenerator { name: `${route} from the perspective of user '${user ?? "anonymous"}'`, test: async (expect) => { const expected: AccessControlResult = expectedRequestToBeAllowed - ? "allowed" - : "forbidden"; // todo: make enum for this? + ? "permitted" + : "denied"; // todo: make enum for this? const testResult: TestResult = { user, @@ -276,7 +250,8 @@ export class TestcaseGenerator { const isAnonymousUser = user === null; const credentials = isAnonymousUser ? null : user.getCredentials(); - const authenticator = openAPIParser.getAuthenticatorByRoute(route); + const authenticator = + this.openApiParser.getAuthenticatorByRoute(route); let response; try { @@ -303,10 +278,10 @@ export class TestcaseGenerator { return; } - const isUnauthorized = + /* const isUnauthorized = response.statusCode === HTTP_UNAUTHORIZED_STATUS_CODE; - /* if (isUnauthorized && !isAnonymousUser) { + if (isUnauthorized && !isAnonymousUser) { // todo: make route toString() const { retryCount } = response; const recurringAuthenticationProblem = retryCount > 0; @@ -326,13 +301,13 @@ export class TestcaseGenerator { return; }*/ - // todo: make it configurable what is considered as forbidden - // for now, forbidden is when the corresponding status code has been sent + // todo: make it configurable what is considered as denied + // for now, denied is when the corresponding status code (Forbidden) has been sent const { statusCode } = response; console.debug("STATUSCODE " + statusCode); let actual: AccessControlResult = - statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "forbidden" : "allowed"; + statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "denied" : "permitted"; if (expectedRequestToBeAllowed) { // can be one of 2XX codes but could also be an error that occurred due to wrong syntax of request @@ -349,7 +324,7 @@ export class TestcaseGenerator { expect(requestForbidden).toBe(true); - actual = requestForbidden ? "forbidden" : "allowed"; // todo: maybe rename to rejected (is either forbidden/unauthorized) + actual = requestForbidden ? "denied" : "permitted"; // todo: maybe rename to rejected (is either denied/unauthorized) } else { expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); } diff --git a/tests/mock-server.ts b/tests/mock-server.ts index 9cbf1de..962022b 100644 --- a/tests/mock-server.ts +++ b/tests/mock-server.ts @@ -1,7 +1,7 @@ import { createServer } from "node:http"; -import { CookieAuthenticator } from "../src/core/authentication/http/cookie-authenticator.js"; -import { Route } from "../src/core/tests/test-utils.js"; -import { AuthEndpointInformation } from "../src/core/types.js"; +import { CookieAuthenticator } from "../src/core/authentication/http/cookie-authenticator.ts"; +import { Route } from "../src/core/tests/test-utils.ts"; +import { AuthEndpointInformation } from "../src/core/types.ts"; const PORT = 5000; const HOST = "127.0.0.1"; diff --git a/tests/unit/policy-decision-point.spec.ts b/tests/unit/policy-decision-point.spec.ts index 1cfc93d..82af222 100644 --- a/tests/unit/policy-decision-point.spec.ts +++ b/tests/unit/policy-decision-point.spec.ts @@ -5,7 +5,7 @@ import { PolicyDecisionPoint } from "../../src/core/policy/policy-decision-point // todo: how to name tests and test files? test.group("PolicyDecisionPoint.isAllowed", () => { - test("allowed access is allowed", ({ expect }) => { + test("allowed access is permitted", ({ expect }) => { const user1 = new User("user1", "password"); const todoResource = new Resource("Todo"); const resourceIdentifier = 123; diff --git a/tests/unit/test-executor.spec.ts b/tests/unit/test-executor.spec.ts deleted file mode 100644 index 4b2ed64..0000000 --- a/tests/unit/test-executor.spec.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { test } from "@japa/runner"; - -// todo: Japa types only available in NodeNext -// but then type issues with library types come up when not set to ESNext - -test.group("TestExecutor", (group) => { - test("runTests() should print a warning when using invalid credentials"); - test( - "runTests() should print a warning when rate limiting that prevents further tests is detected", - ); -}); From 98ab773331419378ecb59f5274ae1e9ee29ffe52 Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Thu, 6 Mar 2025 22:14:44 +0100 Subject: [PATCH 3/7] added skip() function to test context --- src/api/index.ts | 2 - src/core/tests/runner/node-test-runner.ts | 23 ++++++----- src/core/tests/runner/test-runner.ts | 47 +++++++---------------- src/core/tests/testcase-generator.ts | 19 +++------ 4 files changed, 31 insertions(+), 60 deletions(-) diff --git a/src/api/index.ts b/src/api/index.ts index 91ee48c..deab628 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,7 +1,6 @@ import { OpenAPIParser } from "../core/parsers/openapi-parser.js"; import { Resource } from "../core/policy/entities/resource.ts"; import { User } from "../core/policy/entities/user.ts"; -import { type TestRunnerIdentifier } from "../core/tests/runner/test-runner.ts"; import { TestcaseGenerator } from "../core/tests/testcase-generator.ts"; type ActOptions = { @@ -9,7 +8,6 @@ type ActOptions = { openApiUrl: string; users: Array; resources: Array; - testRunner?: TestRunnerIdentifier; }; class Act { diff --git a/src/core/tests/runner/node-test-runner.ts b/src/core/tests/runner/node-test-runner.ts index 51e5c12..4056f7d 100644 --- a/src/core/tests/runner/node-test-runner.ts +++ b/src/core/tests/runner/node-test-runner.ts @@ -1,19 +1,22 @@ import assert from "node:assert"; import { test } from "node:test"; -import { TestCase } from "../testcase-generator.ts"; +import { TestRunner } from "./test-runner.ts"; -export const NodeTestRunner = { - run: (testCases: Array) => { - const expectation = (actual: any) => { - return { - toBe: (expected) => assert.strictEqual(actual, expected), - notToBe: (expected) => assert.notStrictEqual(actual, expected), - }; - }; +export const NodeTestRunner: TestRunner = { + run: (testCases) => { + const expect = (actual: unknown) => ({ + toBe: (expected: unknown) => assert.strictEqual(actual, expected), + notToBe: (expected: unknown) => assert.notStrictEqual(actual, expected), + }); testCases.forEach((testCase) => { test(testCase.name, () => { - testCase.test(expectation); + testCase.test({ + expect, + skip: (reason) => { + test.skip(reason); + }, + }); }); }); }, diff --git a/src/core/tests/runner/test-runner.ts b/src/core/tests/runner/test-runner.ts index 5e2d991..6ad5f81 100644 --- a/src/core/tests/runner/test-runner.ts +++ b/src/core/tests/runner/test-runner.ts @@ -1,37 +1,16 @@ -import { NodeTestRunner } from "./node-test-runner.ts"; +export type Expectation = (actual: unknown) => { + toBe: (expected: unknown) => void; + notToBe: (expected: unknown) => void; +}; +export type TestCase = { + name: string; + test: (testContext: TestContext) => void; +}; +export type TestContext = { + expect: Expectation; + skip: (reason?: string) => void; +}; -export interface Expectation { - toBe(expected: any): void; - notToBe(expected: any): void; -} - -export interface TestContext { - skip(reason?: string): void; -} - -// todo: expose TestRunner in api so a custom test-runner can be used when implementing this interface export interface TestRunner { - group(name: string, callback: () => void): void; - test(name: string, callback: (t: TestContext) => Promise | void): void; - expect(actual: any): Expectation; -} - -export type TestRunnerIdentifier = "node" | "jest"; - -const DEFAULT_TEST_RUNNER = "node"; -export class TestRunnerFactory { - static createTestRunner( - runner: TestRunnerIdentifier = DEFAULT_TEST_RUNNER, - ): TestRunner { - switch (runner) { - case "node": - return new NodeTestRunner(); - /*case "jest": - return new JestTestRunner(); - case "japa": - return new JapaTestRunner();*/ - default: - throw new Error(`Unsupported test runner: ${runner}`); - } - } + run(testCases: Array): void; } diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index 51cb5fa..2a9c8e3 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -9,6 +9,7 @@ import { User } from "../policy/entities/user.ts"; import { PolicyDecisionPoint } from "../policy/policy-decision-point.ts"; import { Action, ResourceIdentifier, ResourceName } from "../policy/types.ts"; import { removeObjectDuplicatesFromArray } from "../utils.ts"; +import { TestCase } from "./runner/test-runner.ts"; import { performRequest, Route } from "./test-utils.ts"; export type TestCombination = { @@ -19,16 +20,6 @@ export type TestCombination = { type AccessControlResult = "permitted" | "denied"; -export type Expectation = (actual: any) => { - toBe: (expected: any) => void; - notToBe: (expected: any) => void; -}; - -export type TestCase = { - name: string; - test: (expect: Expectation) => void; -}; - type TestResult = { user: User | null; route: Route; @@ -223,7 +214,7 @@ export class TestcaseGenerator { return { name: `${route} from the perspective of user '${user ?? "anonymous"}'`, - test: async (expect) => { + test: async ({ expect, skip }) => { const expected: AccessControlResult = expectedRequestToBeAllowed ? "permitted" : "denied"; // todo: make enum for this? @@ -241,9 +232,9 @@ export class TestcaseGenerator { blockedUserIdentifiers.includes(user.getCredentials().identifier); if (userHasBeenBlocked) { testResult.testResult = "⏭️"; - /* t.skip( + skip( `User '${user}' has been blocked since a previous attempt to authenticate failed.`, - );*/ + ); return; } @@ -272,7 +263,7 @@ export class TestcaseGenerator { } testResult.testResult = "⏭️"; - //t.skip(e.message); + skip(e.message); } return; From fddba4a5bd8399c3646415f42d0cf688c87adbde Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Sat, 8 Mar 2025 23:40:50 +0100 Subject: [PATCH 4/7] fixed eslint, prettier and TS problems --- src/core/policy/policy-decision-point.ts | 4 ++-- src/core/tests/runner/node-test-runner.ts | 14 +++++++------- src/core/tests/test-executor.ts | 0 src/core/tests/testcase-generator.ts | 12 ++++++------ tests/unit/test-executor.spec.ts | 0 5 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 src/core/tests/test-executor.ts delete mode 100644 tests/unit/test-executor.spec.ts diff --git a/src/core/policy/policy-decision-point.ts b/src/core/policy/policy-decision-point.ts index 2808a07..f9ed79e 100644 --- a/src/core/policy/policy-decision-point.ts +++ b/src/core/policy/policy-decision-point.ts @@ -17,8 +17,8 @@ export const PolicyDecisionPoint = { * to perform the action on * @param resourceIdentifier The identifier of the specific resource the user * wants to perform the action on - * @returns True if the user is permitted to perform the action on the resource, - * false otherwise + * @returns True if the user is permitted to perform the action on the + * resource, false otherwise */ isAllowed( user: User, diff --git a/src/core/tests/runner/node-test-runner.ts b/src/core/tests/runner/node-test-runner.ts index 7ccb120..54faf8f 100644 --- a/src/core/tests/runner/node-test-runner.ts +++ b/src/core/tests/runner/node-test-runner.ts @@ -2,19 +2,19 @@ import assert from "node:assert"; import { test } from "node:test"; import type { TestRunner } from "./test-runner.ts"; +const expect = (actual: unknown) => ({ + toBe: (expected: unknown) => assert.strictEqual(actual, expected), + notToBe: (expected: unknown) => assert.notStrictEqual(actual, expected), +}); + export const NodeTestRunner: TestRunner = { run: (testCases) => { - const expect = (actual: unknown) => ({ - toBe: (expected: unknown) => assert.strictEqual(actual, expected), - notToBe: (expected: unknown) => assert.notStrictEqual(actual, expected), - }); - testCases.forEach((testCase) => { - test(testCase.name, () => { + void test(testCase.name, () => { testCase.test({ expect, skip: (reason) => { - test.skip(reason); + void test.skip(reason); }, }); }); diff --git a/src/core/tests/test-executor.ts b/src/core/tests/test-executor.ts deleted file mode 100644 index e69de29..0000000 diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index f64200e..842e0c1 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -13,7 +13,7 @@ import type { ResourceName, } from "../policy/types.ts"; import { removeObjectDuplicatesFromArray } from "../utils.ts"; -import { TestCase } from "./runner/test-runner.ts"; +import type { TestCase } from "./runner/test-runner.ts"; import { performRequest, Route } from "./test-utils.ts"; export type TestCombination = { @@ -208,7 +208,7 @@ export class TestcaseGenerator { return [...resourceUserCombinations]; } - public async generateTestCases(): Promise> { + async generateTestCases(): Promise> { const testCombinations = this.generateTestCombinations(); const results: Array = []; @@ -252,10 +252,10 @@ export class TestcaseGenerator { let response; try { response = await performRequest(route, authenticator, credentials); - } catch (e: unknown) { + } catch (error: unknown) { // todo: create two Error instances - if (e instanceof Error) { - console.error(e.message); + if (error instanceof Error) { + console.error(error.message); console.warn( `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. @@ -268,7 +268,7 @@ export class TestcaseGenerator { } testResult.testResult = "⏭️"; - skip(e.message); + skip(error.message); } return; diff --git a/tests/unit/test-executor.spec.ts b/tests/unit/test-executor.spec.ts deleted file mode 100644 index e69de29..0000000 From dc23e8713af91dba85511eda2653c02fa48b1e7b Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Mon, 10 Mar 2025 20:32:49 +0100 Subject: [PATCH 5/7] added table report back --- .../tests/accessControlTesting.ts | 6 +-- src/core/tests/runner/node-test-runner.ts | 36 +++++++++------- src/core/tests/runner/test-runner.ts | 41 +++++++++++++++++-- src/core/tests/testcase-generator.ts | 16 +++----- 4 files changed, 67 insertions(+), 32 deletions(-) diff --git a/demo-application/tests/accessControlTesting.ts b/demo-application/tests/accessControlTesting.ts index 2485b2c..7d9e941 100644 --- a/demo-application/tests/accessControlTesting.ts +++ b/demo-application/tests/accessControlTesting.ts @@ -1,6 +1,5 @@ -// todo: replace this with from 'access-control-testing' later import { Act, Resource, User } from '../../src/api/index.ts' -import { NodeTestRunner } from '../../src/core/tests/runner/node-test-runner.js' +import { NodeTestRunner } from '../../src/core/tests/runner/node-test-runner.ts' const test = async () => { // todo: seed the database with testing data @@ -27,7 +26,8 @@ const test = async () => { }) const testCases = await act.generateTestCases() - NodeTestRunner.run(testCases) + const testRunner = new NodeTestRunner() + await testRunner.run(testCases) } test() diff --git a/src/core/tests/runner/node-test-runner.ts b/src/core/tests/runner/node-test-runner.ts index 54faf8f..fa9b4cd 100644 --- a/src/core/tests/runner/node-test-runner.ts +++ b/src/core/tests/runner/node-test-runner.ts @@ -1,23 +1,29 @@ import assert from "node:assert"; import { test } from "node:test"; -import type { TestRunner } from "./test-runner.ts"; +import { TestRunner, type TestCase } from "./test-runner.ts"; const expect = (actual: unknown) => ({ toBe: (expected: unknown) => assert.strictEqual(actual, expected), notToBe: (expected: unknown) => assert.notStrictEqual(actual, expected), }); -export const NodeTestRunner: TestRunner = { - run: (testCases) => { - testCases.forEach((testCase) => { - void test(testCase.name, () => { - testCase.test({ - expect, - skip: (reason) => { - void test.skip(reason); - }, - }); - }); - }); - }, -}; +export class NodeTestRunner extends TestRunner { + override async run(testCases: Array) { + await Promise.all( + testCases.map((testCase) => + test(testCase.name, async () => { + const testResult = await testCase.test({ + expect, + skip: (reason) => { + void test.skip(reason); + }, + }); + + if (testResult) { + this.testResults.push(testResult); + } + }), + ), + ); + } +} diff --git a/src/core/tests/runner/test-runner.ts b/src/core/tests/runner/test-runner.ts index 6ad5f81..ec405ce 100644 --- a/src/core/tests/runner/test-runner.ts +++ b/src/core/tests/runner/test-runner.ts @@ -1,16 +1,51 @@ +import { User } from "../../policy/entities/user"; +import { Route } from "../test-utils"; + +type AccessControlResult = "permitted" | "denied"; + +export type TestResult = { + user: User | null; + route: Route; + expected: AccessControlResult; + actual?: AccessControlResult; + testResult?: "✅" | "❌" | "⏭️"; +}; + export type Expectation = (actual: unknown) => { toBe: (expected: unknown) => void; notToBe: (expected: unknown) => void; }; + export type TestCase = { name: string; - test: (testContext: TestContext) => void; + test: (testContext: TestContext) => Promise; }; + export type TestContext = { expect: Expectation; skip: (reason?: string) => void; }; -export interface TestRunner { - run(testCases: Array): void; +export abstract class TestRunner { + protected testResults: Array = []; + + constructor() { + process.on("beforeExit", () => this.printReport()); + } + + abstract run(testCases: Array): Promise; + + protected printReport(): void { + console.log("\n=== Test Report ==="); + + const transformedResults = this.testResults.map((result) => ({ + ...result, + user: result.user?.toString() ?? "anonymous", + route: result.route.toString(), + })); + + console.table(transformedResults); + + // todo: enhance this with a detailed report containing all the routes that failed + } } diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index 842e0c1..4453b5b 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -13,7 +13,7 @@ import type { ResourceName, } from "../policy/types.ts"; import { removeObjectDuplicatesFromArray } from "../utils.ts"; -import type { TestCase } from "./runner/test-runner.ts"; +import type { TestCase, TestResult } from "./runner/test-runner.ts"; import { performRequest, Route } from "./test-utils.ts"; export type TestCombination = { @@ -24,14 +24,6 @@ export type TestCombination = { type AccessControlResult = "permitted" | "denied"; -type TestResult = { - user: User | null; - route: Route; - expected: AccessControlResult; - actual?: AccessControlResult; - testResult?: "✅" | "❌" | "⏭️"; -}; - export type TestCombinations = Array; export class TestcaseGenerator { @@ -211,7 +203,7 @@ export class TestcaseGenerator { async generateTestCases(): Promise> { const testCombinations = this.generateTestCombinations(); - const results: Array = []; + //const results: Array = []; const blockedUserIdentifiers: Array = []; // todo: still working? return testCombinations.map((testCombination) => { @@ -230,7 +222,7 @@ export class TestcaseGenerator { expected, testResult: "❌", }; - results.push(testResult); + //results.push(testResult); const userHasBeenBlocked = user !== null && @@ -329,6 +321,8 @@ export class TestcaseGenerator { testResult.actual = actual; //testResult.authenticator = authenticator; testResult.testResult = actual === expected ? "✅" : "❌"; + + return testResult; }, }; }); From a16dcd7460ca7e75d6d2c08b8d4f0736c2424477 Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Mon, 10 Mar 2025 21:03:32 +0100 Subject: [PATCH 6/7] moved test function to TestCaseExecutor --- src/core/tests/runner/test-runner.ts | 8 +- src/core/tests/testcase-executor.ts | 92 ++++++++++++++++++ src/core/tests/testcase-generator.ts | 140 ++------------------------- 3 files changed, 107 insertions(+), 133 deletions(-) create mode 100644 src/core/tests/testcase-executor.ts diff --git a/src/core/tests/runner/test-runner.ts b/src/core/tests/runner/test-runner.ts index ec405ce..aedc4c4 100644 --- a/src/core/tests/runner/test-runner.ts +++ b/src/core/tests/runner/test-runner.ts @@ -1,7 +1,7 @@ import { User } from "../../policy/entities/user"; import { Route } from "../test-utils"; -type AccessControlResult = "permitted" | "denied"; +export type AccessControlResult = "permitted" | "denied"; export type TestResult = { user: User | null; @@ -16,9 +16,13 @@ export type Expectation = (actual: unknown) => { notToBe: (expected: unknown) => void; }; +export type TestCaseFunction = ( + testContext: TestContext, +) => Promise; + export type TestCase = { name: string; - test: (testContext: TestContext) => Promise; + test: TestCaseFunction; }; export type TestContext = { diff --git a/src/core/tests/testcase-executor.ts b/src/core/tests/testcase-executor.ts new file mode 100644 index 0000000..50741e1 --- /dev/null +++ b/src/core/tests/testcase-executor.ts @@ -0,0 +1,92 @@ +import { + HTTP_FORBIDDEN_STATUS_CODE, + HTTP_UNAUTHORIZED_STATUS_CODE, +} from "../constants.ts"; +import { OpenAPIParser } from "../parsers/openapi-parser.ts"; +import type { User } from "../policy/entities/user.ts"; +import type { + AccessControlResult, + TestContext, + TestResult, +} from "./runner/test-runner.ts"; +import { performRequest } from "./test-utils.ts"; +import type { TestCombination } from "./testcase-generator.ts"; + +export class TestCaseExecutor { + private blockedUserIdentifiers: Array = []; + + constructor(private openApiParser: OpenAPIParser) {} + + async execute( + testCombination: TestCombination, + { expect, skip }: TestContext, + ): Promise { + const { user, route, expectedRequestToBeAllowed } = testCombination; + const expected: AccessControlResult = expectedRequestToBeAllowed + ? "permitted" + : "denied"; + const testResult: TestResult = { user, route, expected, testResult: "❌" }; + + const userHasBeenBlocked = + user !== null && + this.blockedUserIdentifiers.includes(user.getCredentials().identifier); + if (userHasBeenBlocked) { + testResult.testResult = "⏭️"; + skip( + `User '${user}' has been blocked since a previous attempt to authenticate failed.`, + ); + return testResult; + } + + const isAnonymousUser = user === null; + const credentials = isAnonymousUser ? null : user.getCredentials(); + const authenticator = this.openApiParser.getAuthenticatorByRoute(route); + + let response; + try { + response = await performRequest(route, authenticator, credentials); + } catch (error: unknown) { + if (error instanceof Error) { + console.error(error.message); + + console.warn( + `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. + This test will be skipped and further testcases for user '${user}' will be cancelled. + Please check whether you provided correct credentials and the authentication setup is properly configured.`, + ); + + if (user !== null) { + this.blockedUserIdentifiers.push(user.getCredentials().identifier); + } + + testResult.testResult = "⏭️"; + skip(error.message); + } + return testResult; + } + + const { statusCode } = response; + console.debug("STATUSCODE " + statusCode); + let actual: AccessControlResult = + statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "denied" : "permitted"; + + if (expectedRequestToBeAllowed) { + expect(statusCode).notToBe(HTTP_FORBIDDEN_STATUS_CODE); + } else { + if (isAnonymousUser) { + const requestForbidden = [ + HTTP_FORBIDDEN_STATUS_CODE, + HTTP_UNAUTHORIZED_STATUS_CODE, + ].includes(statusCode); + expect(requestForbidden).toBe(true); + actual = requestForbidden ? "denied" : "permitted"; + } else { + expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); + } + } + + testResult.actual = actual; + testResult.testResult = actual === expected ? "✅" : "❌"; + return testResult; + } +} diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index 4453b5b..cd0a010 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -1,8 +1,4 @@ import ObjectSet from "object-set-type"; -import { - HTTP_FORBIDDEN_STATUS_CODE, - HTTP_UNAUTHORIZED_STATUS_CODE, -} from "../constants.ts"; import { OpenAPIParser } from "../parsers/openapi-parser.ts"; import { Resource } from "../policy/entities/resource.ts"; import { User } from "../policy/entities/user.ts"; @@ -13,8 +9,9 @@ import type { ResourceName, } from "../policy/types.ts"; import { removeObjectDuplicatesFromArray } from "../utils.ts"; -import type { TestCase, TestResult } from "./runner/test-runner.ts"; -import { performRequest, Route } from "./test-utils.ts"; +import type { TestCase } from "./runner/test-runner.ts"; +import { Route } from "./test-utils.ts"; +import { TestCaseExecutor } from "./testcase-executor.ts"; export type TestCombination = { user: User | null; @@ -22,8 +19,6 @@ export type TestCombination = { expectedRequestToBeAllowed: boolean; }; -type AccessControlResult = "permitted" | "denied"; - export type TestCombinations = Array; export class TestcaseGenerator { @@ -202,129 +197,12 @@ export class TestcaseGenerator { async generateTestCases(): Promise> { const testCombinations = this.generateTestCombinations(); + const testCaseExecutor = new TestCaseExecutor(this.openApiParser); - //const results: Array = []; - const blockedUserIdentifiers: Array = []; // todo: still working? - - return testCombinations.map((testCombination) => { - const { user, route, expectedRequestToBeAllowed } = testCombination; - - return { - name: `${route} from the perspective of user '${user ?? "anonymous"}'`, - test: async ({ expect, skip }) => { - const expected: AccessControlResult = expectedRequestToBeAllowed - ? "permitted" - : "denied"; // todo: make enum for this? - - const testResult: TestResult = { - user, - route, - expected, - testResult: "❌", - }; - //results.push(testResult); - - const userHasBeenBlocked = - user !== null && - blockedUserIdentifiers.includes(user.getCredentials().identifier); - if (userHasBeenBlocked) { - testResult.testResult = "⏭️"; - skip( - `User '${user}' has been blocked since a previous attempt to authenticate failed.`, - ); - return; - } - - const isAnonymousUser = user === null; - const credentials = isAnonymousUser ? null : user.getCredentials(); - - const authenticator = - this.openApiParser.getAuthenticatorByRoute(route); - - let response; - try { - response = await performRequest(route, authenticator, credentials); - } catch (error: unknown) { - // todo: create two Error instances - if (error instanceof Error) { - console.error(error.message); - - console.warn( - `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. - This test will be skipped and further testcases for user '${user}' will be cancelled. - Please check whether you provided correct credentials and the authentication setup is properly configured.`, - ); - - if (user !== null) { - blockedUserIdentifiers.push(user.getCredentials().identifier); - } - - testResult.testResult = "⏭️"; - skip(error.message); - } - - return; - } - - /* const isUnauthorized = - response.statusCode === HTTP_UNAUTHORIZED_STATUS_CODE; - - if (isUnauthorized && !isAnonymousUser) { - // todo: make route toString() - const { retryCount } = response; - const recurringAuthenticationProblem = retryCount > 0; - - console.warn( - `Although the user ${user} has been authenticated using the authentication endpoint, the server responded with status code ${HTTP_UNAUTHORIZED_STATUS_CODE} when trying to access route ${route.method} ${route.url}. - ${recurringAuthenticationProblem ? `The server kept responding with status code ${response.statusCode} after ${retryCount} retries have been made.` : `The server responded with status code ${response.statusCode}.`} - This testcase will be skipped. - Please check whether the credentials are correct and the authentication setup is properly configured.`, - ); - - t.skip( - recurringAuthenticationProblem - ? "Recurring authentication problem" - : "Authentication problem", - ); // todo: new state for skipped? currently only pass/fail - return; - }*/ - - // todo: make it configurable what is considered as denied - // for now, denied is when the corresponding status code (Forbidden) has been sent - const { statusCode } = response; - console.debug("STATUSCODE " + statusCode); - - let actual: AccessControlResult = - statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "denied" : "permitted"; - - if (expectedRequestToBeAllowed) { - // can be one of 2XX codes but could also be an error that occurred due to wrong syntax of request - - // todo: what about anonymous users? for them it should not be forbidden and also not unauthorized - expect(statusCode).notToBe(HTTP_FORBIDDEN_STATUS_CODE); - } else { - // as anonymous user, unauthorized or forbidden is okay - if (isAnonymousUser) { - const requestForbidden = [ - HTTP_FORBIDDEN_STATUS_CODE, // todo: is forbidden really expected for users without authentication details or should it only be Unauthorized? - HTTP_UNAUTHORIZED_STATUS_CODE, - ].includes(statusCode); - - expect(requestForbidden).toBe(true); - - actual = requestForbidden ? "denied" : "permitted"; // todo: maybe rename to rejected (is either denied/unauthorized) - } else { - expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); - } - } - - testResult.actual = actual; - //testResult.authenticator = authenticator; - testResult.testResult = actual === expected ? "✅" : "❌"; - - return testResult; - }, - }; - }); + return testCombinations.map((testCombination) => ({ + name: `${testCombination.route} from the perspective of user '${testCombination.user ?? "anonymous"}'`, + test: async (testContext) => + testCaseExecutor.execute(testCombination, testContext), + })); } } From 3cc8097565dbd29b08c9fa688366df17bccb9cf4 Mon Sep 17 00:00:00 2001 From: Niklas Haug Date: Mon, 10 Mar 2025 21:10:27 +0100 Subject: [PATCH 7/7] added explanation for skip in TextContext --- src/core/tests/runner/test-runner.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/tests/runner/test-runner.ts b/src/core/tests/runner/test-runner.ts index aedc4c4..1e449eb 100644 --- a/src/core/tests/runner/test-runner.ts +++ b/src/core/tests/runner/test-runner.ts @@ -27,6 +27,10 @@ export type TestCase = { export type TestContext = { expect: Expectation; + /** + * Skip the current test case. For test runners that do not support skipping + * an already running test case, an appropriate warning should be displayed. + */ skip: (reason?: string) => void; };