diff --git a/demo-application/tests/accessControlTesting.ts b/demo-application/tests/accessControlTesting.ts index bee60f7..7d9e941 100644 --- a/demo-application/tests/accessControlTesting.ts +++ b/demo-application/tests/accessControlTesting.ts @@ -1,5 +1,5 @@ -// todo: replace this with from 'access-control-testing' later import { Act, Resource, User } from '../../src/api/index.ts' +import { NodeTestRunner } from '../../src/core/tests/runner/node-test-runner.ts' const test = async () => { // todo: seed the database with testing data @@ -24,7 +24,10 @@ const test = async () => { users, resources, }) - await act.scan() + + const testCases = await act.generateTestCases() + const testRunner = new NodeTestRunner() + await testRunner.run(testCases) } test() diff --git a/src/api/index.ts b/src/api/index.ts index 009aacf..839845f 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,38 +1,29 @@ +import { OpenAPIParser } from "../core/parsers/openapi-parser.ts"; import { Resource } from "../core/policy/entities/resource.ts"; import { User } from "../core/policy/entities/user.ts"; -import { - TestRunnerFactory, - type TestRunnerIdentifier, -} from "../core/tests/runner/test-runner.ts"; -import { TestExecutor } from "../core/tests/test-executor.ts"; +import { TestcaseGenerator } from "../core/tests/testcase-generator.ts"; type ActOptions = { apiBaseUrl: string; openApiUrl: string; users: Array; resources: Array; - testRunner?: TestRunnerIdentifier; }; + class Act { constructor(private readonly options: ActOptions) { // todo: check if both are valid urls } - async scan() { - console.log("Scanning..."); + async generateTestCases() { + console.log("Generating testcases..."); + const { openApiUrl, apiBaseUrl, users, resources } = this.options; - const testRunner = this.options.testRunner - ? TestRunnerFactory.createTestRunner(this.options.testRunner) - : TestRunnerFactory.createTestRunner(); + const openAPIParser = await OpenAPIParser.create(openApiUrl, apiBaseUrl); + openAPIParser.validateCustomFields(resources); - const testExecutor = new TestExecutor(); - await testExecutor.runTests( - testRunner, - this.options.openApiUrl, - this.options.apiBaseUrl, - this.options.users, - this.options.resources, - ); + const testCaseGenerator = new TestcaseGenerator(openAPIParser, users); + return testCaseGenerator.generateTestCases(); } } diff --git a/src/core/policy/policy-decision-point.ts b/src/core/policy/policy-decision-point.ts index cd4e109..fe78167 100644 --- a/src/core/policy/policy-decision-point.ts +++ b/src/core/policy/policy-decision-point.ts @@ -6,7 +6,7 @@ import { type Action, type ResourceIdentifier } from "./types.ts"; export const PolicyDecisionPoint = { // todo: some resources could be public for all (-> make that configurable) /** - * Decides if a user is allowed to perform an action on a resource based on + * Decides if a user is permitted to perform an action on a resource based on * the user's privileges that are derived from the user's relationships to * resources. * @@ -16,8 +16,8 @@ export const PolicyDecisionPoint = { * to perform the action on * @param resourceIdentifier The identifier of the specific resource the user * wants to perform the action on - * @returns True if the user is allowed to perform the action on the resource, - * false otherwise + * @returns True if the user is permitted to perform the action on the + * resource, false otherwise */ isAllowed( user: User, diff --git a/src/core/tests/runner/node-test-runner.ts b/src/core/tests/runner/node-test-runner.ts index eca9e6d..fa9b4cd 100644 --- a/src/core/tests/runner/node-test-runner.ts +++ b/src/core/tests/runner/node-test-runner.ts @@ -1,25 +1,29 @@ import assert from "node:assert"; -import { describe, it } from "node:test"; -import type { Expectation, TestRunner } from "./test-runner.ts"; +import { test } from "node:test"; +import { TestRunner, type TestCase } from "./test-runner.ts"; -export class NodeTestRunner implements TestRunner { - group(name: string, callback: () => void) { - void describe(name, callback); - } +const expect = (actual: unknown) => ({ + toBe: (expected: unknown) => assert.strictEqual(actual, expected), + notToBe: (expected: unknown) => assert.notStrictEqual(actual, expected), +}); - test( - name: string, - callback: (t: { skip: (reason?: string) => void }) => Promise | void, - ) { - void it(name, async (t) => - callback({ skip: (reason?: string) => t.skip(reason) }), - ); - } +export class NodeTestRunner extends TestRunner { + override async run(testCases: Array) { + await Promise.all( + testCases.map((testCase) => + test(testCase.name, async () => { + const testResult = await testCase.test({ + expect, + skip: (reason) => { + void test.skip(reason); + }, + }); - expect(actual: any): Expectation { - return { - toBe: (expected) => assert.strictEqual(actual, expected), - notToBe: (expected) => assert.notStrictEqual(actual, expected), - }; + if (testResult) { + this.testResults.push(testResult); + } + }), + ), + ); } } diff --git a/src/core/tests/runner/test-runner.ts b/src/core/tests/runner/test-runner.ts index 775007f..1e449eb 100644 --- a/src/core/tests/runner/test-runner.ts +++ b/src/core/tests/runner/test-runner.ts @@ -1,42 +1,59 @@ -import { NodeTestRunner } from "./node-test-runner.ts"; +import { User } from "../../policy/entities/user"; +import { Route } from "../test-utils"; -export type Expectation = { - toBe: (expected: any) => void; - notToBe: (expected: any) => void; +export type AccessControlResult = "permitted" | "denied"; + +export type TestResult = { + user: User | null; + route: Route; + expected: AccessControlResult; + actual?: AccessControlResult; + testResult?: "✅" | "❌" | "⏭️"; }; -export type TestContext = { - skip: (reason?: string) => void; +export type Expectation = (actual: unknown) => { + toBe: (expected: unknown) => void; + notToBe: (expected: unknown) => void; }; -// todo: expose TestRunner in api so a custom test-runner can be used when implementing this interface -export type TestRunner = { - group: (name: string, callback: () => void) => void; - test: ( - name: string, - callback: (t: TestContext) => Promise | void, - ) => void; - expect: (actual: any) => Expectation; +export type TestCaseFunction = ( + testContext: TestContext, +) => Promise; + +export type TestCase = { + name: string; + test: TestCaseFunction; }; -export type TestRunnerIdentifier = "node" | "jest"; - -const DEFAULT_TEST_RUNNER = "node"; -export const TestRunnerFactory = { - createTestRunner( - runner: TestRunnerIdentifier = DEFAULT_TEST_RUNNER, - ): TestRunner { - switch (runner) { - case "node": { - return new NodeTestRunner(); - } - /*case "jest": - return new JestTestRunner(); - case "japa": - return new JapaTestRunner();*/ - default: { - throw new Error(`Unsupported test runner: ${runner}`); - } - } - }, +export type TestContext = { + expect: Expectation; + /** + * Skip the current test case. For test runners that do not support skipping + * an already running test case, an appropriate warning should be displayed. + */ + skip: (reason?: string) => void; }; + +export abstract class TestRunner { + protected testResults: Array = []; + + constructor() { + process.on("beforeExit", () => this.printReport()); + } + + abstract run(testCases: Array): Promise; + + protected printReport(): void { + console.log("\n=== Test Report ==="); + + const transformedResults = this.testResults.map((result) => ({ + ...result, + user: result.user?.toString() ?? "anonymous", + route: result.route.toString(), + })); + + console.table(transformedResults); + + // todo: enhance this with a detailed report containing all the routes that failed + } +} diff --git a/src/core/tests/test-executor.ts b/src/core/tests/test-executor.ts deleted file mode 100644 index fd9ba89..0000000 --- a/src/core/tests/test-executor.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { - HTTP_FORBIDDEN_STATUS_CODE, - HTTP_UNAUTHORIZED_STATUS_CODE, -} from "../constants.ts"; -import { OpenAPIParser } from "../parsers/openapi-parser.ts"; -import { Resource } from "../policy/entities/resource.ts"; -import { User } from "../policy/entities/user.ts"; -import type { TestRunner } from "./runner/test-runner.ts"; -import { performRequest, Route } from "./test-utils.ts"; -import { TestcaseGenerator } from "./testcase-generator.ts"; - -type AccessControlResult = "allowed" | "forbidden"; - -type TestResult = { - user: User | null; - route: Route; - expected: AccessControlResult; - actual?: AccessControlResult; - testResult?: "✅" | "❌" | "⏭️"; -}; - -export class TestExecutor { - async runTests( - testRunner: TestRunner, - openApiUrl: string, - apiBaseUrl: string, - users: Array, - resources: Array, - ) { - const openAPIParser = await OpenAPIParser.create(openApiUrl, apiBaseUrl); - openAPIParser.validateCustomFields(resources); - - const testController = new TestcaseGenerator(openAPIParser, users); - const testCases = testController.generateTestCases(); //.bind(testController); - - const results: Array = []; - const blockedUserIdentifiers: Array = []; - - for (const testCase of testCases) { - const { user, route, expectedRequestToBeAllowed } = testCase; - - testRunner.test( - `${route} from the perspective of user '${user ?? "anonymous"}'`, - async (t) => { - const expected: AccessControlResult = expectedRequestToBeAllowed - ? "allowed" - : "forbidden"; // todo: make enum for this? - - const testResult: TestResult = { - user, - route, - expected, - testResult: "❌", - }; - results.push(testResult); - - const userHasBeenBlocked = - user !== null && - blockedUserIdentifiers.includes(user.getCredentials().identifier); - if (userHasBeenBlocked) { - testResult.testResult = "⏭️"; - t.skip( - `User '${user}' has been blocked since a previous attempt to authenticate failed.`, - ); - return; - } - - const isAnonymousUser = user === null; - const credentials = isAnonymousUser ? null : user.getCredentials(); - - const authenticator = openAPIParser.getAuthenticatorByRoute(route); - - let response; - try { - response = await performRequest(route, authenticator, credentials); - } catch (error: unknown) { - // todo: create two Error instances - if (error instanceof Error) { - console.error(error.message); - - console.warn( - `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. - This test will be skipped and further testcases for user '${user}' will be cancelled. - Please check whether you provided correct credentials and the authentication setup is properly configured.`, - ); - - if (user !== null) { - blockedUserIdentifiers.push(user.getCredentials().identifier); - } - - testResult.testResult = "⏭️"; - t.skip(error.message); - } - - return; - } - - /*const isUnauthorized = - response.statusCode === HTTP_UNAUTHORIZED_STATUS_CODE;*/ - - /* if (isUnauthorized && !isAnonymousUser) { - // todo: make route toString() - const { retryCount } = response; - const recurringAuthenticationProblem = retryCount > 0; - - console.warn( - `Although the user ${user} has been authenticated using the authentication endpoint, the server responded with status code ${HTTP_UNAUTHORIZED_STATUS_CODE} when trying to access route ${route.method} ${route.url}. - ${recurringAuthenticationProblem ? `The server kept responding with status code ${response.statusCode} after ${retryCount} retries have been made.` : `The server responded with status code ${response.statusCode}.`} - This testcase will be skipped. - Please check whether the credentials are correct and the authentication setup is properly configured.`, - ); - - t.skip( - recurringAuthenticationProblem - ? "Recurring authentication problem" - : "Authentication problem", - ); // todo: new state for skipped? currently only pass/fail - return; - }*/ - - // todo: make it configurable what is considered as forbidden - // for now, forbidden is when the corresponding status code has been sent - const { statusCode } = response; - console.debug("STATUSCODE " + statusCode); - - let actual: AccessControlResult = - statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "forbidden" : "allowed"; - - if (expectedRequestToBeAllowed) { - // can be one of 2XX codes but could also be an error that occurred due to wrong syntax of request - - // todo: what about anonymous users? for them it should not be forbidden and also not unauthorized - testRunner.expect(statusCode).notToBe(HTTP_FORBIDDEN_STATUS_CODE); - } else { - // as anonymous user, unauthorized or forbidden is okay - if (isAnonymousUser) { - const requestForbidden = [ - HTTP_FORBIDDEN_STATUS_CODE, // todo: is forbidden really expected for users without authentication details or should it only be Unauthorized? - HTTP_UNAUTHORIZED_STATUS_CODE, - ].includes(statusCode); - - testRunner.expect(requestForbidden).toBe(true); - - actual = requestForbidden ? "forbidden" : "allowed"; // todo: maybe rename to rejected (is either forbidden/unauthorized) - } else { - testRunner.expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); - } - } - - testResult.actual = actual; - //testResult.authenticator = authenticator; - testResult.testResult = actual === expected ? "✅" : "❌"; - }, - ); - } - - TestExecutor.printResults(results); - } - - private static printResults(results: Array) { - process.on("beforeExit", () => { - console.log("\nTest Results:"); - const transformedResults = results.map((result) => ({ - ...result, - user: result.user?.toString() ?? "anonymous", - route: result.route.toString(), - })); - - console.table(transformedResults); - - // todo: enhance this with a detailed report containing all the routes that failed - }); - } -} diff --git a/src/core/tests/test-reporter.ts b/src/core/tests/test-reporter.ts index 444118d..a9eddeb 100644 --- a/src/core/tests/test-reporter.ts +++ b/src/core/tests/test-reporter.ts @@ -1,5 +1,5 @@ import { BaseReporter } from "@japa/runner/core"; -import type { TestCase } from "./testcase-generator.ts"; +import type { TestCombination } from "./testcase-generator.ts"; // todo: clarify how test report will work export class TestReporter extends BaseReporter { @@ -9,7 +9,7 @@ export class TestReporter extends BaseReporter { const { hasError: testFailed } = testPayload; const testStateRepresentation = testFailed ? "❌" : "✅"; - const testDataset: TestCase = testPayload.dataset.row; + const testDataset: TestCombination = testPayload.dataset.row; const { route, user, expectedRequestToBeAllowed } = testDataset; const { url, method } = route; @@ -29,7 +29,7 @@ export class TestReporter extends BaseReporter { Test Status: ${testStateRepresentation} Request: ${requestRepresentation} User: ${user} -Expection: ${expectedRequestToBeAllowed ? "Should have been allowed" : "Should have been denied"} +Expection: ${expectedRequestToBeAllowed ? "Should have been permitted" : "Should have been denied"} ------------------------------------------- Actual status: ${actual} Expected status: ${expected} diff --git a/src/core/tests/testcase-executor.ts b/src/core/tests/testcase-executor.ts new file mode 100644 index 0000000..50741e1 --- /dev/null +++ b/src/core/tests/testcase-executor.ts @@ -0,0 +1,92 @@ +import { + HTTP_FORBIDDEN_STATUS_CODE, + HTTP_UNAUTHORIZED_STATUS_CODE, +} from "../constants.ts"; +import { OpenAPIParser } from "../parsers/openapi-parser.ts"; +import type { User } from "../policy/entities/user.ts"; +import type { + AccessControlResult, + TestContext, + TestResult, +} from "./runner/test-runner.ts"; +import { performRequest } from "./test-utils.ts"; +import type { TestCombination } from "./testcase-generator.ts"; + +export class TestCaseExecutor { + private blockedUserIdentifiers: Array = []; + + constructor(private openApiParser: OpenAPIParser) {} + + async execute( + testCombination: TestCombination, + { expect, skip }: TestContext, + ): Promise { + const { user, route, expectedRequestToBeAllowed } = testCombination; + const expected: AccessControlResult = expectedRequestToBeAllowed + ? "permitted" + : "denied"; + const testResult: TestResult = { user, route, expected, testResult: "❌" }; + + const userHasBeenBlocked = + user !== null && + this.blockedUserIdentifiers.includes(user.getCredentials().identifier); + if (userHasBeenBlocked) { + testResult.testResult = "⏭️"; + skip( + `User '${user}' has been blocked since a previous attempt to authenticate failed.`, + ); + return testResult; + } + + const isAnonymousUser = user === null; + const credentials = isAnonymousUser ? null : user.getCredentials(); + const authenticator = this.openApiParser.getAuthenticatorByRoute(route); + + let response; + try { + response = await performRequest(route, authenticator, credentials); + } catch (error: unknown) { + if (error instanceof Error) { + console.error(error.message); + + console.warn( + `Could not impersonate user '${user}' while trying to reach route ${route.method} ${route.url}. + This test will be skipped and further testcases for user '${user}' will be cancelled. + Please check whether you provided correct credentials and the authentication setup is properly configured.`, + ); + + if (user !== null) { + this.blockedUserIdentifiers.push(user.getCredentials().identifier); + } + + testResult.testResult = "⏭️"; + skip(error.message); + } + return testResult; + } + + const { statusCode } = response; + console.debug("STATUSCODE " + statusCode); + let actual: AccessControlResult = + statusCode === HTTP_FORBIDDEN_STATUS_CODE ? "denied" : "permitted"; + + if (expectedRequestToBeAllowed) { + expect(statusCode).notToBe(HTTP_FORBIDDEN_STATUS_CODE); + } else { + if (isAnonymousUser) { + const requestForbidden = [ + HTTP_FORBIDDEN_STATUS_CODE, + HTTP_UNAUTHORIZED_STATUS_CODE, + ].includes(statusCode); + expect(requestForbidden).toBe(true); + actual = requestForbidden ? "denied" : "permitted"; + } else { + expect(statusCode).toBe(HTTP_FORBIDDEN_STATUS_CODE); + } + } + + testResult.actual = actual; + testResult.testResult = actual === expected ? "✅" : "❌"; + return testResult; + } +} diff --git a/src/core/tests/testcase-generator.ts b/src/core/tests/testcase-generator.ts index 1cd9b37..cd0a010 100644 --- a/src/core/tests/testcase-generator.ts +++ b/src/core/tests/testcase-generator.ts @@ -9,26 +9,17 @@ import type { ResourceName, } from "../policy/types.ts"; import { removeObjectDuplicatesFromArray } from "../utils.ts"; +import type { TestCase } from "./runner/test-runner.ts"; import { Route } from "./test-utils.ts"; +import { TestCaseExecutor } from "./testcase-executor.ts"; -export type TestCase = { - user: User | null; // alternatively: AnonymousUser (extends User) +export type TestCombination = { + user: User | null; route: Route; expectedRequestToBeAllowed: boolean; }; -export type TestCases = Array; - -/* -user1.canView(userResource); // can view all Users -> /admin/users & /admin/users/:id -console.log( - "user1canview", - PolicyDecisionPoint.isAllowed(user1, "read", userResource, 1), // todo: read vs. view? -);*/ -// todo: unit test for scenarios: -// canView(userResource): -// isAllowed(user1, "read", userResource, 1) -> true && isAllowed(user1, "read", userResource) -> true -// canView(userResource, 1): -// isAllowed(user1, "read", userResource, 1) -> true && isAllowed(user1, "read", userResource) -> false + +export type TestCombinations = Array; export class TestcaseGenerator { constructor( @@ -36,8 +27,7 @@ export class TestcaseGenerator { private readonly users: Array, ) {} - // todo: this shouldn't be async, solve async in source (OpenAPI parser) - generateTestCases(): TestCases { + generateTestCombinations(): TestCombinations { // todo: generate full-formed URLs with parameters // todo: for now only query parameters and path parameters are supported, maybe add support for other types of parameters // resource params mapping @@ -46,8 +36,8 @@ export class TestcaseGenerator { // each url resource mapping has " " pairs with info where to find resource param const resourceUserCombinations = this.generateResourceUserCombinations(); - const testcases: TestCases = pathResourceMappings.flatMap( - (pathResourceMapping) => { + const testcases: TestCombinations = + pathResourceMappings.flatMap((pathResourceMapping) => { // todo: create Route object for url & method to use instead const { path, method, isPublicPath, resources } = pathResourceMapping; @@ -62,7 +52,7 @@ export class TestcaseGenerator { // no resources available -> default: deny (except for "public" routes) // todo: mark routes as public (for anonymous users) or available for all logged-in users - // routes that are public for anonymous users are expected to be allowed for logged-in users too + // routes that are public for anonymous users are expected to be permitted for logged-in users too if (!routeHasResources) { if (isPublicPath) { @@ -153,8 +143,7 @@ export class TestcaseGenerator { expectedRequestToBeAllowed, }; }); - }, - ); + }); return removeObjectDuplicatesFromArray(testcases); } @@ -167,6 +156,7 @@ export class TestcaseGenerator { resourceIdentifier?: ResourceIdentifier; }>(); + // todo: move this explanation to JSDoc // generate combinations between users, actions, resources and resource ids // for that, go through relations of each user with a resource, // create a test case with expected result of true (for current user) and false (for other users) @@ -199,9 +189,20 @@ export class TestcaseGenerator { // todo: knowledge of resources based on openapi definitions // this is only applicable for non-parameterized resources (e.g. GET /users) because no valid resourceIdentifiers are available // there will be no guessing of valid resourceIdentifiers in that case - // adds additional test cases only when resource/access combination is not already covered by someone who is allowed to access it + // adds additional test cases only when resource/access combination is not already covered by someone who is permitted to access it } return [...resourceUserCombinations]; } + + async generateTestCases(): Promise> { + const testCombinations = this.generateTestCombinations(); + const testCaseExecutor = new TestCaseExecutor(this.openApiParser); + + return testCombinations.map((testCombination) => ({ + name: `${testCombination.route} from the perspective of user '${testCombination.user ?? "anonymous"}'`, + test: async (testContext) => + testCaseExecutor.execute(testCombination, testContext), + })); + } } diff --git a/tests/unit/policy-decision-point.spec.ts b/tests/unit/policy-decision-point.spec.ts index 1cfc93d..82af222 100644 --- a/tests/unit/policy-decision-point.spec.ts +++ b/tests/unit/policy-decision-point.spec.ts @@ -5,7 +5,7 @@ import { PolicyDecisionPoint } from "../../src/core/policy/policy-decision-point // todo: how to name tests and test files? test.group("PolicyDecisionPoint.isAllowed", () => { - test("allowed access is allowed", ({ expect }) => { + test("allowed access is permitted", ({ expect }) => { const user1 = new User("user1", "password"); const todoResource = new Resource("Todo"); const resourceIdentifier = 123; diff --git a/tests/unit/test-executor.spec.ts b/tests/unit/test-executor.spec.ts deleted file mode 100644 index 00f035b..0000000 --- a/tests/unit/test-executor.spec.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { test } from "@japa/runner"; - -// todo: Japa types only available in NodeNext -// but then type issues with library types come up when not set to ESNext - -test.group("TestExecutor", () => { - test("runTests() should print a warning when using invalid credentials"); - test( - "runTests() should print a warning when rate limiting that prevents further tests is detected", - ); -});