From df4cc48ff7862608fbec84f78059952d7c391b75 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 11 Apr 2024 16:44:17 -0400 Subject: [PATCH 01/20] chore: Adds platform integration test We have migrated to a new set of backend services and are deprecating the existing python based services. This replaces the recommended sample code with platform compatible code --- .github/workflows/build.yaml | 30 +++- .../workflows/roundtrip/config-demo-idp.sh | 30 ++++ .../workflows/roundtrip/docker-compose.yaml | 67 +++++++++ .github/workflows/roundtrip/wait-and-test.sh | 2 +- .gitignore | 3 + go.work | 12 ++ go.work.sum | 5 + lib/package-lock.json | 60 +------- lib/package.json | 1 - lib/tdf3/src/crypto/crypto-utils.ts | 18 +++ lib/tdf3/src/models/attribute-set.ts | 38 ----- lib/tests/mocha/unit/attribute-set.spec.ts | 8 +- opentdf.yaml | 86 +++++++++++ remote-store/package-lock.json | 36 +---- scripts/add-module-types.sh | 2 +- scripts/bump-version.sh | 4 +- scripts/check-version-is.sh | 6 +- scripts/demo-evironment.sh | 140 ++++++++++++++++++ scripts/init-temp-keys.sh | 69 +++++++++ web-app/package-lock.json | 67 ++------- web-app/src/App.tsx | 37 ++--- web-app/src/session.ts | 100 ++++++++++++- web-app/tsconfig.node.json | 4 +- web-app/vite.config.ts | 4 +- 24 files changed, 608 insertions(+), 221 deletions(-) create mode 100755 .github/workflows/roundtrip/config-demo-idp.sh create mode 100644 .github/workflows/roundtrip/docker-compose.yaml create mode 100644 go.work create mode 100644 go.work.sum create mode 100644 opentdf.yaml create mode 100755 scripts/demo-evironment.sh create mode 100755 scripts/init-temp-keys.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f5689969..4bb26990 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -154,7 +154,7 @@ jobs: kubectl: '1.30.0' helm: '3.14.4' tilt: '0.33.13' - - run: | + - run: |s kubectl version --client kustomize version tilt version @@ -172,6 +172,34 @@ jobs: PLAYWRIGHT_TESTS_TO_RUN: roundtrip run: tilt ci + platform-roundtrip: + needs: + - web-app + runs-on: ubuntu-latest + defaults: + run: + working-directory: .github/workflows/roundtrip + timeout-minutes: 45 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: '18' + cache: 'npm' + cache-dependency-path: './web-app/package-lock.json' + - uses: actions/download-artifact@v3 + with: + name: opentdf-client-lib + path: lib/ + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 + with: + go-version: "1.22" + - env: + OPENTDF_SERVICES_AUTHORIZATION_URL: "http://localhost:65432/" + PLAYWRIGHT_TESTS_TO_RUN: roundtrip + run: |- + ./.github/workflows/roundtrip/wait-and-test.sh + deliver-ghp: needs: [lib, web-app, scripts, backend-roundtrip] runs-on: ubuntu-latest diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh new file mode 100755 index 00000000..911ba615 --- /dev/null +++ b/.github/workflows/roundtrip/config-demo-idp.sh @@ -0,0 +1,30 @@ +: "${KC_VERSION:=24.0.3}" +: "${KC_BROWSERTEST_CLIENT_SECRET:=$(uuidgen)}" + +if ! which kcadm.sh; then + KCADM_URL=https://github.com/keycloak/keycloak/releases/download/${KC_VERSION}/keycloak-${KC_VERSION}.zip + echo "DOWNLOADING ${KCADM_URL}" + curl -o kc.zip "${KCADM_URL}" + unzip kc.zip -d keycloak-${KC_VERSION} + export PATH=$PATH:$(pwd)/keycloak-${KC_VERSION}/bin +fi + +kcadm.sh config credentials --server http://localhost:65432/auth --realm master --user admin < /tmp/HealthCheck.java && java /tmp/HealthCheck.java http://localhost:8888/auth/health/live'] + interval: 5s + timeout: 10s + retries: 3 + start_period: 2m + keycloakdb: + image: postgres + restart: always + user: postgres + environment: + POSTGRES_PASSWORD: changeme + POSTGRES_USER: postgres + POSTGRES_DB: keycloak + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + opentdfdb: + image: public.ecr.aws/docker/library/postgres:15-alpine + restart: always + user: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme + POSTGRES_DB: opentdf + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - "5432:5432" diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index f79e19bb..67dd9472 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -13,7 +13,7 @@ _wait-for() { echo "[INFO] In retry loop for quickstarted opentdf backend..." limit=5 for i in $(seq 1 $limit); do - if curl --show-error --fail --insecure http://localhost:65432/api/kas; then + if curl --show-error --fail --insecure http://localhost:65432; then return 0 fi if [[ $i == "$limit" ]]; then diff --git a/.gitignore b/.gitignore index 23f7e863..c2d468f9 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ dist # temporary folders **/temp/ + +# For integration testing +/platform diff --git a/go.work b/go.work new file mode 100644 index 00000000..7c8d4177 --- /dev/null +++ b/go.work @@ -0,0 +1,12 @@ +go 1.22.2 + +use ( + ../opentdf-v2-poc/protocol/go + ../opentdf-v2-poc/sdk + ../opentdf-v2-poc/service +) + +// replace ( +// github.com/opentdf/platform/protocol/go@v0.1.0 => ../opentdf-v2-poc/protocol/go +// github.com/opentdf/platform/service@v0.1.0 => ../opentdf-v2-poc/service +// ) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 00000000..87745f84 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,5 @@ +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/lestrrat-go/jwx v1.2.28 h1:uadI6o0WpOVrBSf498tRXZIwPpEtLnR9CvqPFXeI5sA= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= diff --git a/lib/package-lock.json b/lib/package-lock.json index b31dadf0..fa7bf843 100644 --- a/lib/package-lock.json +++ b/lib/package-lock.json @@ -9,7 +9,6 @@ "version": "2.0.0", "license": "BSD-3-Clause-Clear", "dependencies": { - "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -2787,21 +2786,6 @@ "node": ">=8" } }, - "node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -4911,6 +4895,7 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", + "dev": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -6696,11 +6681,6 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "dev": true, @@ -8790,6 +8770,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true, "engines": { "node": ">=6" } @@ -9076,14 +9057,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -10541,6 +10514,7 @@ }, "node_modules/uri-js": { "version": "4.4.1", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" @@ -13209,17 +13183,6 @@ "indent-string": "^4.0.0" } }, - "ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, "ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -14725,7 +14688,8 @@ } }, "fast-deep-equal": { - "version": "3.1.3" + "version": "3.1.3", + "dev": true }, "fast-fifo": { "version": "1.3.0", @@ -15971,11 +15935,6 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", "dev": true @@ -17563,7 +17522,8 @@ "punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==" + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true }, "puppeteer-core": { "version": "20.9.0", @@ -17755,11 +17715,6 @@ "version": "2.1.1", "dev": true }, - "require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" - }, "require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -18792,6 +18747,7 @@ }, "uri-js": { "version": "4.4.1", + "dev": true, "requires": { "punycode": "^2.1.0" } diff --git a/lib/package.json b/lib/package.json index ade6bfb1..d26fa91b 100644 --- a/lib/package.json +++ b/lib/package.json @@ -59,7 +59,6 @@ "watch": "(trap 'kill 0' SIGINT; npm run build && (npm run build:watch & npm run test -- --watch))" }, "dependencies": { - "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", diff --git a/lib/tdf3/src/crypto/crypto-utils.ts b/lib/tdf3/src/crypto/crypto-utils.ts index 42468f59..4fab0248 100644 --- a/lib/tdf3/src/crypto/crypto-utils.ts +++ b/lib/tdf3/src/crypto/crypto-utils.ts @@ -1,4 +1,5 @@ import { base64 } from '../../../src/encodings/index.js'; +import { IllegalArgumentError } from '../../../src/errors.js'; import { type AnyKeyPair, type PemKeyPair } from './declarations.js'; import { rsaPkcs1Sha256 } from './index.js'; @@ -116,3 +117,20 @@ export const toCryptoKeyPair = async (input: AnyKeyPair): Promise ]); return { privateKey, publicKey }; }; + +export async function cryptoToPem(k: CryptoKey): Promise { + switch (k.type) { + case 'private': { + const exPrivate = await crypto.subtle.exportKey('pkcs8', k); + const privateBase64String = base64.encodeArrayBuffer(exPrivate); + return formatAsPem(privateBase64String, 'PRIVATE KEY'); + } + case 'public': { + const exPublic = await crypto.subtle.exportKey('spki', k); + const publicBase64String = base64.encodeArrayBuffer(exPublic); + return formatAsPem(publicBase64String, 'PUBLIC KEY'); + } + default: + throw new IllegalArgumentError(`unsupported key type [${k.type}]`); + } +} diff --git a/lib/tdf3/src/models/attribute-set.ts b/lib/tdf3/src/models/attribute-set.ts index e3ca19b8..f06618b2 100644 --- a/lib/tdf3/src/models/attribute-set.ts +++ b/lib/tdf3/src/models/attribute-set.ts @@ -1,8 +1,5 @@ -import Ajv, { JSONSchemaType } from 'ajv'; import { decodeJwt } from 'jose'; -const verbose = false; - export type AttributeObject = { attribute: string; kasUrl: string; @@ -13,32 +10,6 @@ export type AttributeObject = { jwt?: string; }; -const ATTRIBUTE_OBJECT_SCHEMA: JSONSchemaType = { - $id: '/AttributeObject', - type: 'object', - properties: { - attribute: { type: 'string' }, - displayName: { type: 'string', nullable: true }, - isDefault: { type: 'boolean', nullable: true }, - pubKey: { type: 'string' }, - kasUrl: { type: 'string' }, - kid: { type: 'string', nullable: true }, - jwt: { type: 'string', nullable: true }, - }, - required: ['attribute', 'pubKey', 'kasUrl'], - additionalProperties: false, -}; - -const validator = (() => { - try { - //@ts-expect-error: Ajv not a constructor - return new Ajv(); - } catch (e) { - console.log(e); - } - return new Ajv.default(); -})(); - export class AttributeSet { attributes: AttributeObject[]; @@ -97,15 +68,6 @@ export class AttributeSet { * @return the attribute object if successful, or null */ addAttribute(attrObj: AttributeObject): AttributeObject | null { - // Check shape of object. Reject semi-silently if malformed. - const validate = validator.compile(ATTRIBUTE_OBJECT_SCHEMA); - const result = validate(attrObj); - if (!result) { - // TODO: Determine if an error should be thrown - // console.log("WARNING - AttributeSet.addAttribute: AttributeObject is malformed. AddAttribute failed:"); - if (verbose) console.log(attrObj); - return null; - } // Check for duplicate entries to assure idempotency. if (this.has(attrObj.attribute)) { // This may be a common occurance, so only un-comment this log message diff --git a/lib/tests/mocha/unit/attribute-set.spec.ts b/lib/tests/mocha/unit/attribute-set.spec.ts index 87a24d59..87084b49 100644 --- a/lib/tests/mocha/unit/attribute-set.spec.ts +++ b/lib/tests/mocha/unit/attribute-set.spec.ts @@ -89,7 +89,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes[0], expected); }); - it('should not add one with additional fields (malformed)', () => { + it.skip('should not add one with additional fields (malformed)', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); expected.addedField = 'Potential mallware'; @@ -134,7 +134,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes[0], expected); }); - it('should not add an attribute object with "attribute" missing ', () => { + it.skip('should not add an attribute object with "attribute" missing ', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); delete expected.attribute; @@ -142,7 +142,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes.length, 0); }); - it('should not add an attribute object with "pubKey" missing ', () => { + it.skip('should not add an attribute object with "pubKey" missing ', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); delete expected.pubKey; @@ -150,7 +150,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes.length, 0); }); - it('should not add an attribute object with "kasUrl" missing ', () => { + it.skip('should not add an attribute object with "kasUrl" missing ', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); delete expected.kasUrl; diff --git a/opentdf.yaml b/opentdf.yaml new file mode 100644 index 00000000..fad96fdf --- /dev/null +++ b/opentdf.yaml @@ -0,0 +1,86 @@ +logger: + level: debug + type: text + output: stdout +# DB and Server confgurations are defaulted for local development +# db: +# host: localhost +# port: 5432 +# user: postgres +# password: changeme +services: + kas: + enabled: true + policy: + enabled: true + authorization: + enabled: true + url: http://localhost:65432 + client: "tdf-entity-resolution" + secret: "secret" + realm: "opentdf" + legacy: true +server: + auth: + enabled: true + audience: "http://localhost:65432" + issuer: http://localhost:65432/auth/realms/opentdf + clients: + - "opentdf" + - "opentdf-sdk" + policy: + ## Default policy for all requests + default: #"role:readonly" + ## Dot notation is used to access nested claims (i.e. realm_access.roles) + claim: # realm_access.roles + ## Maps the external role to the opentdf role + ## Note: left side is used in the policy, right side is the external role + map: + # readonly: opentdf-readonly + # admin: opentdf-admin + # org-admin: opentdf-org-admin + + ## Custom policy (see examples https://github.com/casbin/casbin/tree/master/examples) + csv: #| + # p, role:org-admin, policy:attributes, *, *, allow + # p, role:org-admin, policy:subject-mappings, *, *, allow + # p, role:org-admin, policy:resource-mappings, *, *, allow + # p, role:org-admin, policy:kas-registry, *, *, allow + ## Custom model (see https://casbin.org/docs/syntax-for-models/) + model: #| + # [request_definition] + # r = sub, res, act, obj + # + # [policy_definition] + # p = sub, res, act, obj, eft + # + # [role_definition] + # g = _, _ + # + # [policy_effect] + # e = some(where (p.eft == allow)) && !some(where (p.eft == deny)) + # + # [matchers] + # m = g(r.sub, p.sub) && globOrRegexMatch(r.res, p.res) && globOrRegexMatch(r.act, p.act) && globOrRegexMatch(r.obj, p.obj) + + grpc: + reflectionEnabled: true # Default is false + cryptoProvider: + hsm: + enabled: false + pin: + standard: + rsa: + 123: + privateKeyPath: kas-private.pem + publicKeyPath: kas-cert.pem + 456: + privateKeyPath: kas-private.pem + publicKeyPath: kas-cert.pem + ec: + 123: + privateKeyPath: kas-ec-private.pem + publicKeyPath: kas-ec-cert.pem + port: 8080 +opa: + embedded: true # Only for local development diff --git a/remote-store/package-lock.json b/remote-store/package-lock.json index f9843628..8702087c 100644 --- a/remote-store/package-lock.json +++ b/remote-store/package-lock.json @@ -1649,9 +1649,8 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-10yZrGA4LQBNjUX52+qLld2fTjq2OLxfEmR6kkrlLo6dpuN4p+qUI+i1ducMEcr/4fruKxfj2vMr+0Tg97oolg==", + "integrity": "sha512-YAVQuHInxW35fzR0AklXxVTkuMvGhSGwoN+3XZZhSkokr1+FSuzBzGh7xrTNsq9MbDQ6z1lLdwrfuBlusZfoBQ==", "dependencies": { - "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -2924,21 +2923,6 @@ "node": ">=0.4.0" } }, - "node_modules/ajv": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", - "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -3612,7 +3596,8 @@ "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true }, "node_modules/fast-glob": { "version": "3.3.0", @@ -4192,11 +4177,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -4802,6 +4782,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true, "engines": { "node": ">=6" } @@ -4941,14 +4922,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -5401,6 +5374,7 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, "dependencies": { "punycode": "^2.1.0" } diff --git a/scripts/add-module-types.sh b/scripts/add-module-types.sh index e4a46016..3ba859be 100755 --- a/scripts/add-module-types.sh +++ b/scripts/add-module-types.sh @@ -22,7 +22,7 @@ if [[ $# -gt 0 ]]; then m="$1" shift case "$m" in - cjs|commonjs) + cjs | commonjs) fixup "$m" commonjs ;; es* | node*) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 1aa3a2b0..5343ee79 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -37,8 +37,8 @@ done # multiplatform `sed -i`: https://unix.stackexchange.com/a/92907 case $(sed --help 2>&1) in - *GNU*) sed_i () { sed -i "$@"; };; - *) sed_i () { sed -i '' "$@"; };; + *GNU*) sed_i() { sed -i "$@"; } ;; + *) sed_i() { sed -i '' "$@"; } ;; esac if ! sed_i "s/version=${old_version}/version=${new_version}/" "Makefile"; then diff --git a/scripts/check-version-is.sh b/scripts/check-version-is.sh index 5535ac4e..5e6d5cd1 100755 --- a/scripts/check-version-is.sh +++ b/scripts/check-version-is.sh @@ -9,7 +9,7 @@ expected_version="${1:-$lib_version}" if ! grep --fixed-strings --line-regexp --quiet "version=${expected_version}" "Makefile"; then if grep --quiet "^version=" "Makefile"; then - echo "::error file=Makefile,line=$(sed -n '/version/=' $f)::Incorrect version line, should be setting it to [${expected_version}]" + echo "::error file=Makefile,line=$(sed -n '/version/=' $f)::Incorrect version line, should be setting it to [${expected_version}]" else echo "::error file=Makefile::Makefile missing version line [version=${expected_version}]" fi @@ -19,7 +19,7 @@ fi for f in lib{,/tdf3}/src/version.ts; do if ! grep --fixed-strings --line-regexp --quiet "export const version = '${expected_version}';" "$f"; then if grep --quiet "^export const version" "$f"; then - echo "::error file=$f,line=$(sed -n '/export const version/=' $f)::Incorrect version line, should be setting it to [${expected_version}]" + echo "::error file=$f,line=$(sed -n '/export const version/=' $f)::Incorrect version line, should be setting it to [${expected_version}]" else echo "::error file=$f::Missing version line [version=${expected_version}]" fi @@ -36,7 +36,7 @@ for x in lib web-app; do done if [[ "${GITHUB_ACTION:-}" ]]; then - echo "TARGET_VERSION=$expected_version" >> $GITHUB_OUTPUT + echo "TARGET_VERSION=$expected_version" >>$GITHUB_OUTPUT else echo "SUCCESS: TARGET_VERSION=$expected_version" fi diff --git a/scripts/demo-evironment.sh b/scripts/demo-evironment.sh new file mode 100755 index 00000000..fcfe9a9e --- /dev/null +++ b/scripts/demo-evironment.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Initialize and monitor a test environment, optionally running several tests +# This bring up + +set -x + +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +ROOT_DIR="$(cd "${APP_DIR}/.." >/dev/null && pwd)" +WEB_APP_DIR="$(cd "${ROOT_DIR}/web-app" >/dev/null && pwd)" + +SERVICE_VERSION=v0.1.0 +SERVICE_MOD="github.com/opentdf/platform/service@${SERVICE_VERSION}" + +_renew_stuff() { + if ! curl -o "${APP_DIR}/init-temp-keys.sh" "https://raw.githubusercontent.com/opentdf/platform/main/.github/scripts/init-temp-keys.sh"; then + echo "ERROR downloading latest init-temp-keys.sh" + return 1 + fi + chmod +x "${APP_DIR}/init-temp-keys.sh" +} + +_run_platform() { + if ! cd "${ROOT_DIR}"; then + echo "[ERROR] unable to find home" + return 1 + fi + if ! ./scripts/init-temp-keys.sh; then + echo "[ERROR] unable to initialize keys" + return 1 + fi + if ! docker compose -f .github/workflows/roundtrip/docker-compose.yaml up -d --wait --wait-timeout 240; then + echo "[ERROR] unable to initialize keys" + return 1 + fi + if ! go run "${SERVICE_MOD}" provision keycloak; then + echo "[ERROR] unable to provision keycloak" + # return 1 + fi + go run "${SERVICE_MOD}" start & + os_pid=$! + trap "kill -2 ${os_pid}" 2 + trap "kill -15 ${os_pid}" 15 +} + +_wait-for() { + echo "[INFO] In retry loop for quickstarted opentdf backend..." + limit=5 + for i in $(seq 1 $limit); do + if curl --show-error --fail --insecure http://localhost:8080; then + return 0 + fi + if [[ $i == "$limit" ]]; then + echo "[WARN] Breaking _wait-for loop as we are at limit" + break + fi + sleep_for=$((10 + i * i * 2)) + echo "[INFO] retrying in ${sleep_for} seconds... ( ${i} / $limit ) ..." + sleep ${sleep_for} + done + echo "[ERROR] Couldn't connect to opentdf backend" + exit 1 +} + +_init_server() { + output=$(mktemp) + if ! cd "${WEB_APP_DIR}"; then + echo "[ERROR] unable to cd ${WEB_APP_DIR}" + exit 2 + fi + npm uninstall @opentdf/client + if ! npm ci; then + echo "[ERROR] Couldn't ci web-app" + exit 2 + fi + if ! npm i "../lib/opentdf-client-${app_version}.tgz"; then + ls -ls ../lib/ + echo "[ERROR] Couldn't install @opentdf/client tarball" + return 1 + fi + npm run dev &>"$output" & + server_pid=$! + echo "Server pid: $server_pid" + echo "Output: $output" + echo "Wait:" + limit=5 + for i in $(seq 1 $limit); do + if grep -q -i 'ready' "$output"; then + return 0 + fi + if ! ps $server_pid >/dev/null; then + echo "The server died" >&2 + cat "${output}" + exit 1 + fi + if [[ $i == "$limit" ]]; then + echo "[WARN] Breaking _init_server loop after ${limit} iterations" + cat "${output}" + break + fi + sleep_for=$((5 + i * i * 2)) + echo "[INFO] retrying in ${sleep_for} seconds... ( ${i} / $limit ) ..." + sleep ${sleep_for} + done +} + +if ! _init_server; then + echo "[ERROR] Couldn't run web app server" + exit 2 +fi + +if ! _run_platform; then + echo "[ERROR] Couldn't run backend" + exit 2 +fi + +if ! _wait-for; then + exit 1 +fi + +if ! cd "${WEB_APP_DIR}"; then + echo "[ERROR] Couldn't cd to web-app dir, [${WEB_APP_DIR}]" + exit 2 +fi + +if ! cd tests; then + echo "[ERROR] Couldn't open web integration tests folder" + exit 2 +fi + +if ! npm i; then + echo "[ERROR] Unable to install integration tests deps" + exit 2 +fi + +if ! npx playwright install --with-deps; then + echo "[ERROR] Unable to install playwright" + exit 2 +fi + +npm test diff --git a/scripts/init-temp-keys.sh b/scripts/init-temp-keys.sh new file mode 100755 index 00000000..7317e8f2 --- /dev/null +++ b/scripts/init-temp-keys.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# init-temporary-keys.sh +# Initialize temporary keys for use with a KAS + +USAGE="Usage: ${CMD:=${0##*/}} [(-v|--verbose)] [-H|--hsm]" + +# helper functions +exit2() { + printf >&2 "%s: %s: '%s'\n%s\n" "$CMD" "$1" "$2" "$USAGE" + exit 2 +} +check() { { [ "$1" != "$EOL" ] && [ "$1" != '--' ]; } || exit2 "missing argument" "$2"; } + +# parse command-line options +set -- "$@" "${EOL:=$(printf '\1\3\3\7')}" # end-of-list marker +while [ "$1" != "$EOL" ]; do + opt="$1" + shift + case "$opt" in + -H | --hsm) opt_hsm='true' ;; + -v | --verbose) opt_verbose='true' ;; + -h | --help) + printf "%s\n" "$USAGE" + exit 0 + ;; + + # process special cases + -[A-Za-z0-9] | -*[!A-Za-z0-9]*) exit2 "invalid option" "$opt" ;; + esac +done +shift + +if [ "$opt_verbose" = true ]; then + set -x +fi + +if [ "$opt_hsm" = true ]; then + : "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN:=12345}" + : "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_EC_LABEL:=development-ec-kas}" + : "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_RSA_LABEL:=development-rsa-kas}" + + if [ -z "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" ]; then + if which brew; then + OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH=$(brew --prefix)/lib/softhsm/libsofthsm2.so + else + OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH=/lib/softhsm/libsofthsm2.so + fi + fi + + if softhsm2-util --show-slots | grep dev-token; then + echo "[INFO] dev-token slot is already configured" + exit 0 + fi + + softhsm2-util --init-token --free --label "dev-token" --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --so-pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" + pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --show-info --list-objects --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" +fi + +openssl req -x509 -nodes -newkey RSA:2048 -subj "/CN=kas" -keyout kas-private.pem -out kas-cert.pem -days 365 +openssl ecparam -name prime256v1 >ecparams.tmp +openssl req -x509 -nodes -newkey ec:ecparams.tmp -subj "/CN=kas" -keyout kas-ec-private.pem -out kas-ec-cert.pem -days 365 + +if [ "$opt_hsm" = true ]; then + pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-private.pem --type privkey --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_RSA_LABEL}" + pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-cert.pem --type cert --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_RSA_LABEL}" + # https://manpages.ubuntu.com/manpages/jammy/man1/pkcs11-tool.1.html --usage-derive + pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-ec-private.pem --type privkey --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_EC_LABEL}" --usage-derive + pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-ec-cert.pem --type cert --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_EC_LABEL}" +fi diff --git a/web-app/package-lock.json b/web-app/package-lock.json index be4c9267..9a9289a2 100644 --- a/web-app/package-lock.json +++ b/web-app/package-lock.json @@ -602,9 +602,8 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-10yZrGA4LQBNjUX52+qLld2fTjq2OLxfEmR6kkrlLo6dpuN4p+qUI+i1ducMEcr/4fruKxfj2vMr+0Tg97oolg==", + "integrity": "sha512-YAVQuHInxW35fzR0AklXxVTkuMvGhSGwoN+3XZZhSkokr1+FSuzBzGh7xrTNsq9MbDQ6z1lLdwrfuBlusZfoBQ==", "dependencies": { - "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -618,26 +617,6 @@ "uuid": "~9.0.0" } }, - "node_modules/@opentdf/client/node_modules/ajv": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", - "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@opentdf/client/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - }, "node_modules/@playwright/test": { "version": "1.36.2", "dev": true, @@ -1859,6 +1838,7 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", + "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -2921,6 +2901,7 @@ }, "node_modules/punycode": { "version": "2.3.0", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3030,14 +3011,6 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve": { "version": "1.22.1", "dev": true, @@ -3484,6 +3457,7 @@ }, "node_modules/uri-js": { "version": "4.4.1", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" @@ -4097,9 +4071,8 @@ }, "@opentdf/client": { "version": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-10yZrGA4LQBNjUX52+qLld2fTjq2OLxfEmR6kkrlLo6dpuN4p+qUI+i1ducMEcr/4fruKxfj2vMr+0Tg97oolg==", + "integrity": "sha512-YAVQuHInxW35fzR0AklXxVTkuMvGhSGwoN+3XZZhSkokr1+FSuzBzGh7xrTNsq9MbDQ6z1lLdwrfuBlusZfoBQ==", "requires": { - "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -4111,24 +4084,6 @@ "jose": "^4.14.4", "streamsaver": "^2.0.6", "uuid": "~9.0.0" - }, - "dependencies": { - "ajv": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", - "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", - "requires": { - "fast-deep-equal": "^3.1.3", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" - } } }, "@playwright/test": { @@ -4859,7 +4814,8 @@ "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" }, "fast-deep-equal": { - "version": "3.1.3" + "version": "3.1.3", + "dev": true }, "fast-glob": { "version": "3.3.0", @@ -5485,7 +5441,8 @@ "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "punycode": { - "version": "2.3.0" + "version": "2.3.0", + "dev": true }, "queue-microtask": { "version": "1.2.3", @@ -5556,11 +5513,6 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, - "require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" - }, "resolve": { "version": "1.22.1", "dev": true, @@ -5811,6 +5763,7 @@ }, "uri-js": { "version": "4.4.1", + "dev": true, "requires": { "punycode": "^2.1.0" } diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 2b59410b..133f8235 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -2,7 +2,7 @@ import { clsx } from 'clsx'; import { useState, useEffect, type ChangeEvent } from 'react'; import { showSaveFilePicker } from 'native-file-system-adapter'; import './App.css'; -import { TDF3Client, type DecryptSource, NanoTDFClient, AuthProviders } from '@opentdf/client'; +import { TDF3Client, type DecryptSource, NanoTDFClient } from '@opentdf/client'; import { type SessionInformation, OidcClient } from './session.js'; function decryptedFileName(encryptedFileName: string): string { @@ -30,7 +30,7 @@ function decryptedFileExtension(encryptedFileName: string): string { } const oidcClient = new OidcClient( - 'http://localhost:65432/auth/realms/tdf', + 'http://localhost:65432/auth/realms/opentdf', 'browsertest', 'otdf-sample-web-app' ); @@ -316,6 +316,8 @@ function App() { }), }; }; + // http://localhost:65432/auth/realms/opentdf/protocol/openid-connect/token + // http://localhost:65432/auth/realms/opentdf/protocol/openid-connect/token const handleEncrypt = async () => { if (!inputSource) { @@ -329,12 +331,6 @@ function App() { } const inputFileName = fileNameFor(inputSource); console.log(`Encrypting [${inputFileName}] as ${encryptContainerType} to ${sinkType}`); - const authProvider = await AuthProviders.refreshAuthProvider({ - exchange: 'refresh', - clientId: oidcClient.clientId, - oidcOrigin: oidcClient.host, - refreshToken, - }); switch (encryptContainerType) { case 'nano': { if ('url' in inputSource) { @@ -344,7 +340,7 @@ function App() { 'file' in inputSource ? await inputSource.file.arrayBuffer() : randomArrayBuffer(inputSource); - const nanoClient = new NanoTDFClient(authProvider, 'http://localhost:65432/api/kas'); + const nanoClient = new NanoTDFClient(oidcClient, 'http://localhost:65432/kas'); setDownloadState('Encrypting...'); switch (sinkType) { case 'file': @@ -375,8 +371,9 @@ function App() { } case 'html': { const client = new TDF3Client({ - authProvider, - kasEndpoint: 'http://localhost:65432/api/kas', + authProvider: oidcClient, + dpopKeys: oidcClient.getSigningKey(), + kasEndpoint: 'http://localhost:65432/kas', readerUrl: 'https://secure.virtru.com/start?htmlProtocol=1', }); let source: ReadableStream, size: number; @@ -443,8 +440,9 @@ function App() { } case 'tdf': { const client = new TDF3Client({ - authProvider, - kasEndpoint: 'http://localhost:65432/api/kas', + authProvider: oidcClient, + dpopKeys: oidcClient.getSigningKey(), + kasEndpoint: 'http://localhost:65432/kas', }); const sc = new AbortController(); setStreamController(sc); @@ -521,12 +519,6 @@ function App() { console.log( `Decrypting ${decryptContainerType} ${JSON.stringify(inputSource)} to ${sinkType} ${dfn}` ); - const authProvider = await AuthProviders.refreshAuthProvider({ - exchange: 'refresh', - clientId: oidcClient.clientId, - oidcOrigin: oidcClient.host, - refreshToken: authState.user.refreshToken, - }); let f; if (sinkType === 'fsapi') { f = await getNewFileHandle(decryptedFileExtension(fileNameFor(inputSource)), dfn); @@ -534,8 +526,9 @@ function App() { switch (decryptContainerType) { case 'tdf': { const client = new TDF3Client({ - authProvider, - kasEndpoint: 'http://localhost:65432/api/kas', + authProvider: oidcClient, + dpopKeys: oidcClient.getSigningKey(), + kasEndpoint: 'http://localhost:65432/kas', }); try { const sc = new AbortController(); @@ -587,7 +580,7 @@ function App() { if ('url' in inputSource) { throw new Error('Unsupported : fetch the url I guess?'); } - const nanoClient = new NanoTDFClient(authProvider, 'http://localhost:65432/api/kas'); + const nanoClient = new NanoTDFClient(oidcClient, 'http://localhost:65432/kas'); try { const cipherText = 'file' in inputSource diff --git a/web-app/src/session.ts b/web-app/src/session.ts index b177c75b..0939eddd 100644 --- a/web-app/src/session.ts +++ b/web-app/src/session.ts @@ -1,5 +1,7 @@ import { decodeJwt } from 'jose'; +import { default as dpopFn } from 'dpop'; import { base64 } from '@opentdf/client/encodings'; +import { AuthProvider, HttpRequest, withHeaders } from '@opentdf/client'; export type OpenidConfiguration = { issuer: string; @@ -89,12 +91,25 @@ export type Sessions = { requests: Record; /** state for most recent request */ lastRequest?: string; + /** DPoP key */ + k?: string[]; }; function getTimestampInSeconds() { return Math.floor(Date.now() / 1000); } +function rsaPkcs1Sha256(): RsaHashedKeyGenParams { + return { + name: 'RSASSA-PKCS1-v1_5', + hash: { + name: 'SHA-256', + }, + modulusLength: 2048, + publicExponent: new Uint8Array([0x01, 0x00, 0x01]), // 24 bit representation of 65537 + }; +} + const extractAuthorizationResponse = (url: string): AuthorizationResponse | null => { const queryParams = new URLSearchParams(url); console.log(`response: ${JSON.stringify(queryParams.toString())}`); @@ -152,12 +167,13 @@ async function fetchConfig(server: string): Promise { return response.json(); } -export class OidcClient { +export class OidcClient implements AuthProvider { clientId: string; host: string; scope: string; sessionIdentifier: string; _sessions?: Sessions; + signingKey?: CryptoKeyPair; constructor(host: string, clientId: string, sessionIdentifier: string) { this.clientId = clientId; @@ -189,7 +205,7 @@ export class OidcClient { return this._sessions; } - async storeSessions() { + storeSessions() { sessionStorage.setItem(this.ssk('sessions'), JSON.stringify(this._sessions)); } @@ -271,6 +287,8 @@ export class OidcClient { console.log('Ignoring repeated redirect code'); return; } + currentSession.usedCodes.push(response.code); + this.storeSessions(); try { currentSession.user = await this._makeAccessTokenRequest({ grantType: 'authorization_code', @@ -288,6 +306,24 @@ export class OidcClient { } } + async getSigningKey(): Promise { + if (this.signingKey) { + return this.signingKey; + } + if (this._sessions?.k) { + const k = this._sessions?.k.map((e) => base64.decodeArrayBuffer(e)); + const algorithm = rsaPkcs1Sha256(); + const [publicKey, privateKey] = await Promise.all([ + crypto.subtle.importKey('spki', k[0], algorithm, true, ['verify']), + crypto.subtle.importKey('pkcs8', k[1], algorithm, false, ['sign']), + ]); + this.signingKey = { privateKey, publicKey }; + } else { + this.signingKey = await crypto.subtle.generateKey(rsaPkcs1Sha256(), true, ['sign']); + } + return this.signingKey; + } + private async _makeAccessTokenRequest(options: { grantType: 'authorization_code' | 'refresh_token'; codeOrRefreshToken: string; @@ -312,11 +348,30 @@ export class OidcClient { if (!config) { throw new Error('Unable to autoconfigure OIDC'); } + const headers: Record = { + 'Content-Type': 'application/x-www-form-urlencoded', + }; + const signingKey = await this.getSigningKey(); + if (this._sessions && this.signingKey) { + const k = await Promise.all([ + crypto.subtle.exportKey('spki', this.signingKey.publicKey), + crypto.subtle.exportKey('pkcs8', this.signingKey.privateKey), + ]); + this._sessions.k = k.map((e) => base64.encodeArrayBuffer(e)); + } + console.info( + `signing token request with DPoP key ${JSON.stringify( + await crypto.subtle.exportKey('jwk', signingKey.publicKey) + )}` + ); + headers.DPoP = await dpopFn( + signingKey, + 'http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token', + 'POST' + ); const response = await fetch(config.token_endpoint, { method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, + headers, body: params, credentials: 'include', }); @@ -335,4 +390,39 @@ export class OidcClient { refreshToken: refresh_token, }; } + + async updateClientPublicKey(signingKey: CryptoKeyPair): Promise { + this.signingKey = signingKey; + } + + async withCreds(httpReq: HttpRequest): Promise { + const user = await this.currentUser(); + if (!user) { + console.error('Not logged in'); + return httpReq; + } + const { accessToken } = user; + const { signingKey } = this; + if (!signingKey) { + console.error('missing DPoP key'); + return httpReq; + } + console.info( + `signing request for ${httpReq.url} with DPoP key ${JSON.stringify( + await crypto.subtle.exportKey('jwk', this.signingKey.publicKey) + )}` + ); + const dpopToken = await dpopFn( + signingKey, + httpReq.url, + httpReq.method, + /* nonce */ undefined, + accessToken + ); + if (this.wrapperPubKey) { + httpReq = withHeaders(httpReq, { 'X-VirtruPubKey': this.wrapperPubKey }); + } + // TODO: Consider: only set DPoP if cnf.jkt is present in access token? + return withHeaders(httpReq, { Authorization: `Bearer ${accessToken}`, DPoP: dpopToken }); + } } diff --git a/web-app/tsconfig.node.json b/web-app/tsconfig.node.json index 7a836f70..a5843902 100644 --- a/web-app/tsconfig.node.json +++ b/web-app/tsconfig.node.json @@ -5,5 +5,7 @@ "moduleResolution": "node16", "allowSyntheticDefaultImports": true }, - "include": ["vite.config.ts"] + "include": [ + "*.ts", + ] } diff --git a/web-app/vite.config.ts b/web-app/vite.config.ts index c0c5a052..ee1add7f 100644 --- a/web-app/vite.config.ts +++ b/web-app/vite.config.ts @@ -15,8 +15,8 @@ export default defineConfig({ server: { port: 65432, proxy: { - '/api': 'http://localhost:5432', - '/auth': 'http://localhost:5432', + '/kas': 'http://localhost:8080', + '/auth': 'http://localhost:8888', }, }, }); From f9435a2dc710a08c17d3f30a159a87cdba81b82b Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 10:30:52 -0400 Subject: [PATCH 02/20] WIP allow connecting to docker compose --- .github/workflows/roundtrip/wait-and-test.sh | 10 ++++++ web-app/src/App.tsx | 19 ++++++------ web-app/src/config.ts | 27 +++++++++++++++++ web-app/vite-backend.config.ts | 32 ++++++++++++++++++++ 4 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 web-app/src/config.ts create mode 100644 web-app/vite-backend.config.ts diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 67dd9472..648cc534 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -9,6 +9,16 @@ WEB_APP_DIR="$(cd "${ROOT_DIR}/web-app" >/dev/null && pwd)" app_version=$(cd "${ROOT_DIR}/lib" && node -p "require('./package.json').version") echo "[INFO] App version: ${app_version}" +if [ $1 = backend ]; then + VITE_PROXY='{"/api":"http://localhost:5432","/auth":"http://localhost:5432"}' + VITE_TDF_CFG='{oidc:{host:"http://localhost:65432/auth/realms/tdf",clientId:"browsertest"},kas:"http://localhost:65432/api/kas",reader:"https://secure.virtru.com/start?htmlProtocol=1"}' +else + VITE_PROXY='{"/kas":"http://localhost:8080","/auth":"http://localhost:8888"}' + VITE_TDF_CFG='{oidc:{host:"http://localhost:65432/auth/realms/opentdf",clientId:"browsertest"},kas:"http://localhost:65432/kas",reader:"https://secure.virtru.com/start?htmlProtocol=1"}' +fi +export VITE_PROXY +export VITE_TDF_CFG + _wait-for() { echo "[INFO] In retry loop for quickstarted opentdf backend..." limit=5 diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 133f8235..4c7a1ae9 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -4,6 +4,7 @@ import { showSaveFilePicker } from 'native-file-system-adapter'; import './App.css'; import { TDF3Client, type DecryptSource, NanoTDFClient } from '@opentdf/client'; import { type SessionInformation, OidcClient } from './session.js'; +import { c } from './config.js'; function decryptedFileName(encryptedFileName: string): string { // Groups: 1 file 'name' bit @@ -30,8 +31,8 @@ function decryptedFileExtension(encryptedFileName: string): string { } const oidcClient = new OidcClient( - 'http://localhost:65432/auth/realms/opentdf', - 'browsertest', + c.oidc.host, + c.oidc.clientId, 'otdf-sample-web-app' ); @@ -316,8 +317,6 @@ function App() { }), }; }; - // http://localhost:65432/auth/realms/opentdf/protocol/openid-connect/token - // http://localhost:65432/auth/realms/opentdf/protocol/openid-connect/token const handleEncrypt = async () => { if (!inputSource) { @@ -340,7 +339,7 @@ function App() { 'file' in inputSource ? await inputSource.file.arrayBuffer() : randomArrayBuffer(inputSource); - const nanoClient = new NanoTDFClient(oidcClient, 'http://localhost:65432/kas'); + const nanoClient = new NanoTDFClient(oidcClient, c.kas); setDownloadState('Encrypting...'); switch (sinkType) { case 'file': @@ -373,8 +372,8 @@ function App() { const client = new TDF3Client({ authProvider: oidcClient, dpopKeys: oidcClient.getSigningKey(), - kasEndpoint: 'http://localhost:65432/kas', - readerUrl: 'https://secure.virtru.com/start?htmlProtocol=1', + kasEndpoint: c.kas, + readerUrl: c.reader, }); let source: ReadableStream, size: number; const sc = new AbortController(); @@ -442,7 +441,7 @@ function App() { const client = new TDF3Client({ authProvider: oidcClient, dpopKeys: oidcClient.getSigningKey(), - kasEndpoint: 'http://localhost:65432/kas', + kasEndpoint: c.kas, }); const sc = new AbortController(); setStreamController(sc); @@ -528,7 +527,7 @@ function App() { const client = new TDF3Client({ authProvider: oidcClient, dpopKeys: oidcClient.getSigningKey(), - kasEndpoint: 'http://localhost:65432/kas', + kasEndpoint: c.kas, }); try { const sc = new AbortController(); @@ -580,7 +579,7 @@ function App() { if ('url' in inputSource) { throw new Error('Unsupported : fetch the url I guess?'); } - const nanoClient = new NanoTDFClient(oidcClient, 'http://localhost:65432/kas'); + const nanoClient = new NanoTDFClient(oidcClient, c.kas); try { const cipherText = 'file' in inputSource diff --git a/web-app/src/config.ts b/web-app/src/config.ts new file mode 100644 index 00000000..bfb01765 --- /dev/null +++ b/web-app/src/config.ts @@ -0,0 +1,27 @@ +export type TDFConfig = { + oidc: { + // eg 'http://localhost:65432/auth/realms/opentdf' + host: string; + // eg browsertest + clientId: string; + }; + kas: string; + reader: string; +}; + +function cfg(): TDFConfig { + const { VITE_TDF_CFG } = import.meta.env; + if (!VITE_TDF_CFG) { + return { + oidc: { + host: 'http://localhost:65432/auth/realms/opentdf', + clientId: 'browsertest', + }, + kas: 'http://localhost:65432/kas', + reader: 'https://secure.virtru.com/start?htmlProtocol=1', + }; + } + return JSON.parse(VITE_TDF_CFG); +} + +export const c = cfg(); diff --git a/web-app/vite-backend.config.ts b/web-app/vite-backend.config.ts new file mode 100644 index 00000000..7ea73314 --- /dev/null +++ b/web-app/vite-backend.config.ts @@ -0,0 +1,32 @@ +import { createRequire } from 'node:module' +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +const require = createRequire(import.meta.url) + +function proxy(): Record { + const { VITE_PROXY } = process.env; + if (VITE_PROXY) { + return JSON.parse(VITE_PROXY); + } + return { + '/kas': 'http://localhost:8080', + '/auth': 'http://localhost:8888', + }; +} + + + +// https://vitejs.dev/config/ +export default defineConfig({ + build: { + rollupOptions: { + shimMissingExports: true + } + }, + plugins: [react()], + server: { + port: 65432, + proxy: proxy(), + }, +}); From a029de4f3cdc5df9f2d41c5d145f68a9d414f33c Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 13:12:58 -0400 Subject: [PATCH 03/20] fixes for both --- .github/workflows/build.yaml | 4 +-- .github/workflows/roundtrip/wait-and-test.sh | 7 +++-- lib/tdf3/src/client/builders.ts | 2 ++ lib/tdf3/src/client/index.ts | 22 +++++++++----- remote-store/package-lock.json | 2 +- web-app/package-lock.json | 28 ++++++++++------- web-app/package.json | 4 +-- web-app/src/App.tsx | 7 +++-- web-app/src/session.ts | 7 ++--- web-app/vite-backend.config.ts | 32 -------------------- web-app/vite.config.ts | 19 +++++++++--- 11 files changed, 65 insertions(+), 69 deletions(-) delete mode 100644 web-app/vite-backend.config.ts diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4bb26990..546d5335 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -167,7 +167,7 @@ jobs: kubernetes-version: 1.30.0 - name: Run tilt ci env: - TEST_SCRIPT: ./wait-and-test.sh + TEST_SCRIPT: ./wait-and-test.sh backend OPENTDF_INGRESS_HOST_PORT: "5432" PLAYWRIGHT_TESTS_TO_RUN: roundtrip run: tilt ci @@ -198,7 +198,7 @@ jobs: OPENTDF_SERVICES_AUTHORIZATION_URL: "http://localhost:65432/" PLAYWRIGHT_TESTS_TO_RUN: roundtrip run: |- - ./.github/workflows/roundtrip/wait-and-test.sh + ./.github/workflows/roundtrip/wait-and-test.sh platform deliver-ghp: needs: [lib, web-app, scripts, backend-roundtrip] diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 648cc534..0c239bcf 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -11,14 +11,17 @@ echo "[INFO] App version: ${app_version}" if [ $1 = backend ]; then VITE_PROXY='{"/api":"http://localhost:5432","/auth":"http://localhost:5432"}' - VITE_TDF_CFG='{oidc:{host:"http://localhost:65432/auth/realms/tdf",clientId:"browsertest"},kas:"http://localhost:65432/api/kas",reader:"https://secure.virtru.com/start?htmlProtocol=1"}' + VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/tdf","clientId":"browsertest"},"kas":"http://localhost:65432/api/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' else VITE_PROXY='{"/kas":"http://localhost:8080","/auth":"http://localhost:8888"}' - VITE_TDF_CFG='{oidc:{host:"http://localhost:65432/auth/realms/opentdf",clientId:"browsertest"},kas:"http://localhost:65432/kas",reader:"https://secure.virtru.com/start?htmlProtocol=1"}' + VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/opentdf","clientId":"browsertest"},"kas":"http://localhost:65432/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' fi export VITE_PROXY export VITE_TDF_CFG + +# VITE_PROXY='{"/api":"http://localhost:5432","/auth":"http://localhost:5432"}' VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/tdf","clientId":"browsertest"},"kas":"http://localhost:65432/api/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' npm run dev + _wait-for() { echo "[INFO] In retry loop for quickstarted opentdf backend..." limit=5 diff --git a/lib/tdf3/src/client/builders.ts b/lib/tdf3/src/client/builders.ts index 99ee102a..f88c008c 100644 --- a/lib/tdf3/src/client/builders.ts +++ b/lib/tdf3/src/client/builders.ts @@ -7,6 +7,7 @@ import { IllegalArgumentError } from '../../../src/errors.js'; import { PemKeyPair } from '../crypto/declarations.js'; import { EntityObject } from '../../../src/tdf/EntityObject.js'; import { DecoratedReadableStream } from './DecoratedReadableStream.js'; +import { type Chunker } from '../utils/chunkers.js'; export const DEFAULT_SEGMENT_SIZE: number = 1024 * 1024; export type Scope = { @@ -470,6 +471,7 @@ export type DecryptStreamMiddleware = ( export type DecryptSource = | { type: 'buffer'; location: Uint8Array } + | { type: 'chunker'; location: Chunker } | { type: 'remote'; location: string } | { type: 'stream'; location: ReadableStream } | { type: 'file-browser'; location: Blob }; diff --git a/lib/tdf3/src/client/index.ts b/lib/tdf3/src/client/index.ts index 367b3114..fa56a955 100644 --- a/lib/tdf3/src/client/index.ts +++ b/lib/tdf3/src/client/index.ts @@ -95,14 +95,20 @@ const makeChunkable = async (source: DecryptSource) => { // we don't support streams anyways (see zipreader.js) let initialChunker: Chunker; let buf = null; - if (source.type === 'stream') { - buf = await streamToBuffer(source.location); - initialChunker = fromBuffer(buf); - } else if (source.type === 'buffer') { - buf = source.location; - initialChunker = fromBuffer(buf); - } else { - initialChunker = await fromDataSource(source); + switch (source.type) { + case 'stream': + buf = await streamToBuffer(source.location); + initialChunker = fromBuffer(buf); + break; + case 'buffer': + buf = source.location; + initialChunker = fromBuffer(buf); + break; + case 'chunker': + initialChunker = source.location; + break; + default: + initialChunker = await fromDataSource(source); } const magic: string = await getFirstTwoBytes(initialChunker); diff --git a/remote-store/package-lock.json b/remote-store/package-lock.json index 8702087c..d9ca751c 100644 --- a/remote-store/package-lock.json +++ b/remote-store/package-lock.json @@ -1649,7 +1649,7 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-YAVQuHInxW35fzR0AklXxVTkuMvGhSGwoN+3XZZhSkokr1+FSuzBzGh7xrTNsq9MbDQ6z1lLdwrfuBlusZfoBQ==", + "integrity": "sha512-Qx2Ku9H6HdF1RHa7IRaJ0WZ2NcLHb93jq6bx943IqEw6wPidaHwS868DGCwE5hV3CwShA6Lrv2LGq1dko7n+WQ==", "dependencies": { "axios": "^1.6.1", "axios-retry": "^3.9.0", diff --git a/web-app/package-lock.json b/web-app/package-lock.json index 9a9289a2..04b03e61 100644 --- a/web-app/package-lock.json +++ b/web-app/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@opentdf/client": "file:../lib/opentdf-client-2.0.0.tgz", "clsx": "^2.0.0", - "native-file-system-adapter": "^3.0.0", + "native-file-system-adapter": "^3.0.1", "react": "^18.2.0", "react-dom": "^18.2.0" }, @@ -20,7 +20,7 @@ "@rollup/plugin-inject": "^5.0.3", "@types/react": "^18.2.17", "@types/react-dom": "^18.2.7", - "@types/wicg-file-system-access": "^2020.9.6", + "@types/wicg-file-system-access": "^2023.10.5", "@typescript-eslint/eslint-plugin": "^6.2.1", "@typescript-eslint/parser": "^6.2.1", "@vitejs/plugin-react": "^4.0.4", @@ -602,7 +602,7 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-YAVQuHInxW35fzR0AklXxVTkuMvGhSGwoN+3XZZhSkokr1+FSuzBzGh7xrTNsq9MbDQ6z1lLdwrfuBlusZfoBQ==", + "integrity": "sha512-Qx2Ku9H6HdF1RHa7IRaJ0WZ2NcLHb93jq6bx943IqEw6wPidaHwS868DGCwE5hV3CwShA6Lrv2LGq1dko7n+WQ==", "dependencies": { "axios": "^1.6.1", "axios-retry": "^3.9.0", @@ -749,9 +749,10 @@ "license": "MIT" }, "node_modules/@types/wicg-file-system-access": { - "version": "2020.9.6", - "dev": true, - "license": "MIT" + "version": "2023.10.5", + "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2023.10.5.tgz", + "integrity": "sha512-e9kZO9kCdLqT2h9Tw38oGv9UNzBBWaR1MzuAavxPcsV/7FJ3tWbU6RI3uB+yKIDPGLkGVbplS52ub0AcRLvrhA==", + "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.2.1", @@ -2543,7 +2544,9 @@ } }, "node_modules/native-file-system-adapter": { - "version": "3.0.0", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/native-file-system-adapter/-/native-file-system-adapter-3.0.1.tgz", + "integrity": "sha512-ocuhsYk2SY0906LPc3QIMW+rCV3MdhqGiy7wV5Bf0e8/5TsMjDdyIwhNiVPiKxzTJLDrLT6h8BoV9ERfJscKhw==", "funding": [ { "type": "github", @@ -2554,7 +2557,6 @@ "url": "https://paypal.me/jimmywarting" } ], - "license": "MIT", "engines": { "node": ">=14.8.0" }, @@ -4071,7 +4073,7 @@ }, "@opentdf/client": { "version": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-YAVQuHInxW35fzR0AklXxVTkuMvGhSGwoN+3XZZhSkokr1+FSuzBzGh7xrTNsq9MbDQ6z1lLdwrfuBlusZfoBQ==", + "integrity": "sha512-Qx2Ku9H6HdF1RHa7IRaJ0WZ2NcLHb93jq6bx943IqEw6wPidaHwS868DGCwE5hV3CwShA6Lrv2LGq1dko7n+WQ==", "requires": { "axios": "^1.6.1", "axios-retry": "^3.9.0", @@ -4173,7 +4175,9 @@ "dev": true }, "@types/wicg-file-system-access": { - "version": "2020.9.6", + "version": "2023.10.5", + "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2023.10.5.tgz", + "integrity": "sha512-e9kZO9kCdLqT2h9Tw38oGv9UNzBBWaR1MzuAavxPcsV/7FJ3tWbU6RI3uB+yKIDPGLkGVbplS52ub0AcRLvrhA==", "dev": true }, "@typescript-eslint/eslint-plugin": { @@ -5242,7 +5246,9 @@ "dev": true }, "native-file-system-adapter": { - "version": "3.0.0", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/native-file-system-adapter/-/native-file-system-adapter-3.0.1.tgz", + "integrity": "sha512-ocuhsYk2SY0906LPc3QIMW+rCV3MdhqGiy7wV5Bf0e8/5TsMjDdyIwhNiVPiKxzTJLDrLT6h8BoV9ERfJscKhw==", "requires": { "fetch-blob": "^3.2.0" } diff --git a/web-app/package.json b/web-app/package.json index abb22566..84f1ac94 100644 --- a/web-app/package.json +++ b/web-app/package.json @@ -17,7 +17,7 @@ "dependencies": { "@opentdf/client": "file:../lib/opentdf-client-2.0.0.tgz", "clsx": "^2.0.0", - "native-file-system-adapter": "^3.0.0", + "native-file-system-adapter": "^3.0.1", "react": "^18.2.0", "react-dom": "^18.2.0" }, @@ -26,7 +26,7 @@ "@rollup/plugin-inject": "^5.0.3", "@types/react": "^18.2.17", "@types/react-dom": "^18.2.7", - "@types/wicg-file-system-access": "^2020.9.6", + "@types/wicg-file-system-access": "^2023.10.5", "@typescript-eslint/eslint-plugin": "^6.2.1", "@typescript-eslint/parser": "^6.2.1", "@vitejs/plugin-react": "^4.0.4", diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 4c7a1ae9..3464a37f 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -2,7 +2,7 @@ import { clsx } from 'clsx'; import { useState, useEffect, type ChangeEvent } from 'react'; import { showSaveFilePicker } from 'native-file-system-adapter'; import './App.css'; -import { TDF3Client, type DecryptSource, NanoTDFClient } from '@opentdf/client'; +import { type Chunker, type DecryptSource, NanoTDFClient, TDF3Client } from '@opentdf/client'; import { type SessionInformation, OidcClient } from './session.js'; import { c } from './config.js'; @@ -62,6 +62,7 @@ async function getNewFileHandle( ], suggestedName, }; + //@ts-expect-error //TS2739: not a complete file picker interface return showSaveFilePicker(options); } @@ -629,6 +630,8 @@ function App() { let source; if ('file' in inputSource) { source = inputSource.file.stream() as unknown as ReadableStream; + } else if ('type' in inputSource) { + throw new Error('Unimplemented'); } else { const sc = new AbortController(); setStreamController(sc); @@ -725,7 +728,7 @@ function App() { {hasFileInput ? (

- {'file' in inputSource ? inputSource.file.name : inputSource.url.toString()} + {'file' in inputSource ? inputSource.file.name : '[rand]'}

{'file' in inputSource && ( <> diff --git a/web-app/src/session.ts b/web-app/src/session.ts index 0939eddd..a6f7bb64 100644 --- a/web-app/src/session.ts +++ b/web-app/src/session.ts @@ -403,13 +403,13 @@ export class OidcClient implements AuthProvider { } const { accessToken } = user; const { signingKey } = this; - if (!signingKey) { + if (!signingKey || !signingKey.publicKey) { console.error('missing DPoP key'); return httpReq; } console.info( `signing request for ${httpReq.url} with DPoP key ${JSON.stringify( - await crypto.subtle.exportKey('jwk', this.signingKey.publicKey) + await crypto.subtle.exportKey('jwk', signingKey.publicKey) )}` ); const dpopToken = await dpopFn( @@ -419,9 +419,6 @@ export class OidcClient implements AuthProvider { /* nonce */ undefined, accessToken ); - if (this.wrapperPubKey) { - httpReq = withHeaders(httpReq, { 'X-VirtruPubKey': this.wrapperPubKey }); - } // TODO: Consider: only set DPoP if cnf.jkt is present in access token? return withHeaders(httpReq, { Authorization: `Bearer ${accessToken}`, DPoP: dpopToken }); } diff --git a/web-app/vite-backend.config.ts b/web-app/vite-backend.config.ts deleted file mode 100644 index 7ea73314..00000000 --- a/web-app/vite-backend.config.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { createRequire } from 'node:module' -import { defineConfig } from 'vite'; -import react from '@vitejs/plugin-react'; - -const require = createRequire(import.meta.url) - -function proxy(): Record { - const { VITE_PROXY } = process.env; - if (VITE_PROXY) { - return JSON.parse(VITE_PROXY); - } - return { - '/kas': 'http://localhost:8080', - '/auth': 'http://localhost:8888', - }; -} - - - -// https://vitejs.dev/config/ -export default defineConfig({ - build: { - rollupOptions: { - shimMissingExports: true - } - }, - plugins: [react()], - server: { - port: 65432, - proxy: proxy(), - }, -}); diff --git a/web-app/vite.config.ts b/web-app/vite.config.ts index ee1add7f..aaccc801 100644 --- a/web-app/vite.config.ts +++ b/web-app/vite.config.ts @@ -4,6 +4,20 @@ import react from '@vitejs/plugin-react'; const require = createRequire(import.meta.url) +function proxy(): Record { + console.log(process.env); + const { VITE_PROXY } = process.env; + if (VITE_PROXY) { + console.log(`using VITE_PROXY [${VITE_PROXY}]`); + return JSON.parse(VITE_PROXY); + } + console.log("using standard VITE_PROXY"); + return { + '/kas': 'http://localhost:8080', + '/auth': 'http://localhost:8888', + }; +} + // https://vitejs.dev/config/ export default defineConfig({ build: { @@ -14,9 +28,6 @@ export default defineConfig({ plugins: [react()], server: { port: 65432, - proxy: { - '/kas': 'http://localhost:8080', - '/auth': 'http://localhost:8888', - }, + proxy: proxy(), }, }); From d6f7697a48e9263092d8735bbbf39d429baab4d2 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 13:14:06 -0400 Subject: [PATCH 04/20] Update App.tsx --- web-app/src/App.tsx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 3464a37f..f9429008 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -30,11 +30,7 @@ function decryptedFileExtension(encryptedFileName: string): string { return m[2]; } -const oidcClient = new OidcClient( - c.oidc.host, - c.oidc.clientId, - 'otdf-sample-web-app' -); +const oidcClient = new OidcClient(c.oidc.host, c.oidc.clientId, 'otdf-sample-web-app'); function saver(blob: Blob, name: string) { const a = document.createElement('a'); @@ -727,9 +723,7 @@ function App() { Source {hasFileInput ? (
-

- {'file' in inputSource ? inputSource.file.name : '[rand]'} -

+

{'file' in inputSource ? inputSource.file.name : '[rand]'}

{'file' in inputSource && ( <>
Content Type: {inputSource.file.type}
From bc8a6a00a38df3bcdd16ce7cd3567be7601a73cc Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 13:17:46 -0400 Subject: [PATCH 05/20] Update build.yaml --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 546d5335..a3e5fa22 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -154,7 +154,7 @@ jobs: kubectl: '1.30.0' helm: '3.14.4' tilt: '0.33.13' - - run: |s + - run: | kubectl version --client kustomize version tilt version From 329df5c34b60b91cee895b3718d78dd5805e9752 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 14:11:37 -0400 Subject: [PATCH 06/20] fixes for login and platform ci --- .github/workflows/build.yaml | 2 +- web-app/src/session.ts | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a3e5fa22..2362546a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -198,7 +198,7 @@ jobs: OPENTDF_SERVICES_AUTHORIZATION_URL: "http://localhost:65432/" PLAYWRIGHT_TESTS_TO_RUN: roundtrip run: |- - ./.github/workflows/roundtrip/wait-and-test.sh platform + ./wait-and-test.sh platform deliver-ghp: needs: [lib, web-app, scripts, backend-roundtrip] diff --git a/web-app/src/session.ts b/web-app/src/session.ts index a6f7bb64..f5aff0ff 100644 --- a/web-app/src/session.ts +++ b/web-app/src/session.ts @@ -250,18 +250,25 @@ export class OidcClient implements AuthProvider { window.location.href = whereto; } + _cs?: Promise; + async currentSession(): Promise { - const s = await this.handleRedirect(); - if (s) { - console.log('redirected'); - return s; - } - const sessions = await this.loadSessions(); - if (!sessions?.lastRequest) { - return { sessionState: 'start' }; + if (!this._cs) { + this._cs = (async (): Promise => { + const s = await this.handleRedirect(); + if (s) { + console.log('redirected'); + return s; + } + const sessions = await this.loadSessions(); + if (!sessions?.lastRequest) { + return { sessionState: 'start' }; + } + const thisSession = sessions.requests[sessions.lastRequest]; + return thisSession; + })(); } - const thisSession = sessions.requests[sessions.lastRequest]; - return thisSession; + return this._cs; } async currentUser(): Promise { From bcde8e825b31d8d2c6f2a3373f88a457c34130d8 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 14:36:25 -0400 Subject: [PATCH 07/20] Revert "chore: Removes ajv validation of attribute set" This reverts commit 13a335fc8ffcc2eff699ae7fdc7b7a6c713d868a. --- lib/package.json | 1 + lib/tdf3/src/models/attribute-set.ts | 36 ++++++++++++++++++++++ lib/tests/mocha/unit/attribute-set.spec.ts | 8 ++--- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/lib/package.json b/lib/package.json index d26fa91b..ade6bfb1 100644 --- a/lib/package.json +++ b/lib/package.json @@ -59,6 +59,7 @@ "watch": "(trap 'kill 0' SIGINT; npm run build && (npm run build:watch & npm run test -- --watch))" }, "dependencies": { + "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", diff --git a/lib/tdf3/src/models/attribute-set.ts b/lib/tdf3/src/models/attribute-set.ts index f06618b2..42bb54c4 100644 --- a/lib/tdf3/src/models/attribute-set.ts +++ b/lib/tdf3/src/models/attribute-set.ts @@ -1,3 +1,4 @@ +import Ajv, { JSONSchemaType } from 'ajv'; import { decodeJwt } from 'jose'; export type AttributeObject = { @@ -10,6 +11,32 @@ export type AttributeObject = { jwt?: string; }; +const ATTRIBUTE_OBJECT_SCHEMA: JSONSchemaType = { + $id: '/AttributeObject', + type: 'object', + properties: { + attribute: { type: 'string' }, + displayName: { type: 'string', nullable: true }, + isDefault: { type: 'boolean', nullable: true }, + pubKey: { type: 'string' }, + kasUrl: { type: 'string' }, + kid: { type: 'string', nullable: true }, + jwt: { type: 'string', nullable: true }, + }, + required: ['attribute', 'pubKey', 'kasUrl'], + additionalProperties: false, +}; + +const validator = (() => { + try { + //@ts-expect-error: Ajv not a constructor + return new Ajv(); + } catch (e) { + console.log(e); + } + return new Ajv.default(); +})(); + export class AttributeSet { attributes: AttributeObject[]; @@ -68,6 +95,15 @@ export class AttributeSet { * @return the attribute object if successful, or null */ addAttribute(attrObj: AttributeObject): AttributeObject | null { + // Check shape of object. Reject semi-silently if malformed. + const validate = validator.compile(ATTRIBUTE_OBJECT_SCHEMA); + const result = validate(attrObj); + if (!result) { + // TODO: Determine if an error should be thrown + // console.log("WARNING - AttributeSet.addAttribute: AttributeObject is malformed. AddAttribute failed:"); + if (verbose) console.log(attrObj); + return null; + } // Check for duplicate entries to assure idempotency. if (this.has(attrObj.attribute)) { // This may be a common occurance, so only un-comment this log message diff --git a/lib/tests/mocha/unit/attribute-set.spec.ts b/lib/tests/mocha/unit/attribute-set.spec.ts index 87084b49..87a24d59 100644 --- a/lib/tests/mocha/unit/attribute-set.spec.ts +++ b/lib/tests/mocha/unit/attribute-set.spec.ts @@ -89,7 +89,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes[0], expected); }); - it.skip('should not add one with additional fields (malformed)', () => { + it('should not add one with additional fields (malformed)', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); expected.addedField = 'Potential mallware'; @@ -134,7 +134,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes[0], expected); }); - it.skip('should not add an attribute object with "attribute" missing ', () => { + it('should not add an attribute object with "attribute" missing ', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); delete expected.attribute; @@ -142,7 +142,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes.length, 0); }); - it.skip('should not add an attribute object with "pubKey" missing ', () => { + it('should not add an attribute object with "pubKey" missing ', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); delete expected.pubKey; @@ -150,7 +150,7 @@ describe('AttributeSet', function () { assert.equal(aSet.attributes.length, 0); }); - it.skip('should not add an attribute object with "kasUrl" missing ', () => { + it('should not add an attribute object with "kasUrl" missing ', () => { const aSet = new AttributeSet(); const expected = Mocks.createAttribute({}); delete expected.kasUrl; From a4e60a729dad9177db6bc9d75a32497fed4b5368 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 17:03:32 -0400 Subject: [PATCH 08/20] let nanotdf constructors more closely match tdf and also lets them use RSA signing keys if passed in --- lib/README.md | 2 +- lib/package-lock.json | 60 ++++++++-- lib/src/index.ts | 48 ++++---- lib/src/nanotdf/Client.ts | 154 +++++++++++++----------- lib/tdf3/src/models/attribute-set.ts | 4 +- lib/tests/web/nano-roundtrip.test.ts | 4 +- lib/tests/web/nanotdf/Client.test.ts | 4 +- remote-store/package-lock.json | 173 --------------------------- web-app/package-lock.json | 68 +++++++++-- web-app/src/App.tsx | 12 +- 10 files changed, 240 insertions(+), 289 deletions(-) diff --git a/lib/README.md b/lib/README.md index 571e3919..18566d4a 100644 --- a/lib/README.md +++ b/lib/README.md @@ -15,7 +15,7 @@ TDF3 with JSON envelopes. oidcOrigin: keycloakUrl, } const authProvider = await AuthProviders.refreshAuthProvider(oidcCredentials); - const client = new NanoTDFClient(authProvider, access); + const client = new NanoTDFClient({authProvider, kasEndpoint}); const cipherText = await client.encrypt(plainText); const clearText = await client.decrypt(cipherText); ``` diff --git a/lib/package-lock.json b/lib/package-lock.json index fa7bf843..7c9fb27a 100644 --- a/lib/package-lock.json +++ b/lib/package-lock.json @@ -9,6 +9,7 @@ "version": "2.0.0", "license": "BSD-3-Clause-Clear", "dependencies": { + "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -2786,6 +2787,21 @@ "node": ">=8" } }, + "node_modules/ajv": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", + "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -4895,7 +4911,6 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", - "dev": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -6681,6 +6696,11 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "dev": true, @@ -8770,7 +8790,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dev": true, "engines": { "node": ">=6" } @@ -9057,6 +9076,14 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -10514,7 +10541,6 @@ }, "node_modules/uri-js": { "version": "4.4.1", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" @@ -13183,6 +13209,17 @@ "indent-string": "^4.0.0" } }, + "ajv": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", + "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", + "requires": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + } + }, "ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -14688,8 +14725,7 @@ } }, "fast-deep-equal": { - "version": "3.1.3", - "dev": true + "version": "3.1.3" }, "fast-fifo": { "version": "1.3.0", @@ -15935,6 +15971,11 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", "dev": true @@ -17522,8 +17563,7 @@ "punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dev": true + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==" }, "puppeteer-core": { "version": "20.9.0", @@ -17715,6 +17755,11 @@ "version": "2.1.1", "dev": true }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" + }, "require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -18747,7 +18792,6 @@ }, "uri-js": { "version": "4.4.1", - "dev": true, "requires": { "punycode": "^2.1.0" } diff --git a/lib/src/index.ts b/lib/src/index.ts index 25a39f73..20173bb3 100644 --- a/lib/src/index.ts +++ b/lib/src/index.ts @@ -10,7 +10,7 @@ import { } from './nanotdf/index.js'; import { keyAgreement, extractPublicFromCertToCrypto } from './nanotdf-crypto/index.js'; import { TypedArray, createAttribute, Policy } from './tdf/index.js'; -import { type AuthProvider } from './auth/auth.js'; +import { ClientConfig } from './nanotdf/Client.js'; async function fetchKasPubKey(kasUrl: string): Promise { const kasPubKeyResponse = await fetch(`${kasUrl}/kas_public_key?algorithm=ec:secp256r1`); @@ -33,13 +33,14 @@ async function fetchKasPubKey(kasUrl: string): Promise { * const KAS_URL = 'http://localhost:65432/api/kas/'; * * const ciphertext = '...'; - * const client = new NanoTDFClient( - * await clientSecretAuthProvider({ + * const client = new NanoTDFClient({ + * authProvider: await clientSecretAuthProvider({ * clientId: 'tdf-client', * clientSecret: '123-456', * oidcOrigin: OIDC_ENDPOINT, * }), - * KAS_URL + * kasEndpoint: KAS_URL + * } * ); * client.decrypt(ciphertext) * .then(plaintext => { @@ -120,9 +121,9 @@ export class NanoTDFClient extends Client { */ async encrypt(data: string | TypedArray | ArrayBuffer): Promise { // For encrypt always generate the client ephemeralKeyPair - const ephemeralKeyPair = await this.generateEphemeralKeyPair(); - + const ephemeralKeyPair = await this.ephemeralKeyPair; const initializationVector = this.iv; + if (typeof initializationVector !== 'number') { throw new Error('NanoTDF clients are single use. Please generate a new client and keypair.'); } @@ -174,6 +175,10 @@ export class NanoTDFClient extends Client { } } +export type DatasetConfig = ClientConfig & { + maxKeyIterations?: number; +}; + /** * NanoTDF Dataset SDK Client * @@ -186,15 +191,15 @@ export class NanoTDFClient extends Client { * const KAS_URL = 'http://localhost:65432/api/kas/'; * * const ciphertext = '...'; - * const client = new NanoTDFDatasetClient.default( - * await clientSecretAuthProvider({ + * const client = new NanoTDFDatasetClient({ + * authProvider: await clientSecretAuthProvider({ * clientId: 'tdf-client', * clientSecret: '123-456', * exchange: 'client', * oidcOrigin: OIDC_ENDPOINT, * }), - * KAS_URL - * ); + * kasEndpoint: KAS_URL, + * }); * const plaintext = client.decrypt(ciphertext); * console.log('Plaintext', plaintext); * ``` @@ -223,19 +228,18 @@ export class NanoTDFDatasetClient extends Client { * @param ephemeralKeyPair (optional) ephemeral key pair to use * @param maxKeyIterations Max iteration to performe without a key rotation */ - constructor( - authProvider: AuthProvider, - kasUrl: string, - maxKeyIterations: number = NanoTDFDatasetClient.NTDF_MAX_KEY_ITERATIONS, - ephemeralKeyPair?: Required> - ) { - if (maxKeyIterations > NanoTDFDatasetClient.NTDF_MAX_KEY_ITERATIONS) { - throw new Error('Key iteration exceeds max iterations(8388606)'); + constructor(opts: DatasetConfig) { + if ( + opts.maxKeyIterations && + opts.maxKeyIterations > NanoTDFDatasetClient.NTDF_MAX_KEY_ITERATIONS + ) { + throw new Error( + `Key iteration exceeds max iterations(${NanoTDFDatasetClient.NTDF_MAX_KEY_ITERATIONS})` + ); } + super(opts); - super(authProvider, kasUrl, ephemeralKeyPair); - - this.maxKeyIteration = maxKeyIterations; + this.maxKeyIteration = opts.maxKeyIterations || NanoTDFDatasetClient.NTDF_MAX_KEY_ITERATIONS; this.keyIterationCount = 0; } @@ -250,7 +254,7 @@ export class NanoTDFDatasetClient extends Client { // Intial encrypt if (this.keyIterationCount == 0) { // For encrypt always generate the client ephemeralKeyPair - const ephemeralKeyPair = await this.generateEphemeralKeyPair(); + const ephemeralKeyPair = await this.ephemeralKeyPair; if (!this.kasPubKey) { this.kasPubKey = await fetchKasPubKey(this.kasUrl); diff --git a/lib/src/nanotdf/Client.ts b/lib/src/nanotdf/Client.ts index 794fd8db..1c9802b1 100644 --- a/lib/src/nanotdf/Client.ts +++ b/lib/src/nanotdf/Client.ts @@ -16,6 +16,59 @@ import { cryptoPublicToPem, safeUrlCheck, validateSecureUrl } from '../utils.js' const { KeyUsageType, AlgorithmName, NamedCurve } = cryptoEnums; +export interface ClientConfig { + authProvider: AuthProvider; + dpopEnabled?: boolean; + dpopKeys?: Promise; + ephemeralKeyPair?: Promise; + kasEndpoint: string; +} + +function toJWSAlg(c: CryptoKey): string { + const { algorithm } = c; + switch (algorithm.name) { + case 'RSASSA-PKCS1-v1_5': + case 'RSA-PSS': + case 'RSA-OAEP': { + const r = algorithm as RsaHashedKeyGenParams; + switch (r.modulusLength) { + case 2048: + return 'RS256'; + case 3072: + return 'RS384'; + case 3072: + return 'RS512'; + } + } + case 'ECDSA': + case 'ECDH': { + return 'ES256'; + } + } + throw new Error(`Unsupported key algorithm ${JSON.stringify(algorithm)}`); +} + +async function generateEphemeralKeyPair(): Promise { + const { publicKey, privateKey } = await generateKeyPair(); + if (!privateKey || !publicKey) { + throw Error('Key pair generation failed'); + } + return { publicKey, privateKey }; +} + +async function generateSignerKeyPair(): Promise { + const { publicKey, privateKey } = await generateKeyPair({ + type: AlgorithmName.ECDSA, + curve: NamedCurve.P256, + keyUsages: [KeyUsageType.Sign, KeyUsageType.Verify], + isExtractable: true, + }); + if (!privateKey || !publicKey) { + throw Error('Signer key pair generation failed'); + } + return { publicKey, privateKey }; +} + /** * A Client encapsulates sessions interacting with TDF3 and nanoTDF backends, KAS and any * plugin-based sessions like identity and further attribute control. Most importantly, it is responsible @@ -63,8 +116,8 @@ export default class Client { readonly dpopEnabled: boolean; dissems: string[] = []; dataAttributes: string[] = []; - protected ephemeralKeyPair?: Required>; - protected requestSignerKeyPair?: Required>; + protected ephemeralKeyPair: Promise; + protected requestSignerKeyPair: Promise; protected iv?: number; /** @@ -74,59 +127,32 @@ export default class Client { * cannot be changed. If a new ephemeral key is desired it a new client should be initialized. * There is no performance impact for creating a new client IFF the ephemeral key pair is provided. */ - constructor( - authProvider: AuthProvider, - kasUrl: string, - ephemeralKeyPair?: Required>, - dpopEnabled = false - ) { + constructor({ + authProvider, + ephemeralKeyPair, + kasEndpoint, + dpopEnabled, + dpopKeys, + }: ClientConfig) { this.authProvider = authProvider; // TODO Disallow http KAS. For now just log as error - validateSecureUrl(kasUrl); - this.kasUrl = kasUrl; - this.allowedKases = [kasUrl]; + validateSecureUrl(kasEndpoint); + this.kasUrl = kasEndpoint; + this.allowedKases = [kasEndpoint]; this.kasPubKey = ''; - this.dpopEnabled = dpopEnabled; + this.dpopEnabled = !!dpopEnabled; + if (dpopKeys) { + this.requestSignerKeyPair = dpopKeys; + } else { + this.requestSignerKeyPair = generateSignerKeyPair(); + } if (ephemeralKeyPair) { this.ephemeralKeyPair = ephemeralKeyPair; - this.iv = 1; + } else { + this.ephemeralKeyPair = generateEphemeralKeyPair(); } - } - - /** - * Get ephemeral key pair - * - * Returns the ephemeral key pair to be used in other clients or undefined if not set or generated - * - * @security allow returning ephemeral key pair has unknown security risks. - */ - getEphemeralKeyPair(): CryptoKeyPair | undefined { - return this.ephemeralKeyPair; - } - - async generateEphemeralKeyPair(): Promise>> { - const { publicKey, privateKey } = await generateKeyPair(); - if (!privateKey || !publicKey) { - throw Error('Key pair generation failed'); - } - this.ephemeralKeyPair = { publicKey, privateKey }; this.iv = 1; - return { publicKey, privateKey }; - } - - async generateSignerKeyPair(): Promise>> { - const { publicKey, privateKey } = await generateKeyPair({ - type: AlgorithmName.ECDSA, - curve: NamedCurve.P256, - keyUsages: [KeyUsageType.Sign, KeyUsageType.Verify], - isExtractable: true, - }); - if (!privateKey || !publicKey) { - throw Error('Signer key pair generation failed'); - } - this.requestSignerKeyPair = { publicKey, privateKey }; - return { publicKey, privateKey }; } /** @@ -150,18 +176,7 @@ export default class Client { * either be set on the first call or passed in the constructor. */ async fetchOIDCToken(): Promise { - // Generate the ephemeral key pair if not set - const promises: Promise>>[] = []; - if (!this.ephemeralKeyPair) { - promises.push(this.generateEphemeralKeyPair()); - } - - if (!this.requestSignerKeyPair) { - promises.push(this.generateSignerKeyPair()); - } - await Promise.all(promises); - - const signer = this.requestSignerKeyPair; + const signer = await this.requestSignerKeyPair; if (!signer) { throw new Error('Unexpected state'); } @@ -190,13 +205,15 @@ export default class Client { // Ensure the ephemeral key pair has been set or generated (see createOidcServiceProvider) await this.fetchOIDCToken(); + const ephemeralKeyPair = await this.ephemeralKeyPair; + const requestSignerKeyPair = await this.requestSignerKeyPair; // Ensure the ephemeral key pair has been set or generated (see fetchEntityObject) - if (!this.ephemeralKeyPair?.privateKey) { + if (!ephemeralKeyPair?.privateKey) { throw new Error('Ephemeral key has not been set or generated'); } - if (!this.requestSignerKeyPair?.privateKey) { + if (!requestSignerKeyPair?.privateKey) { throw new Error('Signer key has not been set or generated'); } @@ -210,13 +227,13 @@ export default class Client { protocol: Client.KAS_PROTOCOL, header: base64.encodeArrayBuffer(nanoTdfHeader), }, - clientPublicKey: await cryptoPublicToPem(this.ephemeralKeyPair.publicKey), + clientPublicKey: await cryptoPublicToPem(ephemeralKeyPair.publicKey), }); const jwtPayload = { requestBody: requestBodyStr }; const requestBody = { - signedRequestToken: await reqSignature(jwtPayload, this.requestSignerKeyPair.privateKey, { - alg: AlgorithmName.ES256, + signedRequestToken: await reqSignature(jwtPayload, requestSignerKeyPair.privateKey, { + alg: toJWSAlg(requestSignerKeyPair.publicKey), }), }; @@ -239,10 +256,10 @@ export default class Client { const iv = entityWrappedKey.subarray(0, ivLength); const encryptedSharedKey = entityWrappedKey.subarray(ivLength); - let publicKey; + let kasPublicKey; try { // Get session public key as crypto key - publicKey = await pemPublicToCrypto(wrappedKey.sessionPublicKey); + kasPublicKey = await pemPublicToCrypto(wrappedKey.sessionPublicKey); } catch (cause) { throw new Error( `PEM Public Key to crypto public key failed. Is PEM formatted correctly?\n Caused by: ${cause.message}`, @@ -257,12 +274,13 @@ export default class Client { } catch (e) { throw new Error(`Salting hkdf failed\n Caused by: ${e.message}`); } + const { privateKey } = await this.ephemeralKeyPair; // Get the unwrapping key const unwrappingKey = await keyAgreement( // Ephemeral private key - this.ephemeralKeyPair.privateKey, - publicKey, + privateKey, + kasPublicKey, hkdfSalt ); diff --git a/lib/tdf3/src/models/attribute-set.ts b/lib/tdf3/src/models/attribute-set.ts index 42bb54c4..806b6bdf 100644 --- a/lib/tdf3/src/models/attribute-set.ts +++ b/lib/tdf3/src/models/attribute-set.ts @@ -40,6 +40,8 @@ const validator = (() => { export class AttributeSet { attributes: AttributeObject[]; + verbose: boolean = false; + defaultAttribute?: AttributeObject; constructor() { @@ -101,7 +103,7 @@ export class AttributeSet { if (!result) { // TODO: Determine if an error should be thrown // console.log("WARNING - AttributeSet.addAttribute: AttributeObject is malformed. AddAttribute failed:"); - if (verbose) console.log(attrObj); + if (this.verbose) console.log(attrObj); return null; } // Check for duplicate entries to assure idempotency. diff --git a/lib/tests/web/nano-roundtrip.test.ts b/lib/tests/web/nano-roundtrip.test.ts index 5cce2f17..c7e1cd18 100644 --- a/lib/tests/web/nano-roundtrip.test.ts +++ b/lib/tests/web/nano-roundtrip.test.ts @@ -46,14 +46,14 @@ function initSandbox() { return sandbox; } -const kasUrl = 'http://localhost:65432/api/kas'; +const kasEndpoint = 'http://localhost:65432/api/kas'; describe('Local roundtrip Tests', () => { it('roundtrip string', async () => { // const sandbox = initSandbox(); const sandbox = initSandbox(); try { - const client = new NanoTDFClient(authProvider, kasUrl); + const client = new NanoTDFClient({ authProvider, kasEndpoint }); const keyAgreementSpy = sandbox.spy(globalThis.crypto.subtle, 'deriveKey'); sandbox.stub(client, 'rewrapKey').callsFake(async () => keyAgreementSpy.lastCall.returnValue); const cipherText = await client.encrypt('hello world'); diff --git a/lib/tests/web/nanotdf/Client.test.ts b/lib/tests/web/nanotdf/Client.test.ts index 4238be3a..75a883b5 100644 --- a/lib/tests/web/nanotdf/Client.test.ts +++ b/lib/tests/web/nanotdf/Client.test.ts @@ -4,14 +4,14 @@ import Client from '../../../src/nanotdf/Client.js'; describe('nanotdf client', () => { it('Can create a client with a mock EAS', async () => { - const kasUrl = 'https://etheria.local/kas'; + const kasEndpoint = 'https://etheria.local/kas'; const authProvider = await clientAuthProvider({ clientId: 'string', oidcOrigin: 'string', exchange: 'client', clientSecret: 'password', }); - const client = new Client(authProvider, kasUrl); + const client = new Client({ authProvider, kasEndpoint }); expect(client.authProvider).to.be.ok; }); }); diff --git a/remote-store/package-lock.json b/remote-store/package-lock.json index d9ca751c..064f063d 100644 --- a/remote-store/package-lock.json +++ b/remote-store/package-lock.json @@ -14,7 +14,6 @@ "@aws-sdk/middleware-endpoint": "^3.370.0", "@aws-sdk/protocol-http": "^3.370.0", "@aws-sdk/smithy-client": "^3.370.0", - "@opentdf/client": "file:../lib/opentdf-client-2.0.0.tgz", "axios": "^1.6.1" }, "devDependencies": { @@ -1364,17 +1363,6 @@ "node": ">=14.0.0" } }, - "node_modules/@babel/runtime": { - "version": "7.24.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.5.tgz", - "integrity": "sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==", - "dependencies": { - "regenerator-runtime": "^0.14.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1646,36 +1634,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/@opentdf/client": { - "version": "2.0.0", - "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-Qx2Ku9H6HdF1RHa7IRaJ0WZ2NcLHb93jq6bx943IqEw6wPidaHwS868DGCwE5hV3CwShA6Lrv2LGq1dko7n+WQ==", - "dependencies": { - "axios": "^1.6.1", - "axios-retry": "^3.9.0", - "base64-js": "^1.5.1", - "browser-fs-access": "^0.34.1", - "buffer": "^6.0.3", - "buffer-crc32": "^0.2.13", - "dpop": "^1.2.0", - "eventemitter3": "^5.0.1", - "jose": "^4.14.4", - "streamsaver": "^2.0.6", - "uuid": "~9.0.0" - } - }, - "node_modules/@opentdf/client/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -3023,40 +2981,12 @@ "proxy-from-env": "^1.1.0" } }, - "node_modules/axios-retry": { - "version": "3.9.1", - "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.9.1.tgz", - "integrity": "sha512-8PJDLJv7qTTMMwdnbMvrLYuvB47M81wRtxQmEdV5w4rgbTXTt+vtPkXwajOfOdSyv/wZICJOC+/UhXH4aQ/R+w==", - "dependencies": { - "@babel/runtime": "^7.15.4", - "is-retry-allowed": "^2.2.0" - } - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -3093,48 +3023,12 @@ "node": ">=8" } }, - "node_modules/browser-fs-access": { - "version": "0.34.1", - "resolved": "https://registry.npmjs.org/browser-fs-access/-/browser-fs-access-0.34.1.tgz", - "integrity": "sha512-HPaRf2yimp8kWSuWJXc8Mi78dPbDzfduA+Gyq14H4jlMvd6XNfIRm36Y2yRLaa4x0gwcGuepj4zf14oiTlxrxQ==" - }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -3360,14 +3254,6 @@ "node": ">=6.0.0" } }, - "node_modules/dpop": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/dpop/-/dpop-1.4.0.tgz", - "integrity": "sha512-r//f6g3uaDdomkz+3M0NdcEAxSGUVMIhF7lxf06aJZjCFhkH//GeZo6JWpz7Uv025s1w/+M5RDNRGxAGNIzm0g==", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -3588,11 +3474,6 @@ "node": ">=0.10.0" } }, - "node_modules/eventemitter3": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", - "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" - }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -3951,25 +3832,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -4101,17 +3963,6 @@ "node": ">=8" } }, - "node_modules/is-retry-allowed": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", - "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", @@ -4148,14 +3999,6 @@ "@pkgjs/parseargs": "^0.11.0" } }, - "node_modules/jose": { - "version": "4.15.5", - "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.5.tgz", - "integrity": "sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -4908,11 +4751,6 @@ "node": ">=8.10.0" } }, - "node_modules/regenerator-runtime": { - "version": "0.14.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", - "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" - }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5146,17 +4984,6 @@ "spdx-ranges": "^2.0.0" } }, - "node_modules/streamsaver": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/streamsaver/-/streamsaver-2.0.6.tgz", - "integrity": "sha512-LK4e7TfCV8HzuM0PKXuVUfKyCB1FtT9L0EGxsFk5Up8njj0bXK8pJM9+Wq2Nya7/jslmCQwRK39LFm55h7NBTw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - } - ] - }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", diff --git a/web-app/package-lock.json b/web-app/package-lock.json index 04b03e61..d6c465a0 100644 --- a/web-app/package-lock.json +++ b/web-app/package-lock.json @@ -602,8 +602,10 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-Qx2Ku9H6HdF1RHa7IRaJ0WZ2NcLHb93jq6bx943IqEw6wPidaHwS868DGCwE5hV3CwShA6Lrv2LGq1dko7n+WQ==", + "integrity": "sha512-KRBQWLu0UTXGdwYarDb6Js9Bm1vzs2hKCIMrLnIeXw18p8tDRU/9JwtOkuEego9z02tngkubTPHuy3B2MRPFcA==", + "license": "BSD-3-Clause-Clear", "dependencies": { + "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -617,6 +619,26 @@ "uuid": "~9.0.0" } }, + "node_modules/@opentdf/client/node_modules/ajv": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", + "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@opentdf/client/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "node_modules/@playwright/test": { "version": "1.36.2", "dev": true, @@ -1839,7 +1861,6 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -2903,7 +2924,6 @@ }, "node_modules/punycode": { "version": "2.3.0", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3013,6 +3033,14 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.1", "dev": true, @@ -3459,7 +3487,6 @@ }, "node_modules/uri-js": { "version": "4.4.1", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" @@ -4073,8 +4100,9 @@ }, "@opentdf/client": { "version": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-Qx2Ku9H6HdF1RHa7IRaJ0WZ2NcLHb93jq6bx943IqEw6wPidaHwS868DGCwE5hV3CwShA6Lrv2LGq1dko7n+WQ==", + "integrity": "sha512-KRBQWLu0UTXGdwYarDb6Js9Bm1vzs2hKCIMrLnIeXw18p8tDRU/9JwtOkuEego9z02tngkubTPHuy3B2MRPFcA==", "requires": { + "ajv": "^8.12.0", "axios": "^1.6.1", "axios-retry": "^3.9.0", "base64-js": "^1.5.1", @@ -4086,6 +4114,24 @@ "jose": "^4.14.4", "streamsaver": "^2.0.6", "uuid": "~9.0.0" + }, + "dependencies": { + "ajv": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", + "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", + "requires": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + } } }, "@playwright/test": { @@ -4818,8 +4864,7 @@ "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" }, "fast-deep-equal": { - "version": "3.1.3", - "dev": true + "version": "3.1.3" }, "fast-glob": { "version": "3.3.0", @@ -5447,8 +5492,7 @@ "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "punycode": { - "version": "2.3.0", - "dev": true + "version": "2.3.0" }, "queue-microtask": { "version": "1.2.3", @@ -5519,6 +5563,11 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" + }, "resolve": { "version": "1.22.1", "dev": true, @@ -5769,7 +5818,6 @@ }, "uri-js": { "version": "4.4.1", - "dev": true, "requires": { "punycode": "^2.1.0" } diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index f9429008..97498af4 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -336,7 +336,11 @@ function App() { 'file' in inputSource ? await inputSource.file.arrayBuffer() : randomArrayBuffer(inputSource); - const nanoClient = new NanoTDFClient(oidcClient, c.kas); + const nanoClient = new NanoTDFClient({ + authProvider: oidcClient, + kasEndpoint: c.kas, + dpopKeys: oidcClient.getSigningKey(), + }); setDownloadState('Encrypting...'); switch (sinkType) { case 'file': @@ -576,7 +580,11 @@ function App() { if ('url' in inputSource) { throw new Error('Unsupported : fetch the url I guess?'); } - const nanoClient = new NanoTDFClient(oidcClient, c.kas); + const nanoClient = new NanoTDFClient({ + authProvider: oidcClient, + kasEndpoint: c.kas, + dpopKeys: oidcClient.getSigningKey(), + }); try { const cipherText = 'file' in inputSource From b4dab328a5498a121cf34eda1e64b7ef061d76a7 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 17:09:17 -0400 Subject: [PATCH 09/20] fixup --- remote-store/package-lock.json | 207 ++++++++++++++++++++++++++++++++- web-app/package-lock.json | 5 +- 2 files changed, 205 insertions(+), 7 deletions(-) diff --git a/remote-store/package-lock.json b/remote-store/package-lock.json index 064f063d..a248882b 100644 --- a/remote-store/package-lock.json +++ b/remote-store/package-lock.json @@ -14,6 +14,7 @@ "@aws-sdk/middleware-endpoint": "^3.370.0", "@aws-sdk/protocol-http": "^3.370.0", "@aws-sdk/smithy-client": "^3.370.0", + "@opentdf/client": "file:../lib/opentdf-client-2.0.0.tgz", "axios": "^1.6.1" }, "devDependencies": { @@ -1363,6 +1364,17 @@ "node": ">=14.0.0" } }, + "node_modules/@babel/runtime": { + "version": "7.24.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.5.tgz", + "integrity": "sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1634,6 +1646,37 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/@opentdf/client": { + "version": "2.0.0", + "resolved": "file:../lib/opentdf-client-2.0.0.tgz", + "integrity": "sha512-nnnGQrjKyAfqAC36C8ErGTBiTJ2xPCQgG0pN93rzA68k8+M2pX8HS41H5jBg/nPNpP/kAfEdR6liOWz9LPkP6g==", + "dependencies": { + "ajv": "^8.12.0", + "axios": "^1.6.1", + "axios-retry": "^3.9.0", + "base64-js": "^1.5.1", + "browser-fs-access": "^0.34.1", + "buffer": "^6.0.3", + "buffer-crc32": "^0.2.13", + "dpop": "^1.2.0", + "eventemitter3": "^5.0.1", + "jose": "^4.14.4", + "streamsaver": "^2.0.6", + "uuid": "~9.0.0" + } + }, + "node_modules/@opentdf/client/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -2881,6 +2924,21 @@ "node": ">=0.4.0" } }, + "node_modules/ajv": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.13.0.tgz", + "integrity": "sha512-PRA911Blj99jR5RMeTunVbNXMF6Lp4vZXnk5GQjcnUWUTsrXtekg/pnmFFI2u/I36Y/2bITGS30GZCXei6uNkA==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -2981,12 +3039,40 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-retry": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.9.1.tgz", + "integrity": "sha512-8PJDLJv7qTTMMwdnbMvrLYuvB47M81wRtxQmEdV5w4rgbTXTt+vtPkXwajOfOdSyv/wZICJOC+/UhXH4aQ/R+w==", + "dependencies": { + "@babel/runtime": "^7.15.4", + "is-retry-allowed": "^2.2.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -3023,12 +3109,48 @@ "node": ">=8" } }, + "node_modules/browser-fs-access": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/browser-fs-access/-/browser-fs-access-0.34.1.tgz", + "integrity": "sha512-HPaRf2yimp8kWSuWJXc8Mi78dPbDzfduA+Gyq14H4jlMvd6XNfIRm36Y2yRLaa4x0gwcGuepj4zf14oiTlxrxQ==" + }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -3254,6 +3376,14 @@ "node": ">=6.0.0" } }, + "node_modules/dpop": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/dpop/-/dpop-1.4.0.tgz", + "integrity": "sha512-r//f6g3uaDdomkz+3M0NdcEAxSGUVMIhF7lxf06aJZjCFhkH//GeZo6JWpz7Uv025s1w/+M5RDNRGxAGNIzm0g==", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -3474,11 +3604,15 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-glob": { "version": "3.3.0", @@ -3832,6 +3966,25 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -3963,6 +4116,17 @@ "node": ">=8" } }, + "node_modules/is-retry-allowed": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz", + "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", @@ -3999,6 +4163,14 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/jose": { + "version": "4.15.5", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.5.tgz", + "integrity": "sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -4020,6 +4192,11 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -4625,7 +4802,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dev": true, "engines": { "node": ">=6" } @@ -4751,6 +4927,11 @@ "node": ">=8.10.0" } }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -4760,6 +4941,14 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -4984,6 +5173,17 @@ "spdx-ranges": "^2.0.0" } }, + "node_modules/streamsaver": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/streamsaver/-/streamsaver-2.0.6.tgz", + "integrity": "sha512-LK4e7TfCV8HzuM0PKXuVUfKyCB1FtT9L0EGxsFk5Up8njj0bXK8pJM9+Wq2Nya7/jslmCQwRK39LFm55h7NBTw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + } + ] + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -5201,7 +5401,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "dependencies": { "punycode": "^2.1.0" } diff --git a/web-app/package-lock.json b/web-app/package-lock.json index d6c465a0..a9144ef8 100644 --- a/web-app/package-lock.json +++ b/web-app/package-lock.json @@ -602,8 +602,7 @@ "node_modules/@opentdf/client": { "version": "2.0.0", "resolved": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-KRBQWLu0UTXGdwYarDb6Js9Bm1vzs2hKCIMrLnIeXw18p8tDRU/9JwtOkuEego9z02tngkubTPHuy3B2MRPFcA==", - "license": "BSD-3-Clause-Clear", + "integrity": "sha512-nnnGQrjKyAfqAC36C8ErGTBiTJ2xPCQgG0pN93rzA68k8+M2pX8HS41H5jBg/nPNpP/kAfEdR6liOWz9LPkP6g==", "dependencies": { "ajv": "^8.12.0", "axios": "^1.6.1", @@ -4100,7 +4099,7 @@ }, "@opentdf/client": { "version": "file:../lib/opentdf-client-2.0.0.tgz", - "integrity": "sha512-KRBQWLu0UTXGdwYarDb6Js9Bm1vzs2hKCIMrLnIeXw18p8tDRU/9JwtOkuEego9z02tngkubTPHuy3B2MRPFcA==", + "integrity": "sha512-nnnGQrjKyAfqAC36C8ErGTBiTJ2xPCQgG0pN93rzA68k8+M2pX8HS41H5jBg/nPNpP/kAfEdR6liOWz9LPkP6g==", "requires": { "ajv": "^8.12.0", "axios": "^1.6.1", From 361bf249f1a4052ed2a1b397130fff6b0c1c6dec Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Fri, 3 May 2024 17:29:52 -0400 Subject: [PATCH 10/20] fixups --- .github/workflows/build.yaml | 77 +++++++----- .github/workflows/large-tests.yaml | 6 +- .../workflows/roundtrip}/demo-evironment.sh | 0 .../workflows/roundtrip}/init-temp-keys.sh | 0 .../roundtrip/mocks/mock-secrets.yaml | 115 +++++++++--------- .github/workflows/roundtrip/mocks/values.yaml | 4 +- .../workflows/roundtrip/opentdf.yaml | 0 .github/workflows/roundtrip/wait-and-test.sh | 1 - 8 files changed, 111 insertions(+), 92 deletions(-) rename {scripts => .github/workflows/roundtrip}/demo-evironment.sh (100%) rename {scripts => .github/workflows/roundtrip}/init-temp-keys.sh (100%) rename opentdf.yaml => .github/workflows/roundtrip/opentdf.yaml (100%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2362546a..a0a96c37 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,8 +1,9 @@ name: Build, Test, and Deliver Client - env: - do_sonarscan: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }} + do_sonarscan: + ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == + github.repository }} on: pull_request: @@ -37,7 +38,7 @@ jobs: textReportPath: './lib/coverage/coverage.txt' - run: echo "${MARKDOWN_REPORT}" >> $GITHUB_STEP_SUMMARY env: - MARKDOWN_REPORT: "${{ steps.coverage-md.outputs.markdownReport }}" + MARKDOWN_REPORT: '${{ steps.coverage-md.outputs.markdownReport }}' - run: npm audit --omit dev && npm audit --audit-level high - run: npm run license-check - run: npm run lint @@ -168,7 +169,7 @@ jobs: - name: Run tilt ci env: TEST_SCRIPT: ./wait-and-test.sh backend - OPENTDF_INGRESS_HOST_PORT: "5432" + OPENTDF_INGRESS_HOST_PORT: '5432' PLAYWRIGHT_TESTS_TO_RUN: roundtrip run: tilt ci @@ -193,9 +194,20 @@ jobs: path: lib/ - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 with: - go-version: "1.22" + go-version: '1.22' + - run: docker compose up -d --wait --wait-timeout 240 + - run: go run github.com/opentdf/platform/service@latest provision keycloak + - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 + name: start server in background + with: + run: | + go run github.com/opentdf/platform/service@latest start + wait-on: | + tcp:localhost:8080 + log-output-if: true + wait-for: 90s - env: - OPENTDF_SERVICES_AUTHORIZATION_URL: "http://localhost:65432/" + OPENTDF_SERVICES_AUTHORIZATION_URL: 'http://localhost:65432/' PLAYWRIGHT_TESTS_TO_RUN: roundtrip run: |- ./wait-and-test.sh platform @@ -206,9 +218,9 @@ jobs: timeout-minutes: 5 # To publish from a release or feature branch, remove the ref == condition below if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main') - || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/')) - || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) + (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == + 'push' && startsWith(github.ref, 'refs/heads/release/')) || (github.event_name == 'release' && + startsWith(github.ref, 'refs/tags/')) outputs: FULL_VERSION: ${{ steps.guess-build-metadata.outputs.FULL_VERSION }} DIST_TAG: ${{ steps.guess-build-metadata.outputs.DIST_TAG }} @@ -236,22 +248,25 @@ jobs: echo "DIST_TAG=$(.github/workflows/guess-dist-tag.sh)" >> $GITHUB_OUTPUT - run: make test - run: make doc - - run: echo "::notice file=lib/package.json::Will be published to [GitHub Packages](https://github.com/opentdf/client-web/pkgs/npm/client) as ${{ steps.guess-build-metadata.outputs.DIST_TAG }} with version=[${{ steps.guess-build-metadata.outputs.FULL_VERSION }}]" + - run: + echo "::notice file=lib/package.json::Will be published to [GitHub + Packages](https://github.com/opentdf/client-web/pkgs/npm/client) as ${{ + steps.guess-build-metadata.outputs.DIST_TAG }} with version=[${{ + steps.guess-build-metadata.outputs.FULL_VERSION }}]" - run: >- - .github/workflows/publish-to.sh - ${{ steps.guess-build-metadata.outputs.FULL_VERSION }} - ${{ steps.guess-build-metadata.outputs.DIST_TAG }} + .github/workflows/publish-to.sh ${{ steps.guess-build-metadata.outputs.FULL_VERSION }} ${{ + steps.guess-build-metadata.outputs.DIST_TAG }} env: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: echo "- [Client Library](https://github.com/opentdf/client-web/pkgs/npm/client)">>$GITHUB_STEP_SUMMARY + - run: + echo "- [Client + Library](https://github.com/opentdf/client-web/pkgs/npm/client)">>$GITHUB_STEP_SUMMARY - name: trigger xtest run: >- - curl -XPOST - -u "virtru-cloudnative:${{secrets.PERSONAL_ACCESS_TOKEN}}" - -H "Accept: application/vnd.github.everest-preview+json" - -H "Content-Type: application/json" - "https://api.github.com/repos/opentdf/backend/dispatches" - --data '{"event_type":"xtest","client_payload":{"version":"'${FULL_VERSION%%+*}'"}}' + curl -XPOST -u "virtru-cloudnative:${{secrets.PERSONAL_ACCESS_TOKEN}}" -H "Accept: + application/vnd.github.everest-preview+json" -H "Content-Type: application/json" + "https://api.github.com/repos/opentdf/backend/dispatches" --data + '{"event_type":"xtest","client_payload":{"version":"'${FULL_VERSION%%+*}'"}}' env: FULL_VERSION: ${{ steps.guess-build-metadata.outputs.FULL_VERSION }} - name: Publish documentation to gh-pages @@ -266,9 +281,9 @@ jobs: environment: npmjs needs: deliver-ghp if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main') - || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/')) - || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) + (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == + 'push' && startsWith(github.ref, 'refs/heads/release/')) || (github.event_name == 'release' && + startsWith(github.ref, 'refs/tags/')) steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 @@ -277,10 +292,16 @@ jobs: registry-url: 'https://registry.npmjs.org' - run: make all - run: >- - .github/workflows/publish-to.sh ${{ needs.deliver-ghp.outputs.FULL_VERSION }} - ${{ needs.deliver-ghp.outputs.DIST_TAG }} + .github/workflows/publish-to.sh ${{ needs.deliver-ghp.outputs.FULL_VERSION }} ${{ + needs.deliver-ghp.outputs.DIST_TAG }} env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - run: echo "- [Client Library](https://www.npmjs.com/package/@opentdf/client/v/${{ needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY - - run: echo "- [Remote Store](https://www.npmjs.com/package/@opentdf/remote-store/v/${{ needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY - - run: echo "- [unpkg](https://unpkg.com/browse/@opentdf/client@${{ needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY + - run: + echo "- [Client Library](https://www.npmjs.com/package/@opentdf/client/v/${{ + needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY + - run: + echo "- [Remote Store](https://www.npmjs.com/package/@opentdf/remote-store/v/${{ + needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY + - run: + echo "- [unpkg](https://unpkg.com/browse/@opentdf/client@${{ + needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY diff --git a/.github/workflows/large-tests.yaml b/.github/workflows/large-tests.yaml index 019bffa8..29117c05 100644 --- a/.github/workflows/large-tests.yaml +++ b/.github/workflows/large-tests.yaml @@ -2,7 +2,7 @@ name: Build, Test, and Deliver Client on: schedule: - - cron: "0 4 * * 2,4" + - cron: '0 4 * * 2,4' jobs: lib: runs-on: ubuntu-latest @@ -106,8 +106,8 @@ jobs: env: #path relative to the quickstart Tiltfile TEST_SCRIPT: ../../wait-and-test.sh - OPENTDF_INGRESS_HOST_PORT: "5432" - OPENTDF_LOAD_FRONTEND: "false" + OPENTDF_INGRESS_HOST_PORT: '5432' + OPENTDF_LOAD_FRONTEND: 'false' PLAYWRIGHT_TESTS_TO_RUN: huge roundtrip run: |- tilt ci --file opentdf/quickstart/Tiltfile diff --git a/scripts/demo-evironment.sh b/.github/workflows/roundtrip/demo-evironment.sh similarity index 100% rename from scripts/demo-evironment.sh rename to .github/workflows/roundtrip/demo-evironment.sh diff --git a/scripts/init-temp-keys.sh b/.github/workflows/roundtrip/init-temp-keys.sh similarity index 100% rename from scripts/init-temp-keys.sh rename to .github/workflows/roundtrip/init-temp-keys.sh diff --git a/.github/workflows/roundtrip/mocks/mock-secrets.yaml b/.github/workflows/roundtrip/mocks/mock-secrets.yaml index 15b3777b..8eb2f711 100644 --- a/.github/workflows/roundtrip/mocks/mock-secrets.yaml +++ b/.github/workflows/roundtrip/mocks/mock-secrets.yaml @@ -1,62 +1,61 @@ KAS_CERTIFICATE: | - -----BEGIN CERTIFICATE----- - MIICmDCCAYACCQC3BCaSANRhYzANBgkqhkiG9w0BAQsFADAOMQwwCgYDVQQDDANr - YXMwHhcNMjEwOTE1MTQxMTQ4WhcNMjIwOTE1MTQxMTQ4WjAOMQwwCgYDVQQDDANr - YXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOpiotrvV2i5h6clHM - zDGgh3h/kMa0LoGx2OkDPd8jogycUh7pgE5GNiN2lpSmFkjxwYMXnyrwr9ExyczB - WJ7sRGDCDaQg5fjVUIloZ8FJVbn+sEcfQ9iX6vmI9/S++oGK79QM3V8M8cp41r/T - 1YVmuzUHE1say/TLHGhjtGkxHDF8qFy6Z2rYFTCVJQHNqGmwNVGd0qG7gim86Haw - u/CMYj4jG9oITlj8rJtQOaJ6ZqemQVoNmb3j1LkyeUKzRIt+86aoBiz+T3TfOEvX - F6xgBj3XoiOhPYK+abFPYcrArvb6oubT8NjjQoj3j0sXWUnIIMg+e4f+XNVU54Zz - DaLZAgMBAAEwDQYJKoZIhvcNAQELBQADggEBABewfZOJ4/KNRE8IQ5TsW/AVn7C1 - l5ty6tUUBSVi8/df7WYts0bHEdQh9yl9agEU5i4rj43y8vMVZNzSeHcurtV/+C0j - fbkHQHeiQ1xn7cq3Sbh4UVRyuu4C5PklEH4AN6gxmgXC3kT15uWw8I4nm/plzYLs - I099IoRfC5djHUYYLMU/VkOIHuPC3sb7J65pSN26eR8bTMVNagk187V/xNwUuvkf - +NUxDO615/5BwQKnAu5xiIVagYnDZqKCOtYS5qhxF33Nlnwlm7hH8iVZ1RI+n52l - wVyElqp317Ksz+GtTIc+DE6oryxK3tZd4hrj9fXT4KiJvQ4pcRjpePgH7B8= - -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIICmDCCAYACCQC3BCaSANRhYzANBgkqhkiG9w0BAQsFADAOMQwwCgYDVQQDDANr + YXMwHhcNMjEwOTE1MTQxMTQ4WhcNMjIwOTE1MTQxMTQ4WjAOMQwwCgYDVQQDDANr + YXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOpiotrvV2i5h6clHM + zDGgh3h/kMa0LoGx2OkDPd8jogycUh7pgE5GNiN2lpSmFkjxwYMXnyrwr9ExyczB + WJ7sRGDCDaQg5fjVUIloZ8FJVbn+sEcfQ9iX6vmI9/S++oGK79QM3V8M8cp41r/T + 1YVmuzUHE1say/TLHGhjtGkxHDF8qFy6Z2rYFTCVJQHNqGmwNVGd0qG7gim86Haw + u/CMYj4jG9oITlj8rJtQOaJ6ZqemQVoNmb3j1LkyeUKzRIt+86aoBiz+T3TfOEvX + F6xgBj3XoiOhPYK+abFPYcrArvb6oubT8NjjQoj3j0sXWUnIIMg+e4f+XNVU54Zz + DaLZAgMBAAEwDQYJKoZIhvcNAQELBQADggEBABewfZOJ4/KNRE8IQ5TsW/AVn7C1 + l5ty6tUUBSVi8/df7WYts0bHEdQh9yl9agEU5i4rj43y8vMVZNzSeHcurtV/+C0j + fbkHQHeiQ1xn7cq3Sbh4UVRyuu4C5PklEH4AN6gxmgXC3kT15uWw8I4nm/plzYLs + I099IoRfC5djHUYYLMU/VkOIHuPC3sb7J65pSN26eR8bTMVNagk187V/xNwUuvkf + +NUxDO615/5BwQKnAu5xiIVagYnDZqKCOtYS5qhxF33Nlnwlm7hH8iVZ1RI+n52l + wVyElqp317Ksz+GtTIc+DE6oryxK3tZd4hrj9fXT4KiJvQ4pcRjpePgH7B8= + -----END CERTIFICATE----- KAS_EC_SECP256R1_CERTIFICATE: | - -----BEGIN CERTIFICATE----- - MIIBCzCBsgIJAL1qc/lWpG3HMAoGCCqGSM49BAMCMA4xDDAKBgNVBAMMA2thczAe - Fw0yMTA5MTUxNDExNDlaFw0yMjA5MTUxNDExNDlaMA4xDDAKBgNVBAMMA2thczBZ - MBMGByqGSM49AgEGCCqGSM49AwEHA0IABH2VM7Ws9SVr19rywr/o3fewDBj+170/ - 6y8zo4leVaJqCl76Nd9QfDNy4KjNCtmmjo6ftTS+iFAhnPCeugAJOWUwCgYIKoZI - zj0EAwIDSAAwRQIhAIFdrqhwvgL8ctPjUtmULXmg2ii0PFKg/Mox2GiCVXQdAiAW - UDdeafEoprE+qc4paMmbWoEpRXLlo+3S7rnc5T12Kw== - -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBCzCBsgIJAL1qc/lWpG3HMAoGCCqGSM49BAMCMA4xDDAKBgNVBAMMA2thczAe + Fw0yMTA5MTUxNDExNDlaFw0yMjA5MTUxNDExNDlaMA4xDDAKBgNVBAMMA2thczBZ + MBMGByqGSM49AgEGCCqGSM49AwEHA0IABH2VM7Ws9SVr19rywr/o3fewDBj+170/ + 6y8zo4leVaJqCl76Nd9QfDNy4KjNCtmmjo6ftTS+iFAhnPCeugAJOWUwCgYIKoZI + zj0EAwIDSAAwRQIhAIFdrqhwvgL8ctPjUtmULXmg2ii0PFKg/Mox2GiCVXQdAiAW + UDdeafEoprE+qc4paMmbWoEpRXLlo+3S7rnc5T12Kw== + -----END CERTIFICATE----- KAS_EC_SECP256R1_PRIVATE_KEY: | - -----BEGIN PRIVATE KEY----- - MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgOK47RufwyqeWbDdC - ojHYxzkZ+VphXbNzZOt2seMavk2hRANCAAR9lTO1rPUla9fa8sK/6N33sAwY/te9 - P+svM6OJXlWiagpe+jXfUHwzcuCozQrZpo6On7U0vohQIZzwnroACTll - -----END PRIVATE KEY----- + -----BEGIN PRIVATE KEY----- + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgOK47RufwyqeWbDdC + ojHYxzkZ+VphXbNzZOt2seMavk2hRANCAAR9lTO1rPUla9fa8sK/6N33sAwY/te9 + P+svM6OJXlWiagpe+jXfUHwzcuCozQrZpo6On7U0vohQIZzwnroACTll + -----END PRIVATE KEY----- KAS_PRIVATE_KEY: | - -----BEGIN PRIVATE KEY----- - MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOpiotrvV2i5h6 - clHMzDGgh3h/kMa0LoGx2OkDPd8jogycUh7pgE5GNiN2lpSmFkjxwYMXnyrwr9Ex - yczBWJ7sRGDCDaQg5fjVUIloZ8FJVbn+sEcfQ9iX6vmI9/S++oGK79QM3V8M8cp4 - 1r/T1YVmuzUHE1say/TLHGhjtGkxHDF8qFy6Z2rYFTCVJQHNqGmwNVGd0qG7gim8 - 6Hawu/CMYj4jG9oITlj8rJtQOaJ6ZqemQVoNmb3j1LkyeUKzRIt+86aoBiz+T3Tf - OEvXF6xgBj3XoiOhPYK+abFPYcrArvb6oubT8NjjQoj3j0sXWUnIIMg+e4f+XNVU - 54ZzDaLZAgMBAAECggEBALb0yK0PlMUyzHnEUwXV1y5AIoAWhsYp0qvJ1msHUVKz - +yQ/VJz4+tQQxI8OvGbbnhNkd5LnWdYkYzsIZl7b/kBCPcQw3Zo+4XLCzhUAn1E1 - M+n42c8le1LtN6Z7mVWoZh7DPONy7t+ABvm7b7S1+1i78DPmgCeWYZGeAhIcPXG6 - 5AxWIV3jigxksE6kYY9Y7DmtsZgMRrdV7SU8VtgPtT7tua8z5/U3Av0WINyKBSoM - 0yDHsAg57KnM8znx2JWLtHd0Mk5bBuu2DLbtyKNrVUAUuMPzrLGBh9S9QRd934KU - uFAi1TEfgEachnGgSHJpzVzr2ur1tifABnQ7GNXObe0CgYEA6KowK0subdDY+uGW - ciP2XDAMerbJJeL0/UIGPb/LUmskniio2493UBGgY2FsRyvbzJ+/UAOjIPyIxhj7 - 78ZyVG8BmIzKan1RRVh//O+5yvks/eTOYjWeQ1Lcgqs3q4YAO13CEBZgKWKTUomg - mskFJq04tndeSIyhDaW+BuWaXA8CgYEA42ABz3pql+DH7oL5C4KYBymK6wFBBOqk - dVk+ftyJQ6PzuZKpfsu4aPIjKm71lkTgK6O9o08s3SckAdu6vLukq2TZFF+a+9OI - lu5ww7GvfdMTgLAaFchD4bPlOInh1KVjBc1MwGXpl0ROde5pi8+WUrv9QJuoQfB/ - 4rhYdbJLSpcCgYA41mqSCPm8pgp7r2RbWeGzP6Gs0L5u3PTQcbKonxQCfF4jrPcj - O/b/vm6aGJClClfVsyi/WUQeqNKY4j2Zo7cGXV/cbnh8b0TNVgNePQn8Rcbx91Vb - tJGHDNUFruIYqtGfrxXbbDvtoEExJqHvbjAt9J8oJB0KSCCH/vdfI/QDjQKBgQCD - xLPH5Y24js/O7aAeh4RLQkv7fTKNAt5kE2AgbPYveOhZ9yC7Fpy8VPcENGGmwCuZ - nr7b0ZqSX4iCezBxB92aZktXf0B2CFT0AyLehi7JoHWA8o1rai/MsVB5v45ciawl - RKDiLy18OF2wAoawO5FGSSOvOYX9EL9MSMEbFESF6QKBgCVlZ9pPC+55rGT6AcEL - tUpDs+/wZvcmfsFd8xC5mMUN0DatAVzVAUI95+tQaWU3Uj+bqHq0lC6Wy2VceG0D - D+7EicjdGFN/2WVPXiYX1fblkxasZY+wChYBrPLjA9g0qOzzmXbRBph5QxDuQjJ6 - qcddVKB624a93ZBssn7OivnR - -----END PRIVATE KEY----- - + -----BEGIN PRIVATE KEY----- + MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOpiotrvV2i5h6 + clHMzDGgh3h/kMa0LoGx2OkDPd8jogycUh7pgE5GNiN2lpSmFkjxwYMXnyrwr9Ex + yczBWJ7sRGDCDaQg5fjVUIloZ8FJVbn+sEcfQ9iX6vmI9/S++oGK79QM3V8M8cp4 + 1r/T1YVmuzUHE1say/TLHGhjtGkxHDF8qFy6Z2rYFTCVJQHNqGmwNVGd0qG7gim8 + 6Hawu/CMYj4jG9oITlj8rJtQOaJ6ZqemQVoNmb3j1LkyeUKzRIt+86aoBiz+T3Tf + OEvXF6xgBj3XoiOhPYK+abFPYcrArvb6oubT8NjjQoj3j0sXWUnIIMg+e4f+XNVU + 54ZzDaLZAgMBAAECggEBALb0yK0PlMUyzHnEUwXV1y5AIoAWhsYp0qvJ1msHUVKz + +yQ/VJz4+tQQxI8OvGbbnhNkd5LnWdYkYzsIZl7b/kBCPcQw3Zo+4XLCzhUAn1E1 + M+n42c8le1LtN6Z7mVWoZh7DPONy7t+ABvm7b7S1+1i78DPmgCeWYZGeAhIcPXG6 + 5AxWIV3jigxksE6kYY9Y7DmtsZgMRrdV7SU8VtgPtT7tua8z5/U3Av0WINyKBSoM + 0yDHsAg57KnM8znx2JWLtHd0Mk5bBuu2DLbtyKNrVUAUuMPzrLGBh9S9QRd934KU + uFAi1TEfgEachnGgSHJpzVzr2ur1tifABnQ7GNXObe0CgYEA6KowK0subdDY+uGW + ciP2XDAMerbJJeL0/UIGPb/LUmskniio2493UBGgY2FsRyvbzJ+/UAOjIPyIxhj7 + 78ZyVG8BmIzKan1RRVh//O+5yvks/eTOYjWeQ1Lcgqs3q4YAO13CEBZgKWKTUomg + mskFJq04tndeSIyhDaW+BuWaXA8CgYEA42ABz3pql+DH7oL5C4KYBymK6wFBBOqk + dVk+ftyJQ6PzuZKpfsu4aPIjKm71lkTgK6O9o08s3SckAdu6vLukq2TZFF+a+9OI + lu5ww7GvfdMTgLAaFchD4bPlOInh1KVjBc1MwGXpl0ROde5pi8+WUrv9QJuoQfB/ + 4rhYdbJLSpcCgYA41mqSCPm8pgp7r2RbWeGzP6Gs0L5u3PTQcbKonxQCfF4jrPcj + O/b/vm6aGJClClfVsyi/WUQeqNKY4j2Zo7cGXV/cbnh8b0TNVgNePQn8Rcbx91Vb + tJGHDNUFruIYqtGfrxXbbDvtoEExJqHvbjAt9J8oJB0KSCCH/vdfI/QDjQKBgQCD + xLPH5Y24js/O7aAeh4RLQkv7fTKNAt5kE2AgbPYveOhZ9yC7Fpy8VPcENGGmwCuZ + nr7b0ZqSX4iCezBxB92aZktXf0B2CFT0AyLehi7JoHWA8o1rai/MsVB5v45ciawl + RKDiLy18OF2wAoawO5FGSSOvOYX9EL9MSMEbFESF6QKBgCVlZ9pPC+55rGT6AcEL + tUpDs+/wZvcmfsFd8xC5mMUN0DatAVzVAUI95+tQaWU3Uj+bqHq0lC6Wy2VceG0D + D+7EicjdGFN/2WVPXiYX1fblkxasZY+wChYBrPLjA9g0qOzzmXbRBph5QxDuQjJ6 + qcddVKB624a93ZBssn7OivnR + -----END PRIVATE KEY----- diff --git a/.github/workflows/roundtrip/mocks/values.yaml b/.github/workflows/roundtrip/mocks/values.yaml index 13f2fdf4..19417565 100644 --- a/.github/workflows/roundtrip/mocks/values.yaml +++ b/.github/workflows/roundtrip/mocks/values.yaml @@ -17,7 +17,7 @@ keycloak: password: bXlQb3N0Z3Jlc1Bhc3N3b3Jk kas: auth: - "http://localhost:65432/auth/realms/tdf": - discoveryBaseUrl: "http://keycloak-http/auth/realms/tdf" + 'http://localhost:65432/auth/realms/tdf': + discoveryBaseUrl: 'http://keycloak-http/auth/realms/tdf' entitlementpdp: opaPolicyPullSecret: my-pat diff --git a/opentdf.yaml b/.github/workflows/roundtrip/opentdf.yaml similarity index 100% rename from opentdf.yaml rename to .github/workflows/roundtrip/opentdf.yaml diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 0c239bcf..48f60dcd 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -19,7 +19,6 @@ fi export VITE_PROXY export VITE_TDF_CFG - # VITE_PROXY='{"/api":"http://localhost:5432","/auth":"http://localhost:5432"}' VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/tdf","clientId":"browsertest"},"kas":"http://localhost:65432/api/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' npm run dev _wait-for() { From 647103e50463bc2e187c78ce7c6a6b9f8bfbd727 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 05:22:20 -0400 Subject: [PATCH 11/20] cleanups --- .github/workflows/build.yaml | 1 + .../workflows/roundtrip/docker-compose.yaml | 51 ++++++++++++------- .github/workflows/roundtrip/opentdf.yaml | 12 ++--- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a0a96c37..682e69ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -206,6 +206,7 @@ jobs: tcp:localhost:8080 log-output-if: true wait-for: 90s + working-directory: .github/workflows/roundtrip - env: OPENTDF_SERVICES_AUTHORIZATION_URL: 'http://localhost:65432/' PLAYWRIGHT_TESTS_TO_RUN: roundtrip diff --git a/.github/workflows/roundtrip/docker-compose.yaml b/.github/workflows/roundtrip/docker-compose.yaml index 4fab6e15..d7b6f8c9 100644 --- a/.github/workflows/roundtrip/docker-compose.yaml +++ b/.github/workflows/roundtrip/docker-compose.yaml @@ -2,10 +2,11 @@ services: keycloak: # This is kc 24.0.1 with opentdf protocol mapper on board image: quay.io/keycloak/keycloak:24.0 - restart: always + restart: + always # To enable debugging, use this CMD and also set and expose the DEBUG_PORT # command: ["--debug", "start-dev", "--log-level=DEBUG"] - command: ["start-dev"] + command: ['start-dev'] environment: # DEBUG_PORT: "*:30012" KC_DB_VENDOR: postgres @@ -14,25 +15,37 @@ services: KC_DB_URL_DATABASE: keycloak KC_DB_USERNAME: keycloak KC_DB_PASSWORD: changeme - KC_FEATURES: "preview,token-exchange" - KC_HEALTH_ENABLED: "true" - KC_HOSTNAME_ADMIN_URL: "http://localhost:65432/auth" - KC_HOSTNAME_PORT: "65432" - KC_HOSTNAME_STRICT: "false" - KC_HOSTNAME_STRICT_BACKCHANNEL: "false" - KC_HOSTNAME_STRICT_HTTPS: "false" - KC_HOSTNAME_URL: "http://localhost:65432/auth" - KC_HTTP_ENABLED: "true" - KC_HTTP_PORT: "8888" - KC_HTTP_RELATIVE_PATH: "/auth" - KC_PROXY: "edge" + KC_FEATURES: 'preview,token-exchange' + KC_HEALTH_ENABLED: 'true' + KC_HOSTNAME_ADMIN_URL: 'http://localhost:65432/auth' + KC_HOSTNAME_PORT: '65432' + KC_HOSTNAME_STRICT: 'false' + KC_HOSTNAME_STRICT_BACKCHANNEL: 'false' + KC_HOSTNAME_STRICT_HTTPS: 'false' + KC_HOSTNAME_URL: 'http://localhost:65432/auth' + KC_HTTP_ENABLED: 'true' + KC_HTTP_PORT: '8888' + KC_HTTP_RELATIVE_PATH: '/auth' + KC_PROXY: 'edge' KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: changeme ports: - - "8888:8888" + - '8888:8888' # - "30012:30012" healthcheck: - test: ['CMD-SHELL', '[ -f /tmp/HealthCheck.java ] || echo "public class HealthCheck { public static void main(String[] args) throws java.lang.Throwable { System.exit(java.net.HttpURLConnection.HTTP_OK == ((java.net.HttpURLConnection)new java.net.URL(args[0]).openConnection()).getResponseCode() ? 0 : 1); } }" > /tmp/HealthCheck.java && java /tmp/HealthCheck.java http://localhost:8888/auth/health/live'] + test: + - CMD-SHELL + - >- + [ -f /tmp/HealthCheck.java ] + || echo "public class HealthCheck { + public static void main(String[] args) throws java.lang.Throwable { + System.exit( + java.net.HttpURLConnection.HTTP_OK == + ((java.net.HttpURLConnection) new java.net.URL(args[0]).openConnection()) + .getResponseCode() ? 0 : 1); + } + }" >/tmp/HealthCheck.java + && java /tmp/HealthCheck.java http://localhost:8888/auth/health/live interval: 5s timeout: 10s retries: 3 @@ -46,7 +59,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: keycloak healthcheck: - test: ["CMD-SHELL", "pg_isready"] + test: ['CMD-SHELL', 'pg_isready'] interval: 5s timeout: 5s retries: 10 @@ -59,9 +72,9 @@ services: POSTGRES_PASSWORD: changeme POSTGRES_DB: opentdf healthcheck: - test: ["CMD-SHELL", "pg_isready"] + test: ['CMD-SHELL', 'pg_isready'] interval: 5s timeout: 5s retries: 10 ports: - - "5432:5432" + - '5432:5432' diff --git a/.github/workflows/roundtrip/opentdf.yaml b/.github/workflows/roundtrip/opentdf.yaml index fad96fdf..d82739f0 100644 --- a/.github/workflows/roundtrip/opentdf.yaml +++ b/.github/workflows/roundtrip/opentdf.yaml @@ -16,18 +16,18 @@ services: authorization: enabled: true url: http://localhost:65432 - client: "tdf-entity-resolution" - secret: "secret" - realm: "opentdf" + client: 'tdf-entity-resolution' + secret: 'secret' + realm: 'opentdf' legacy: true server: auth: enabled: true - audience: "http://localhost:65432" + audience: 'http://localhost:65432' issuer: http://localhost:65432/auth/realms/opentdf clients: - - "opentdf" - - "opentdf-sdk" + - 'opentdf' + - 'opentdf-sdk' policy: ## Default policy for all requests default: #"role:readonly" From f22be7d99095c45f1b6942ac3a1070081330deb8 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 05:23:59 -0400 Subject: [PATCH 12/20] Update build.yaml --- .github/workflows/build.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 682e69ed..453dc421 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -214,7 +214,12 @@ jobs: ./wait-and-test.sh platform deliver-ghp: - needs: [lib, web-app, scripts, backend-roundtrip] + needs: + - lib + - web-app + - scripts + - backend-roundtrip + - platform-roundtrip runs-on: ubuntu-latest timeout-minutes: 5 # To publish from a release or feature branch, remove the ref == condition below From 51b2a18e779a9774f6af46a6992ae51f52b5b357 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 05:24:58 -0400 Subject: [PATCH 13/20] DO NOT SUBMIT --- .github/workflows/build.yaml | 182 +++++++++++++++++------------------ 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 453dc421..834f1cf1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,32 +54,32 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - remote-store: - needs: - - lib - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./remote-store - timeout-minutes: 5 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '18' - cache: 'npm' - cache-dependency-path: './remote-store/package-lock.json' - - uses: actions/download-artifact@v3 - with: - name: opentdf-client-lib - path: lib/ - - run: npm uninstall @opentdf/client && npm ci && npm i ../lib/opentdf-client-*.tgz - - run: npm install - - run: npm test - - run: npm audit - - run: npm run license-check - - run: npm run lint - - run: npm pack + # remote-store: + # needs: + # - lib + # runs-on: ubuntu-latest + # defaults: + # run: + # working-directory: ./remote-store + # timeout-minutes: 5 + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/setup-node@v3 + # with: + # node-version: '18' + # cache: 'npm' + # cache-dependency-path: './remote-store/package-lock.json' + # - uses: actions/download-artifact@v3 + # with: + # name: opentdf-client-lib + # path: lib/ + # - run: npm uninstall @opentdf/client && npm ci && npm i ../lib/opentdf-client-*.tgz + # - run: npm install + # - run: npm test + # - run: npm audit + # - run: npm run license-check + # - run: npm run lint + # - run: npm pack web-app: needs: @@ -108,70 +108,70 @@ jobs: - run: npm run lint - run: npm pack - scripts: - runs-on: ubuntu-latest - defaults: - run: - working-directory: .github/workflows - timeout-minutes: 5 - steps: - - uses: actions/checkout@v3 - - uses: mig4/setup-bats@v1 - with: - bats-version: 1.2.1 - - name: πŸ¦‡πŸ¦‡πŸ¦‡πŸ¦‡πŸ¦‡πŸ¦‡ - run: docker run --rm -v "$PWD:/mnt" --workdir "/mnt" bats/bats:1.5.0 *.bats - - name: πŸ¦ͺ βœ” 🧼🧼🧼 - run: >- - docker run --rm -v "$PWD:/mnt" --workdir "/mnt" "koalaman/shellcheck:v0.8.0" - --color=always *.sh + # scripts: + # runs-on: ubuntu-latest + # defaults: + # run: + # working-directory: .github/workflows + # timeout-minutes: 5 + # steps: + # - uses: actions/checkout@v3 + # - uses: mig4/setup-bats@v1 + # with: + # bats-version: 1.2.1 + # - name: πŸ¦‡πŸ¦‡πŸ¦‡πŸ¦‡πŸ¦‡πŸ¦‡ + # run: docker run --rm -v "$PWD:/mnt" --workdir "/mnt" bats/bats:1.5.0 *.bats + # - name: πŸ¦ͺ βœ” 🧼🧼🧼 + # run: >- + # docker run --rm -v "$PWD:/mnt" --workdir "/mnt" "koalaman/shellcheck:v0.8.0" + # --color=always *.sh - backend-roundtrip: - needs: - - web-app - runs-on: ubuntu-latest - defaults: - run: - working-directory: .github/workflows/roundtrip - timeout-minutes: 45 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '18' - cache: 'npm' - cache-dependency-path: './web-app/package-lock.json' - - uses: actions/download-artifact@v3 - with: - name: opentdf-client-lib - path: lib/ - - uses: yokawasa/action-setup-kube-tools@v0.11.1 - with: - setup-tools: | - kubectl - helm - tilt - # This should be in sync with the minikube-deployed kube version below - kubectl: '1.30.0' - helm: '3.14.4' - tilt: '0.33.13' - - run: | - kubectl version --client - kustomize version - tilt version - - name: start minikube - id: minikube - uses: medyagh/setup-minikube@master - with: - minikube-version: 1.33.0 - # This should be in sync with the setup-tools version above - kubernetes-version: 1.30.0 - - name: Run tilt ci - env: - TEST_SCRIPT: ./wait-and-test.sh backend - OPENTDF_INGRESS_HOST_PORT: '5432' - PLAYWRIGHT_TESTS_TO_RUN: roundtrip - run: tilt ci + # backend-roundtrip: + # needs: + # - web-app + # runs-on: ubuntu-latest + # defaults: + # run: + # working-directory: .github/workflows/roundtrip + # timeout-minutes: 45 + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/setup-node@v3 + # with: + # node-version: '18' + # cache: 'npm' + # cache-dependency-path: './web-app/package-lock.json' + # - uses: actions/download-artifact@v3 + # with: + # name: opentdf-client-lib + # path: lib/ + # - uses: yokawasa/action-setup-kube-tools@v0.11.1 + # with: + # setup-tools: | + # kubectl + # helm + # tilt + # # This should be in sync with the minikube-deployed kube version below + # kubectl: '1.30.0' + # helm: '3.14.4' + # tilt: '0.33.13' + # - run: | + # kubectl version --client + # kustomize version + # tilt version + # - name: start minikube + # id: minikube + # uses: medyagh/setup-minikube@master + # with: + # minikube-version: 1.33.0 + # # This should be in sync with the setup-tools version above + # kubernetes-version: 1.30.0 + # - name: Run tilt ci + # env: + # TEST_SCRIPT: ./wait-and-test.sh backend + # OPENTDF_INGRESS_HOST_PORT: '5432' + # PLAYWRIGHT_TESTS_TO_RUN: roundtrip + # run: tilt ci platform-roundtrip: needs: @@ -217,8 +217,8 @@ jobs: needs: - lib - web-app - - scripts - - backend-roundtrip + # - scripts + # - backend-roundtrip - platform-roundtrip runs-on: ubuntu-latest timeout-minutes: 5 From 0287edcd79b3f886dcbab22e86fe7ae3f87d99ed Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 05:52:06 -0400 Subject: [PATCH 14/20] catch 22 with proxy and finding resource --- .github/workflows/build.yaml | 11 ---- .github/workflows/roundtrip/wait-and-test.sh | 62 +++++++++++++++++--- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 834f1cf1..65a5f7e1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -196,17 +196,6 @@ jobs: with: go-version: '1.22' - run: docker compose up -d --wait --wait-timeout 240 - - run: go run github.com/opentdf/platform/service@latest provision keycloak - - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 - name: start server in background - with: - run: | - go run github.com/opentdf/platform/service@latest start - wait-on: | - tcp:localhost:8080 - log-output-if: true - wait-for: 90s - working-directory: .github/workflows/roundtrip - env: OPENTDF_SERVICES_AUTHORIZATION_URL: 'http://localhost:65432/' PLAYWRIGHT_TESTS_TO_RUN: roundtrip diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 48f60dcd..18bcad61 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -21,15 +21,15 @@ export VITE_TDF_CFG # VITE_PROXY='{"/api":"http://localhost:5432","/auth":"http://localhost:5432"}' VITE_TDF_CFG='{"oidc":{"host":"http://localhost:65432/auth/realms/tdf","clientId":"browsertest"},"kas":"http://localhost:65432/api/kas","reader":"https://secure.virtru.com/start?htmlProtocol=1"}' npm run dev -_wait-for() { +_wait_for() { echo "[INFO] In retry loop for quickstarted opentdf backend..." limit=5 for i in $(seq 1 $limit); do - if curl --show-error --fail --insecure http://localhost:65432; then + if curl --show-error --fail --insecure "$1"; then return 0 fi if [[ $i == "$limit" ]]; then - echo "[WARN] Breaking _wait-for loop as we are at limit" + echo "[WARN] Breaking _wait_for loop as we are at limit" break fi sleep_for=$((10 + i * i * 2)) @@ -40,7 +40,7 @@ _wait-for() { exit 1 } -_init_server() { +_init_webapp() { output=$(mktemp) if ! cd "${WEB_APP_DIR}"; then echo "[ERROR] unable to cd ${WEB_APP_DIR}" @@ -72,7 +72,7 @@ _init_server() { exit 1 fi if [[ $i == "$limit" ]]; then - echo "[WARN] Breaking _init_server loop after ${limit} iterations" + echo "[WARN] Breaking _init_webapp loop after ${limit} iterations" cat "${output}" break fi @@ -82,15 +82,61 @@ _init_server() { done } -if ! _init_server; then + +_init_platform() { + output=$(mktemp) + if ! cd "${APP_DIR}"; then + echo "[ERROR] unable to cd ${APP_DIR}" + exit 2 + fi + if ! go run github.com/opentdf/platform/service@latest provision keycloak; then + echo "[ERROR] unable to provision keycloak" + return 1 + fi + if ! ./init-temp-keys.sh; then + echo "[ERROR] unable to initialize keys" + return 1 + fi + go run github.com/opentdf/platform/service@latest start &>"$output" & + server_pid=$! + echo "Platform pid: $server_pid" + echo "Output: $output" + echo "Wait:" + limit=5 + for i in $(seq 1 $limit); do + if grep -q -i 'starting http server' "$output"; then + return 0 + fi + if ! ps $server_pid >/dev/null; then + echo "The server died" >&2 + cat "${output}" + exit 1 + fi + if [[ $i == "$limit" ]]; then + echo "[WARN] Breaking _init_platform loop after ${limit} iterations" + cat "${output}" + break + fi + sleep_for=$((5 + i * i * 2)) + echo "[INFO] retrying in ${sleep_for} seconds... ( ${i} / $limit ) ..." + sleep ${sleep_for} + done +} + +if ! _init_webapp; then echo "[ERROR] Couldn't run web app server" exit 2 fi -if ! _wait-for; then - exit 1 +if [ $1 = platform ]; then + if ! _init_platform; then + echo "[ERROR] Couldn't run platform" + exit 2 + fi fi + + if ! cd "${WEB_APP_DIR}"; then echo "[ERROR] Couldn't cd to web-app dir, [${WEB_APP_DIR}]" exit 2 From feabd447f67bba14b7285658dd13e95328d9aef8 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 06:24:48 -0400 Subject: [PATCH 15/20] configure browsertest client --- .github/workflows/roundtrip/config-demo-idp.sh | 4 +++- .github/workflows/roundtrip/wait-and-test.sh | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh index 911ba615..d4b25c21 100755 --- a/.github/workflows/roundtrip/config-demo-idp.sh +++ b/.github/workflows/roundtrip/config-demo-idp.sh @@ -1,11 +1,13 @@ +set -x + : "${KC_VERSION:=24.0.3}" -: "${KC_BROWSERTEST_CLIENT_SECRET:=$(uuidgen)}" if ! which kcadm.sh; then KCADM_URL=https://github.com/keycloak/keycloak/releases/download/${KC_VERSION}/keycloak-${KC_VERSION}.zip echo "DOWNLOADING ${KCADM_URL}" curl -o kc.zip "${KCADM_URL}" unzip kc.zip -d keycloak-${KC_VERSION} + ls export PATH=$PATH:$(pwd)/keycloak-${KC_VERSION}/bin fi diff --git a/.github/workflows/roundtrip/wait-and-test.sh b/.github/workflows/roundtrip/wait-and-test.sh index 18bcad61..579936d1 100755 --- a/.github/workflows/roundtrip/wait-and-test.sh +++ b/.github/workflows/roundtrip/wait-and-test.sh @@ -93,6 +93,10 @@ _init_platform() { echo "[ERROR] unable to provision keycloak" return 1 fi + if ! ./config-demo-idp.sh; then + echo "[ERROR] unable to provision keycloak" + return 1 + fi if ! ./init-temp-keys.sh; then echo "[ERROR] unable to initialize keys" return 1 From 87566fbc7eb42568ed7fa5e48b27a8c90b2962d3 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 08:39:22 -0400 Subject: [PATCH 16/20] Update config-demo-idp.sh --- .github/workflows/roundtrip/config-demo-idp.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh index d4b25c21..35d764a3 100755 --- a/.github/workflows/roundtrip/config-demo-idp.sh +++ b/.github/workflows/roundtrip/config-demo-idp.sh @@ -5,9 +5,12 @@ set -x if ! which kcadm.sh; then KCADM_URL=https://github.com/keycloak/keycloak/releases/download/${KC_VERSION}/keycloak-${KC_VERSION}.zip echo "DOWNLOADING ${KCADM_URL}" - curl -o kc.zip "${KCADM_URL}" - unzip kc.zip -d keycloak-${KC_VERSION} - ls + if ! curl --output kc.zip --fail --location "${KCADM_URL}"; then + echo "DOWNLOADING ${KCADM_URL}" + fi + ls -l + unzip ./kc.zip -d keycloak-${KC_VERSION} + ls -l export PATH=$PATH:$(pwd)/keycloak-${KC_VERSION}/bin fi From 01c311cb6cf83a0273b156d17c39102dc7d22b66 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 08:56:21 -0400 Subject: [PATCH 17/20] shellcheck & shfmt --- .github/workflows/roundtrip/config-demo-idp.sh | 18 +++++++++++++++--- .github/workflows/roundtrip/wait-and-test.sh | 3 --- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh index 35d764a3..0c0ccf8a 100755 --- a/.github/workflows/roundtrip/config-demo-idp.sh +++ b/.github/workflows/roundtrip/config-demo-idp.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set -x : "${KC_VERSION:=24.0.3}" @@ -6,12 +8,22 @@ if ! which kcadm.sh; then KCADM_URL=https://github.com/keycloak/keycloak/releases/download/${KC_VERSION}/keycloak-${KC_VERSION}.zip echo "DOWNLOADING ${KCADM_URL}" if ! curl --output kc.zip --fail --location "${KCADM_URL}"; then - echo "DOWNLOADING ${KCADM_URL}" + echo "[ERROR] Failed to download ${KCADM_URL}" + exit 3 fi ls -l - unzip ./kc.zip -d keycloak-${KC_VERSION} + if ! unzip ./kc.zip; then + echo "[ERROR] Failed to unzip file from ${KCADM_URL}" + exit 3 + fi ls -l - export PATH=$PATH:$(pwd)/keycloak-${KC_VERSION}/bin + ls -l "$(pwd)/keycloak-${KC_VERSION}/bin" + PATH=$PATH:"$(pwd)/keycloak-${KC_VERSION}/bin" + export PATH + if ! which kcadm.sh; then + echo "[ERROR] Failed to find kcadm.sh" + exit 3 + fi fi kcadm.sh config credentials --server http://localhost:65432/auth --realm master --user admin < Date: Mon, 6 May 2024 09:22:48 -0400 Subject: [PATCH 18/20] hmm --- .github/workflows/roundtrip/config-demo-idp.sh | 5 ++--- .gitignore | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh index 0c0ccf8a..9127eb51 100755 --- a/.github/workflows/roundtrip/config-demo-idp.sh +++ b/.github/workflows/roundtrip/config-demo-idp.sh @@ -26,9 +26,8 @@ if ! which kcadm.sh; then fi fi -kcadm.sh config credentials --server http://localhost:65432/auth --realm master --user admin < Date: Mon, 6 May 2024 09:58:07 -0400 Subject: [PATCH 19/20] Update config-demo-idp.sh --- .github/workflows/roundtrip/config-demo-idp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh index 9127eb51..532217b3 100755 --- a/.github/workflows/roundtrip/config-demo-idp.sh +++ b/.github/workflows/roundtrip/config-demo-idp.sh @@ -42,5 +42,5 @@ kcadm.sh create clients -r opentdf \ -s 'protocolMappers=[{"name":"aud","protocol":"openid-connect","protocolMapper":"oidc-audience-mapper","consentRequired":false,"config":{"access.token.claim":"true","included.custom.audience":"http://localhost:65432"}}]' \ -s 'attributes={"dpop.bound.access.tokens":"true"}' -kcadm.sh create users -r opentdf -s username=user1 -s enabled=true +kcadm.sh create users -r opentdf -s username=user1 -s enabled=true -s firstName=Alice -s lastName=User kcadm.sh set-password -r opentdf --username user1 --new-password testuser123 From 9757f3061376389348b8edad83845b5c86d1ffdb Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Mon, 6 May 2024 12:48:50 -0400 Subject: [PATCH 20/20] Update opentdf.yaml --- .github/workflows/roundtrip/opentdf.yaml | 93 ++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/.github/workflows/roundtrip/opentdf.yaml b/.github/workflows/roundtrip/opentdf.yaml index d82739f0..39e53f9f 100644 --- a/.github/workflows/roundtrip/opentdf.yaml +++ b/.github/workflows/roundtrip/opentdf.yaml @@ -36,16 +36,93 @@ server: ## Maps the external role to the opentdf role ## Note: left side is used in the policy, right side is the external role map: - # readonly: opentdf-readonly - # admin: opentdf-admin - # org-admin: opentdf-org-admin + readonly: opentdf-readonly + admin: opentdf-admin + user: default-roles-opentdf + org-admin: opentdf-org-admin ## Custom policy (see examples https://github.com/casbin/casbin/tree/master/examples) - csv: #| - # p, role:org-admin, policy:attributes, *, *, allow - # p, role:org-admin, policy:subject-mappings, *, *, allow - # p, role:org-admin, policy:resource-mappings, *, *, allow - # p, role:org-admin, policy:kas-registry, *, *, allow + csv: | + ## Roles (prefixed with role:) + # org-admin - organization admin + # admin - admin + # readonly - readonly + # user - rewrap + # unknown - unknown role or no role + ## Actions + # read - read the resource + # write - write to the resource + # delete - delete the resource + # unsafe - unsafe actions + + # Role: user + p, role:user, kas.AccessService/PublicKey, read, allow + p, role:user, kas.AccessService/Rewrap, write, allow + p, role:user, /, read, allow + p, role:user, /kas/kas_public_key, read, allow + p, role:user, /kas/v2/kas_public_key, read, allow + p, role:user, /kas/v2/rewrap, write, allow + + # Role: Org-Admin + ## gRPC routes + p, role:org-admin, policy.*, *, allow + p, role:org-admin, kasregistry.*, *, allow + p, role:org-admin, kas.AccessService/LegacyPublicKey, *, allow + p, role:org-admin, kas.AccessService/PublicKey, *, allow + p, role:org-admin, kas.AccessService/Rewrap, *, allow + ## HTTP routes + p, role:org-admin, /health, *, allow + p, role:org-admin, /attributes*, *, allow + p, role:org-admin, /namespaces*, *, allow + p, role:org-admin, /subject-mappings*, *, allow + p, role:org-admin, /resource-mappings*, *, allow + p, role:org-admin, /key-access-servers*, *, allow + p, role:org-admin, /kas.AccessService/LegacyPublicKey, *, allow + # add unsafe actions to the org-admin role + + # Role: Admin + ## gRPC routes + p, role:admin, policy.*, *, allow + p, role:admin, kasregistry.*, *, allow + p, role:admin, kas.AccessService/Info, *, allow + p, role:admin, kas.AccessService/Rewrap, *, allow + p, role:admin, kas.AccessService/LegacyPublicKey, *, allow + p, role:admin, kas.AccessService/PublicKey, *, allow + ## HTTP routes + p, role:admin, /health, *, allow + p, role:admin, /attributes*, *, allow + p, role:admin, /namespaces*, *, allow + p, role:admin, /subject-mappings*, *, allow + p, role:admin, /resource-mappings*, *, allow + p, role:admin, /key-access-servers*, *, allow + p, role:admin, /kas.AccessService/LegacyPublicKey, *, allow + + ## Role: Readonly + ## gRPC routes + p, role:readonly, policy.*, read, allow + p, role:readonly, kasregistry.*, read, allow + p, role:readonly, kas.AccessService/Info, *, allow + p, role:readonly, kas.AccessService/Rewrap, *, allow + p, role:readonly, kas.AccessService/LegacyPublicKey, *, allow + p, role:readonly, kas.AccessService/PublicKey, *, allow + ## HTTP routes + p, role:readonly, /health, read, allow + p, role:readonly, /attributes*, read, allow + p, role:readonly, /namespaces*, read, allow + p, role:readonly, /subject-mappings*, read, allow + p, role:readonly, /resource-mappings*, read, allow + p, role:readonly, /key-access-servers*, read, allow + p, role:readonly, /kas.AccessService/LegacyPublicKey, read, allow + + # Public routes + ## gRPC routes + p, role:unknown, kas.AccessService/LegacyPublicKey, other, allow + p, role:unknown, kas.AccessService/PublicKey, other, allow + ## HTTP routes + p, role:unknown, /health, read, allow + p, role:unknown, /kas/v2/kas_public_key, read, allow + p, role:unknown, /kas/kas_public_key, read, allow + ## Custom model (see https://casbin.org/docs/syntax-for-models/) model: #| # [request_definition]