From 2331e9ccea733322d7e9bd1e1905d97782c0aef3 Mon Sep 17 00:00:00 2001 From: Cas Lubbers Date: Thu, 27 Nov 2025 13:08:19 +0100 Subject: [PATCH 1/5] feat: add filestore as in-memory object --- package-lock.json | 62 +- package.json | 4 +- src/api.authz.test.ts | 47 +- src/api/v1/apps/{teamId}/{appId}.ts | 4 +- src/app.ts | 12 +- src/error.ts | 8 +- src/fileStore/file-map.ts | 260 ++++ src/fileStore/file-store.ts | 209 +++ src/middleware/jwt.test.ts | 7 +- src/middleware/jwt.ts | 4 +- src/middleware/session.ts | 2 +- src/openapi/api.yaml | 20 - src/otomi-models.ts | 137 +- src/otomi-stack.test.ts | 790 ++++------- src/otomi-stack.ts | 1729 ++++++++++++------------ src/services/RepoService.test.ts | 203 --- src/services/RepoService.ts | 287 ---- src/services/TeamConfigService.test.ts | 606 --------- src/services/TeamConfigService.ts | 567 -------- src/utils/manifests.ts | 12 +- 20 files changed, 1806 insertions(+), 3164 deletions(-) create mode 100644 src/fileStore/file-map.ts create mode 100644 src/fileStore/file-store.ts delete mode 100644 src/services/RepoService.test.ts delete mode 100644 src/services/RepoService.ts delete mode 100644 src/services/TeamConfigService.test.ts delete mode 100644 src/services/TeamConfigService.ts diff --git a/package-lock.json b/package-lock.json index 90327ad03..e5883e3b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,8 @@ "swagger-ui-express": "5.0.1", "ts-custom-error": "3.3.1", "uuid": "13.0.0", - "yaml": "2.8.1" + "yaml": "2.8.1", + "zod": "^4.1.12" }, "devDependencies": { "@babel/core": "7.28.5", @@ -50,6 +51,7 @@ "@redocly/openapi-cli": "1.0.0-beta.95", "@semantic-release/changelog": "6.0.3", "@semantic-release/git": "10.0.1", + "@types/debug": "^4.1.12", "@types/expect": "24.3.2", "@types/express": "^5.0.0", "@types/fs-extra": "11.0.4", @@ -193,6 +195,7 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -2242,6 +2245,7 @@ "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -2844,7 +2848,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-css/-/dict-css-4.0.18.tgz", "integrity": "sha512-EF77RqROHL+4LhMGW5NTeKqfUd/e4OOv6EDFQ/UQQiFyWuqkEKyEz0NDILxOFxWUEVdjT2GQ2cC7t12B6pESwg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-dart": { "version": "2.3.1", @@ -2984,14 +2989,16 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-html/-/dict-html-4.0.12.tgz", "integrity": "sha512-JFffQ1dDVEyJq6tCDWv0r/RqkdSnV43P2F/3jJ9rwLgdsOIXwQbXrz6QDlvQLVvNSnORH9KjDtenFTGDyzfCaA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-html-symbol-entities": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.4.tgz", "integrity": "sha512-afea+0rGPDeOV9gdO06UW183Qg6wRhWVkgCFwiO3bDupAoyXRuvupbb5nUyqSTsLXIKL8u8uXQlJ9pkz07oVXw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-java": { "version": "5.0.12", @@ -3189,7 +3196,8 @@ "resolved": "https://registry.npmjs.org/@cspell/dict-typescript/-/dict-typescript-3.2.3.tgz", "integrity": "sha512-zXh1wYsNljQZfWWdSPYwQhpwiuW0KPW1dSd8idjMRvSD0aSvWWHoWlrMsmZeRl4qM4QCEAjua8+cjflm41cQBg==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@cspell/dict-vue": { "version": "3.0.5", @@ -4369,6 +4377,7 @@ "integrity": "sha512-b63wmnKPaK+6ZZfpYhz9K61oybvbI1aMcIs80++JI1O1rR1vaxHUCNqo3ITu6NU0d4V34yZFoHMn/uoKr/Rwfw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.2.0", "@jest/expect": "30.2.0", @@ -4904,6 +4913,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -6227,6 +6237,16 @@ "@types/node": "*" } }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz", @@ -6335,7 +6355,8 @@ "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/json5": { "version": "0.0.29", @@ -6422,6 +6443,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.0.tgz", "integrity": "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -6581,6 +6603,7 @@ "integrity": "sha512-6m1I5RmHBGTnUGS113G04DMu3CpSdxCAU/UvtjNWL4Nuf3MW9tQhiJqRlHzChIkhy6kZSAQmc+I1bcGjE3yNKg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.3", "@typescript-eslint/types": "8.46.3", @@ -7137,6 +7160,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8122,6 +8146,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -10817,6 +10842,7 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10877,6 +10903,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -11003,6 +11030,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -11566,6 +11594,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", "integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", @@ -11633,6 +11662,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -14267,6 +14297,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -15149,6 +15180,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -16527,6 +16559,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -19758,6 +19791,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -20907,6 +20941,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -22009,6 +22044,7 @@ "integrity": "sha512-6qGjWccl5yoyugHt3jTgztJ9Y0JVzyH8/Voc/D8PlLat9pwxQYXz7W1Dpnq5h0/G5GCYGUaDSlYcyk3AMh5A6g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -24222,6 +24258,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -24444,6 +24481,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -24708,6 +24746,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -24887,6 +24926,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "napi-postinstall": "^0.3.0" }, @@ -25316,6 +25356,7 @@ "version": "7.5.7", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz", "integrity": "sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A==", + "peer": true, "engines": { "node": ">=8.3.0" }, @@ -25497,6 +25538,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", + "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 9ef0d8226..1b9534091 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "swagger-ui-express": "5.0.1", "ts-custom-error": "3.3.1", "uuid": "13.0.0", - "yaml": "2.8.1" + "yaml": "2.8.1", + "zod": "^4.1.12" }, "description": "The brain of the Otomi Container Platform. Handling console input and talking to core, it knows what to do.", "devDependencies": { @@ -50,6 +51,7 @@ "@redocly/openapi-cli": "1.0.0-beta.95", "@semantic-release/changelog": "6.0.3", "@semantic-release/git": "10.0.1", + "@types/debug": "^4.1.12", "@types/expect": "24.3.2", "@types/express": "^5.0.0", "@types/fs-extra": "11.0.4", diff --git a/src/api.authz.test.ts b/src/api.authz.test.ts index 0e264c8f2..6fab914a1 100644 --- a/src/api.authz.test.ts +++ b/src/api.authz.test.ts @@ -7,10 +7,10 @@ import request from 'supertest' import { HttpError } from './error' import { Git } from './git' import { getSessionStack } from './middleware' -import { App, CodeRepo, Repo, SealedSecret } from './otomi-models' -import { RepoService } from './services/RepoService' +import { App, CodeRepo, SealedSecret } from './otomi-models' import * as getValuesSchemaModule from './utils' import TestAgent from 'supertest/lib/agent' +import { FileStore } from './fileStore/file-store' const platformAdminToken = getToken(['platform-admin']) const teamAdminToken = getToken(['team-admin', 'team-team1']) @@ -40,22 +40,33 @@ describe('API authz tests', () => { beforeAll(async () => { const _otomiStack = await getSessionStack() _otomiStack.git = mockDeep() - _otomiStack.transformApps = jest.fn().mockReturnValue([]) - _otomiStack.repoService = new RepoService({} as Repo) + _otomiStack.fileStore = new FileStore() otomiStack = _otomiStack as jest.Mocked otomiStack.saveTeam = jest.fn().mockResolvedValue(undefined) - otomiStack.doRepoDeployment = jest.fn().mockImplementation(() => Promise.resolve()) - otomiStack.doTeamDeployment = jest.fn().mockImplementation(() => Promise.resolve()) - otomiStack.isLoaded = true - await otomiStack.createTeam({ - name: 'team1', - resourceQuota: [], - }) - await otomiStack.createTeam({ - name: 'team2', - resourceQuota: [], + otomiStack.doDeleteDeployment = jest.fn().mockImplementation(() => Promise.resolve()) + otomiStack.doDeployment = jest.fn().mockImplementation(() => Promise.resolve()) + otomiStack.fileStore.set('env/teams/team1/settings.yaml', { + kind: 'AplTeamSettingSet', + spec: {}, + metadata: { + name: 'team1', + labels: { + 'apl.io/teamId': 'team1', + }, + }, + }) + otomiStack.fileStore.set('env/teams/team2/settings.yaml', { + kind: 'AplTeamSettingSet', + spec: {}, + metadata: { + name: 'team2', + labels: { + 'apl.io/teamId': 'team2', + }, + }, }) + otomiStack.isLoaded = true app = await initApp(otomiStack) agent = request.agent(app) agent.set('Accept', 'application/json') @@ -92,6 +103,7 @@ describe('API authz tests', () => { }) test('platform admin can update team self-service-flags', async () => { + jest.spyOn(otomiStack, 'editTeam').mockReturnValue({} as any) await agent .put('/v1/teams/team1') .send({ @@ -746,6 +758,7 @@ describe('API authz tests', () => { }) test('platform admin can update policies', async () => { + jest.spyOn(otomiStack, 'editAplPolicy').mockReturnValue({} as any) await agent .put('/v1/teams/team1/policies/disallow-selinux') .send(data) @@ -1031,7 +1044,7 @@ describe('API authz tests', () => { }) test('platform admin can get specific agent', async () => { - jest.spyOn(otomiStack, 'getAplAgent').mockResolvedValue({} as any) + jest.spyOn(otomiStack, 'getAplAgent').mockReturnValue({} as any) await agent .get('/alpha/teams/team1/agents/test-agent') .set('Authorization', `Bearer ${platformAdminToken}`) @@ -1040,7 +1053,7 @@ describe('API authz tests', () => { }) test('team admin can get specific agent', async () => { - jest.spyOn(otomiStack, 'getAplAgent').mockResolvedValue({} as any) + jest.spyOn(otomiStack, 'getAplAgent').mockReturnValue({} as any) await agent .get('/alpha/teams/team1/agents/test-agent') .set('Authorization', `Bearer ${teamAdminToken}`) @@ -1049,7 +1062,7 @@ describe('API authz tests', () => { }) test('team member can get specific agent', async () => { - jest.spyOn(otomiStack, 'getAplAgent').mockResolvedValue({} as any) + jest.spyOn(otomiStack, 'getAplAgent').mockReturnValue({} as any) await agent .get('/alpha/teams/team1/agents/test-agent') .set('Authorization', `Bearer ${teamMemberToken}`) diff --git a/src/api/v1/apps/{teamId}/{appId}.ts b/src/api/v1/apps/{teamId}/{appId}.ts index c8c52b582..b88686d07 100644 --- a/src/api/v1/apps/{teamId}/{appId}.ts +++ b/src/api/v1/apps/{teamId}/{appId}.ts @@ -14,7 +14,7 @@ export const getTeamApp = (req: OpenApiRequestExt, res: Response): void => { * PUT /v1/apps/{teamId}/{appId} * Edit a team app */ -export const editApp = async (req: OpenApiRequestExt, res: Response): Promise => { +export const editTeamApp = async (req: OpenApiRequestExt, res: Response): Promise => { const { teamId, appId } = req.params - res.json(await req.otomi.editApp(teamId, appId, req.body as App)) + res.json(await req.otomi.editTeamApp(teamId, appId, req.body as App)) } diff --git a/src/app.ts b/src/app.ts index 92e456d38..ae0f4a0a0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -86,10 +86,10 @@ const resourceStatus = async (errorSet) => { const { cluster } = otomiStack.getSettings(['cluster']) const domainSuffix = cluster?.domainSuffix const resources: Record = { - workloads: otomiStack.repoService.getAllWorkloads(), - builds: otomiStack.repoService.getAllBuilds(), - services: otomiStack.repoService.getAllServices(), - secrets: otomiStack.repoService.getAllSealedSecrets(), + workloads: otomiStack.getAllAplWorkloads(), + builds: otomiStack.getAllAplBuilds(), + services: otomiStack.getAllAplServices(), + secrets: otomiStack.getAllAplSealedSecrets(), } const statusFunctions = { workloads: getWorkloadStatus, @@ -132,6 +132,10 @@ export const loadSpec = async (): Promise => { export const getSpec = (): OtomiSpec => { return otomiSpec } +export function getSecretPaths(): string[] { + const { secretPaths } = getSpec() + return secretPaths +} export const getAppSchema = (appId: string): Schema => { let id: string = appId if (appId.startsWith('ingress-nginx')) id = 'ingress-nginx-platform' diff --git a/src/error.ts b/src/error.ts index f7e4334b0..253271221 100644 --- a/src/error.ts +++ b/src/error.ts @@ -14,19 +14,19 @@ export class OtomiError extends CustomError { } export class ForbiddenError extends OtomiError { public constructor(err?: string) { - super('Forbidden', err) + super(err || 'Forbidden', err) this.code = 403 } } export class NotExistError extends OtomiError { public constructor(err?: string) { - super('Not Found', err) + super(err || 'Not Found', err) this.code = 404 } } export class AlreadyExists extends OtomiError { public constructor(err?: string) { - super('Conflict', err) + super(err || 'Conflict', err) this.code = 409 } } @@ -45,7 +45,7 @@ export class PublicUrlExists extends OtomiError { } export class ValidationError extends OtomiError { public constructor(err?: string) { - super('Invalid values detected', err) + super(err || 'Invalid values detected', err) this.code = 422 } } diff --git a/src/fileStore/file-map.ts b/src/fileStore/file-map.ts new file mode 100644 index 000000000..f9a880536 --- /dev/null +++ b/src/fileStore/file-map.ts @@ -0,0 +1,260 @@ +import { AplKind } from '../otomi-models' + +export interface FileMap { + envDir: string + kind: AplKind + pathGlob: string + pathTemplate: string // e.g., 'env/teams/{teamId}/workloads/{name}.yaml' + name: string +} + +export function getFileMaps(envDir: string): Map { + const maps = new Map() + + maps.set('AplApp', { + kind: 'AplApp', + envDir, + pathGlob: `${envDir}/env/apps/*.{yaml,yaml.dec}`, + pathTemplate: 'env/apps/{name}.yaml', + name: 'apps', + }) + + maps.set('AplAlertSet', { + kind: 'AplAlertSet', + envDir, + pathGlob: `${envDir}/env/settings/*alerts.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/alerts.yaml', + name: 'alerts', + }) + + maps.set('AplCluster', { + kind: 'AplCluster', + envDir, + pathGlob: `${envDir}/env/settings/cluster.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/cluster.yaml', + name: 'cluster', + }) + + maps.set('AplDatabase', { + kind: 'AplDatabase', + envDir, + pathGlob: `${envDir}/env/databases/*.{yaml,yaml.dec}`, + pathTemplate: 'env/databases/{name}.yaml', + name: 'databases', + }) + + maps.set('AplDns', { + kind: 'AplDns', + envDir, + pathGlob: `${envDir}/env/settings/*dns.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/dns.yaml', + name: 'dns', + }) + + maps.set('AplIngress', { + kind: 'AplIngress', + envDir, + pathGlob: `${envDir}/env/settings/ingress.yaml`, + pathTemplate: 'env/settings/ingress.yaml', + name: 'ingress', + }) + + maps.set('AplKms', { + kind: 'AplKms', + envDir, + pathGlob: `${envDir}/env/settings/*kms.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/kms.yaml', + name: 'kms', + }) + + maps.set('AplObjectStorage', { + kind: 'AplObjectStorage', + envDir, + pathGlob: `${envDir}/env/settings/*obj.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/obj.yaml', + name: 'obj', + }) + + maps.set('AplIdentityProvider', { + kind: 'AplIdentityProvider', + envDir, + pathGlob: `${envDir}/env/settings/*oidc.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/oidc.yaml', + name: 'oidc', + }) + + maps.set('AplCapabilitySet', { + kind: 'AplCapabilitySet', + envDir, + pathGlob: `${envDir}/env/settings/*otomi.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/otomi.yaml', + name: 'otomi', + }) + + maps.set('AplBackupCollection', { + kind: 'AplBackupCollection', + envDir, + pathGlob: `${envDir}/env/settings/*platformBackups.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/platformBackups.yaml', + name: 'platformBackups', + }) + + maps.set('AplSmtp', { + kind: 'AplSmtp', + envDir, + pathGlob: `${envDir}/env/settings/*smtp.{yaml,yaml.dec}`, + pathTemplate: 'env/settings/smtp.yaml', + name: 'smtp', + }) + + maps.set('AplUser', { + kind: 'AplUser', + envDir, + pathGlob: `${envDir}/env/users/*.{yaml,yaml.dec}`, + pathTemplate: 'env/users/{name}.yaml', + name: 'users', + }) + + maps.set('AplVersion', { + kind: 'AplVersion', + envDir, + pathGlob: `${envDir}/env/settings/versions.yaml`, + pathTemplate: 'env/settings/versions.yaml', + name: 'versions', + }) + + maps.set('AplTeamCodeRepo', { + kind: 'AplTeamCodeRepo', + envDir, + pathGlob: `${envDir}/env/teams/*/codeRepos/*.yaml`, + pathTemplate: 'env/teams/{teamId}/codeRepos/{name}.yaml', + name: 'codeRepos', + }) + + maps.set('AplTeamBuild', { + kind: 'AplTeamBuild', + envDir, + pathGlob: `${envDir}/env/teams/*/builds/*.yaml`, + pathTemplate: 'env/teams/{teamId}/builds/{name}.yaml', + name: 'builds', + }) + + maps.set('AplTeamWorkload', { + kind: 'AplTeamWorkload', + envDir, + pathGlob: `${envDir}/env/teams/*/workloads/*.yaml`, + pathTemplate: 'env/teams/{teamId}/workloads/{name}.yaml', + name: 'workloads', + }) + + maps.set('AplTeamWorkloadValues', { + kind: 'AplTeamWorkloadValues', + envDir, + pathGlob: `${envDir}/env/teams/*/workloadValues/*.yaml`, + pathTemplate: 'env/teams/{teamId}/workloadValues/{name}.yaml', + name: 'workloadValues', + }) + + maps.set('AplTeamService', { + kind: 'AplTeamService', + envDir, + pathGlob: `${envDir}/env/teams/*/services/*.yaml`, + pathTemplate: 'env/teams/{teamId}/services/{name}.yaml', + name: 'services', + }) + + maps.set('AplTeamSecret', { + kind: 'AplTeamSecret', + envDir, + pathGlob: `${envDir}/env/teams/*/sealedsecrets/*.yaml`, + pathTemplate: 'env/teams/{teamId}/sealedsecrets/{name}.yaml', + name: 'sealedsecrets', + }) + + maps.set('AkamaiKnowledgeBase', { + kind: 'AkamaiKnowledgeBase', + envDir, + pathGlob: `${envDir}/env/teams/*/knowledgebases/*.yaml`, + pathTemplate: 'env/teams/{teamId}/knowledgebases/{name}.yaml', + name: 'knowledgebases', + }) + + maps.set('AkamaiAgent', { + kind: 'AkamaiAgent', + envDir, + pathGlob: `${envDir}/env/teams/*/agents/*.yaml`, + pathTemplate: 'env/teams/{teamId}/agents/{name}.yaml', + name: 'agents', + }) + + maps.set('AplTeamNetworkControl', { + kind: 'AplTeamNetworkControl', + envDir, + pathGlob: `${envDir}/env/teams/*/netpols/*.yaml`, + pathTemplate: 'env/teams/{teamId}/netpols/{name}.yaml', + name: 'netpols', + }) + + maps.set('AplTeamSettingSet', { + kind: 'AplTeamSettingSet', + envDir, + pathGlob: `${envDir}/env/teams/*/*settings{.yaml,.yaml.dec}`, + pathTemplate: 'env/teams/{teamId}/settings.yaml', + name: 'settings', + }) + + maps.set('AplTeamTool', { + kind: 'AplTeamTool', + envDir, + pathGlob: `${envDir}/env/teams/*/*apps{.yaml,.yaml.dec}`, + pathTemplate: 'env/teams/{teamId}/apps.yaml', + name: 'apps', + }) + + maps.set('AplTeamPolicy', { + kind: 'AplTeamPolicy', + envDir, + pathGlob: `${envDir}/env/teams/*/policies/*.yaml`, + pathTemplate: 'env/teams/{teamId}/policies/{name}.yaml', + name: 'policies', + }) + + return maps +} +export function getFileMapForKind(kind: AplKind): FileMap { + return getFileMaps('').get(kind)! +} + +export function getResourceFilePath(kind: AplKind, name: string, teamId?: string): string { + const fileMap = getFileMapForKind(kind) + if (!fileMap) { + throw new Error(`Unknown kind: ${kind}`) + } + + return fileMap.pathTemplate.replace('{teamId}', teamId || '').replace('{name}', name) +} + +// Derive secret file path from main file path +// e.g., 'env/teams/demo/settings.yaml' -> 'env/teams/demo/secrets.settings.yaml' +// e.g., 'env/apps/harbor.yaml' -> 'env/apps/secrets.harbor.yaml' +export function getSecretFilePath(mainFilePath: string): string { + const parts = mainFilePath.split('/') + const filename = parts[parts.length - 1] + const dir = parts.slice(0, -1).join('/') + + return `${dir}/secrets.${filename}` +} + +// Get all FileMap entries that are settings (env/settings/*) +export function getSettingsFileMaps(envDir: string): Map { + const allMaps = getFileMaps(envDir) + const settingsMaps = new Map() + + for (const [, fileMap] of allMaps.entries()) { + if (fileMap.pathTemplate.startsWith('env/settings/')) { + settingsMaps.set(fileMap.name, fileMap) + } + } + + return settingsMaps +} diff --git a/src/fileStore/file-store.ts b/src/fileStore/file-store.ts new file mode 100644 index 000000000..792fd3090 --- /dev/null +++ b/src/fileStore/file-store.ts @@ -0,0 +1,209 @@ +// The in-memory key-value store: file path -> parsed content +import path from 'path' +import { globSync } from 'glob' +import { ensureDir } from 'fs-extra' +import { writeFile } from 'fs/promises' +import { stringify as stringifyYaml } from 'yaml' +import { z } from 'zod' +import { merge } from 'lodash' +import { loadYaml } from '../utils' +import { getFileMapForKind, getFileMaps, getResourceFilePath } from './file-map' +import { APL_KINDS, AplKind, AplObject, AplPlatformObject, AplRecord, AplTeamObject } from '../otomi-models' +import Debug from 'debug' + +const debug = Debug('otomi:file-store') + +// Zod schema for validating APL objects (team resources) +const AplObjectSchema = z.object({ + kind: z.enum(APL_KINDS), + metadata: z.looseObject({ + name: z.string(), + labels: z.record(z.string(), z.string()).optional(), + }), + spec: z.record(z.string(), z.any()), + status: z.record(z.string(), z.any()).optional(), +}) + +export function getTeamIdFromPath(filePath: string): string | undefined { + const match = filePath.match(/\/teams\/([^/]+)/) + return match ? match[1] : undefined +} + +export async function writeFileToDisk(repoPath: string, relativePath: string, content: AplObject): Promise { + const fullPath = path.join(repoPath, relativePath) + await ensureDir(path.dirname(fullPath)) + const yamlContent = stringifyYaml(content) + await writeFile(fullPath, yamlContent, 'utf8') +} + +function hasDecryptedFile(filePath: string, fileList: string[]): boolean { + return fileList.includes(`${filePath}.dec`) +} + +function shouldSkipValidation(filePath: string): boolean { + return filePath.includes('/sealedsecrets/') || filePath.includes('/workloadValues/') +} + +export class FileStore { + private store: Map = new Map() + + // Static factory method to load FileStore from disk + static async load(envDir: string): Promise { + const store = new FileStore() + const fileMaps = getFileMaps(envDir) + + // PASS 1: Load all files into temporary storage + const allFiles = new Map() + + const fileMapResults = await Promise.all( + Array.from(fileMaps.values()).map(async (fileMap) => { + const files = globSync(fileMap.pathGlob, { nodir: true, dot: false }) + return files.sort() + }), + ) + + const filesToLoad = fileMapResults.flatMap((files) => + files.filter((filePath) => !hasDecryptedFile(filePath, files)), + ) + + await Promise.all( + filesToLoad.map(async (filePath) => { + const rawContent = await loadYaml(filePath) + const relativePath = path.relative(envDir, filePath).replace(/\.dec$/, '') + + // Skip validation for specific file paths + if (shouldSkipValidation(filePath)) { + allFiles.set(relativePath, rawContent as AplObject) + return + } + + // Validate all other kinds + const result = AplObjectSchema.safeParse(rawContent) + + if (!result.success) { + debug(`Validation failed for ${relativePath}:`, result.error.message) + return + } + + if (!result.data) { + debug(`No content found for ${relativePath}`) + return + } + + allFiles.set(relativePath, result.data as AplObject) + }), + ) + + // PASS 2: Merge secret files into main files + for (const [filePath, content] of allFiles.entries()) { + if (filePath.includes('/secrets.')) { + // This is a secret file - find its main file + const mainFilePath = filePath.replace('/secrets.', '/') + const mainContent = allFiles.get(mainFilePath) + + if (mainContent) { + // Normal case: merge secret spec into main spec using DEEP merge + mainContent.spec = merge({}, mainContent.spec, content.spec) + // Keep the merged main file in allFiles for final storage + } else { + // Special case (users): no main file exists, secret IS the main + // Store at main path (without secrets. prefix) + allFiles.set(mainFilePath, content) + } + // Remove secret file from map (don't store separately) + allFiles.delete(filePath) + } + } + + // Store final merged files + for (const [filePath, content] of allFiles.entries()) { + store.set(filePath, content) + } + + return store + } + + get(filePath: string): AplObject | undefined { + return this.store.get(filePath) + } + + //Set types that do not adhere to AplObject e.g. SealedSecrets and WorkloadValues + set(filePath: string, content: any): AplRecord { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + this.store.set(filePath, content) + return { filePath, content } + } + + delete(filePath: string): string { + this.store.delete(filePath) + return filePath + } + + keys(): IterableIterator { + return this.store.keys() + } + + // Typed methods for team resources + getTeamResource(kind: AplKind, teamId: string, name: string): AplObject | undefined { + const filePath = getResourceFilePath(kind, name, teamId) + return this.store.get(filePath) + } + + setTeamResource(aplTeamObject: AplTeamObject): string { + const filePath = getResourceFilePath( + aplTeamObject.kind, + aplTeamObject.metadata.name, + aplTeamObject.metadata.labels['apl.io/teamId'], + ) + this.store.set(filePath, aplTeamObject) + return filePath + } + + setPlatformResource(aplPlatformObject: AplPlatformObject): string { + const filePath = getResourceFilePath(aplPlatformObject.kind, aplPlatformObject.metadata.name) + this.store.set(filePath, aplPlatformObject) + return filePath + } + + deleteTeamResource(kind: AplKind, teamId: string, name: string): string { + const filePath = getResourceFilePath(kind, name, teamId) + this.store.delete(filePath) + return filePath + } + + // Generic method for all resources (platform and team) + getByKind(kind: AplKind, teamId?: string): Map { + const fileMap = getFileMapForKind(kind) + if (!fileMap) { + throw new Error(`Unknown kind: ${kind}`) + } + + // Generate path prefix from template (e.g., 'env/teams/team1/workloads/') + const prefix = fileMap.pathTemplate.replace('{teamId}', teamId || '').replace('{name}.yaml', '') + + const result = new Map() + for (const filePath of this.store.keys()) { + if (filePath.startsWith(prefix) && filePath.endsWith('.yaml')) { + const content = this.store.get(filePath) + if (content) result.set(filePath, content) + } + } + return result + } + + getTeamIds(): string[] { + const teamIds = new Set() + for (const filePath of this.store.keys()) { + const teamId = getTeamIdFromPath(filePath) + if (teamId) teamIds.add(teamId) + } + return Array.from(teamIds).sort() + } + + // Copy from another FileStore + copyFrom(other: FileStore): void { + for (const filePath of other.keys()) { + this.store.set(filePath, other.get(filePath)!) + } + } +} diff --git a/src/middleware/jwt.test.ts b/src/middleware/jwt.test.ts index 23230b22f..23be850e8 100644 --- a/src/middleware/jwt.test.ts +++ b/src/middleware/jwt.test.ts @@ -34,8 +34,7 @@ describe('JWT claims mapping', () => { beforeEach(async () => { otomiStack = new OtomiStack() otomiStack.git = mockDeep() - jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) - + otomiStack.doDeployment = jest.fn().mockImplementation(() => Promise.resolve()) await otomiStack.init() await otomiStack.loadValues() }) @@ -57,7 +56,7 @@ describe('JWT claims mapping', () => { }) test('Multiple team groups should result in the same amount of teams existing', async () => { - await Promise.all(multiTeamUser.map(async (teamId) => otomiStack.createTeam({ name: teamId }, false))) + await Promise.all(multiTeamUser.map(async (teamId) => otomiStack.createTeam({ name: teamId }))) const user = getUser(multiTeamJWT, otomiStack) expect(user.teams).toEqual(multiTeamUser) expect(user.isPlatformAdmin).toBeFalsy() @@ -65,7 +64,7 @@ describe('JWT claims mapping', () => { test("Non existing team groups should not be added to the user's list of teams", async () => { const extraneousTeamsList = [...multiTeamUser, 'nonexist'] - await Promise.all(extraneousTeamsList.map(async (teamId) => otomiStack.createTeam({ name: teamId }, false))) + await Promise.all(extraneousTeamsList.map(async (teamId) => otomiStack.createTeam({ name: teamId }))) const user = getUser(multiTeamJWT, otomiStack) expect(user.teams).toEqual(multiTeamUser) expect(user.isPlatformAdmin).toBeFalsy() diff --git a/src/middleware/jwt.ts b/src/middleware/jwt.ts index 53586b763..af31d6d99 100644 --- a/src/middleware/jwt.ts +++ b/src/middleware/jwt.ts @@ -38,8 +38,8 @@ export function getUser(user: JWT, otomi: OtomiStack): SessionUser { if (group.substring(0, 5) === 'team-' && !sessionUser.teams.includes(teamId)) { // we might be assigned team-* without that team yet existing in the values, so ignore those if (otomi.isLoaded) { - const existing = otomi.repoService.getTeamConfig(teamId) - if (existing) { + const exists = otomi.getTeamIds().includes(teamId) + if (exists) { sessionUser.teams.push(teamId) } } diff --git a/src/middleware/session.ts b/src/middleware/session.ts index e4874c374..1cc8b78d7 100644 --- a/src/middleware/session.ts +++ b/src/middleware/session.ts @@ -42,7 +42,7 @@ export const setSessionStack = async (editor: string, sessionId: string): Promis debug(`Creating session ${sessionId} for user ${editor}`) sessions[sessionId] = new OtomiStack(editor, sessionId) await sessions[sessionId].initGitWorktree(readOnlyStack.git) - sessions[sessionId].repoService = cloneDeep(readOnlyStack.repoService) + sessions[sessionId].fileStore.copyFrom(readOnlyStack.fileStore) } else sessions[sessionId].sessionId = sessionId return sessions[sessionId] } diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index b992a4149..f9ca18fd6 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -2651,26 +2651,6 @@ paths: application/json: schema: $ref: '#/components/schemas/App' - put: - operationId: editApp - x-eov-operation-handler: v1/apps/{teamId}/{appId} - description: Edit app values - x-aclSchema: App - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/App' - description: Edit app values - required: true - responses: - '400': - $ref: '#/components/responses/BadRequest' - '404': - $ref: '#/components/responses/NotFound' - '200': - description: Successfully edited app values. - '/alpha/ai/models': get: operationId: getAIModels diff --git a/src/otomi-models.ts b/src/otomi-models.ts index a341cd402..c882a09c0 100644 --- a/src/otomi-models.ts +++ b/src/otomi-models.ts @@ -1,6 +1,6 @@ import { Request } from 'express' import { JSONSchema4 } from 'json-schema' -import { components, external, operations, paths } from 'src/generated-schema' +import { components, operations, paths } from 'src/generated-schema' import OtomiStack from 'src/otomi-stack' export type App = components['schemas']['App'] @@ -92,36 +92,73 @@ export type AplResponseObject = | AplServiceResponse | AplWorkloadResponse | AplTeamSettingsResponse -export type AplKind = - | 'AplApp' - | 'AplAlertSet' - | 'AplCluster' - | 'AplDatabase' - | 'AplDns' - | 'AplIngress' - | 'AplObjectStorage' - | 'AplKms' - | 'AplIdentityProvider' - | 'AplCapabilitySet' - | 'AplSmtp' - | 'AplBackupCollection' - | 'AplUser' - | 'AplPlatformSettingSet' - | 'AkamaiKnowledgeBase' - | 'AkamaiAgent' - | 'AplTeamCodeRepo' - | 'AplTeamBuild' - | 'AplTeamPolicy' - | 'AplTeamSettingSet' - | 'AplTeamNetworkControl' - | 'AplTeamSecret' - | 'AplTeamService' - | 'AplTeamWorkload' - | 'AplTeamWorkloadValues' - | 'AplTeamTool' - | 'AplVersion' +export const APL_KINDS = [ + 'AplApp', + 'AplAlertSet', + 'AplCluster', + 'AplDatabase', + 'AplDns', + 'AplIngress', + 'AplObjectStorage', + 'AplKms', + 'AplIdentityProvider', + 'AplCapabilitySet', + 'AplSmtp', + 'AplBackupCollection', + 'AplUser', + 'AplPlatformSettingSet', + 'AkamaiKnowledgeBase', + 'AkamaiAgent', + 'AplTeamCodeRepo', + 'AplTeamBuild', + 'AplTeamPolicy', + 'AplTeamSettingSet', + 'AplTeamNetworkControl', + 'AplTeamSecret', + 'AplTeamService', + 'AplTeamWorkload', + 'AplTeamWorkloadValues', + 'AplTeamTool', + 'AplVersion', +] as const +export type AplKind = (typeof APL_KINDS)[number] export type V1ApiObject = Build | CodeRepo | Netpol | SealedSecret | Service | Workload +// AplObject: The disk storage format with optional labels and status +export type AplObject = { + kind: AplKind + metadata: { + name: string + labels?: Record + } + spec: Record + status?: Record +} + +export type AplTeamObject = { + kind: AplKind + metadata: { + name: string + labels: { + 'apl.io/teamId': string + } + } + spec: Record +} + +export type AplPlatformObject = { + kind: AplKind + metadata: { + name: string + } + spec: Record +} + +export type AplRecord = { + filePath: string + content: AplObject +} + export type DeepPartial = T extends object ? { [K in keyof T]?: DeepPartial } : T export interface OpenApiRequest extends Request { @@ -175,7 +212,6 @@ export interface OtomiSpec { components: components paths: paths operations: operations - external: external } export type SchemaType = 'object' | 'array' @@ -278,3 +314,44 @@ export interface TeamConfig { settings: AplTeamSettingsResponse workloads: AplWorkloadResponse[] } + +export function toTeamObject(teamId: string, request: AplRequestObject): AplTeamObject { + return { + kind: request.kind, + metadata: { + name: request.metadata.name, + labels: { + 'apl.io/teamId': teamId, + }, + }, + spec: request.spec, + } +} + +export function buildTeamObject(existing: AplResponseObject, updatedSpec: Record): AplTeamObject { + return { + kind: existing.kind, + metadata: { + name: existing.metadata.name, + labels: existing.metadata.labels as { 'apl.io/teamId': string }, + }, + spec: updatedSpec, + } +} + +export function buildPlatformObject(kind: AplKind, name: string, spec: Record): AplPlatformObject { + return { + kind, + metadata: { name }, + spec, + } +} + +export function toPlatformObject(kind: AplKind, name: string, spec: Record): AplObject { + return { + kind, + metadata: { name }, + spec, + status: {}, + } +} diff --git a/src/otomi-stack.test.ts b/src/otomi-stack.test.ts index fd2fe4baf..bd886589a 100644 --- a/src/otomi-stack.test.ts +++ b/src/otomi-stack.test.ts @@ -4,17 +4,13 @@ import { AplServiceRequest, AplTeamSettingsRequest, App, - CodeRepo, SessionUser, - TeamConfig, User, } from 'src/otomi-models' import OtomiStack from 'src/otomi-stack' import { loadSpec } from './app' import { PublicUrlExists, ValidationError } from './error' import { Git } from './git' -import { RepoService } from './services/RepoService' -import { TeamConfigService } from './services/TeamConfigService' jest.mock('src/middleware', () => ({ ...jest.requireActual('src/middleware'), @@ -51,57 +47,64 @@ beforeAll(async () => { await loadSpec() }) +// Helper functions for FileStore-based tests +function createTestUser(otomiStack: OtomiStack, user: User): void { + const { buildPlatformObject } = require('./otomi-models') + const aplUser = buildPlatformObject('AplUser', user.id!, user as any) + otomiStack.fileStore.setPlatformResource(aplUser) +} + +function createTestTeam(otomiStack: OtomiStack, teamId: string, spec: any = {}): void { + const teamSettings: AplTeamSettingsRequest = { + kind: 'AplTeamSettingSet', + metadata: { + name: teamId, + labels: { + 'apl.io/teamId': teamId, + }, + }, + spec, + } + otomiStack.fileStore.setTeamResource(teamSettings) +} + +function createTestService(otomiStack: OtomiStack, teamId: string, name: string, spec: any): void { + const service: AplServiceRequest & { metadata: { labels: { 'apl.io/teamId': string } } } = { + kind: 'AplTeamService', + metadata: { + name, + labels: { + 'apl.io/teamId': teamId, + }, + }, + spec, + } + otomiStack.fileStore.setTeamResource(service) +} + describe('Data validation', () => { let otomiStack: OtomiStack - const teamId = 'aa' - let mockRepoService: jest.Mocked - let mockTeamConfigService: jest.Mocked + const teamId = 'team-1' + let mockGit: jest.Mocked beforeEach(async () => { otomiStack = new OtomiStack() await otomiStack.init() - otomiStack.git = mockDeep() - mockRepoService = mockDeep() - otomiStack.repoService = mockRepoService - - // Mock TeamConfigService - mockTeamConfigService = mockDeep() - - // Mocking getServices() to return a list of services - mockTeamConfigService.getServices.mockReturnValue([ - { - kind: 'AplTeamService', - metadata: { - name: 'svc', - labels: { - 'apl.io/teamId': 'team-1', - }, - }, - spec: { - domain: 'b.a.com', - }, - status: {}, - }, - { - kind: 'AplTeamService', - metadata: { - name: 'svc', - labels: { - 'apl.io/teamId': 'team-1', - }, - }, - spec: { - domain: 'b.a.com', - paths: ['/test/'], - }, - status: {}, - }, - ]) - // Ensure getTeamConfigService() returns our mocked TeamConfigService - mockRepoService.getTeamConfigService.mockReturnValue(mockTeamConfigService) - jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() - jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() + // Initialize FileStore + const { FileStore } = require('./fileStore/file-store') + otomiStack.fileStore = new FileStore() + + // Mock Git operations + mockGit = mockDeep() + otomiStack.git = mockGit + + // Pre-populate FileStore with test services for duplicate URL tests + createTestService(otomiStack, teamId, 'svc1', { domain: 'b.a.com' }) + createTestService(otomiStack, teamId, 'svc2', { domain: 'b.a.com', paths: ['/test/'] }) + + jest.spyOn(otomiStack, 'doDeleteDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() }) test('should throw exception on duplicated domain', () => { @@ -151,66 +154,29 @@ describe('Data validation', () => { }) test('should create a password when password is not specified', async () => { - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'team1', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - } - const createItemSpy = jest.spyOn(otomiStack.repoService, 'createTeamConfig').mockReturnValue({ - agents: [], - builds: [], - codeRepos: [], - workloads: [], - services: [], - sealedsecrets: [], - netpols: [], - settings: teamSettings, - apps: [], - policies: [], - workloadValues: [], - knowledgeBases: [], - } as TeamConfig) - await otomiStack.createTeam({ name: 'test' }, false) - expect(createItemSpy.mock.calls[0][0].spec.password).not.toEqual('') - createItemSpy.mockRestore() + await otomiStack.createTeam({ name: 'test' }) + + // Verify FileStore was updated with a generated password + const teamSettings = otomiStack.fileStore.getTeamResource('AplTeamSettingSet', 'test', 'settings') + expect(teamSettings).toBeDefined() + expect(teamSettings?.spec.password).toBeTruthy() + expect(teamSettings?.spec.password.length).toBeGreaterThan(0) + + // Verify Git operations were called + expect(mockGit.writeFile).toHaveBeenCalled() }) test('should not create a password when password is specified', async () => { - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'team1', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - } - const createItemSpy = jest.spyOn(otomiStack.repoService, 'createTeamConfig').mockReturnValue({ - agents: [], - builds: [], - codeRepos: [], - workloads: [], - services: [], - sealedsecrets: [], - netpols: [], - settings: teamSettings, - apps: [], - policies: [], - workloadValues: [], - knowledgeBases: [], - } as TeamConfig) const myPassword = 'someAwesomePassword' - await otomiStack.createTeam({ name: 'test', password: myPassword }, false) - expect(createItemSpy.mock.calls[0][0].spec.password).toEqual(myPassword) - createItemSpy.mockRestore() + await otomiStack.createTeam({ name: 'test', password: myPassword }) + + // Verify FileStore was updated with the specified password + const teamSettings = otomiStack.fileStore.getTeamResource('AplTeamSettingSet', 'test', 'settings') + expect(teamSettings).toBeDefined() + expect(teamSettings?.spec.password).toBe(myPassword) + + // Verify Git operations were called + expect(mockGit.writeFile).toHaveBeenCalled() }) test('should throw ValidationError when team name is under 3 characters', async () => { @@ -225,7 +191,7 @@ describe('Data validation', () => { spec: {}, } - await expect(otomiStack.createAplTeam(teamData, false)).rejects.toThrow( + await expect(otomiStack.createAplTeam(teamData)).rejects.toThrow( new ValidationError('Team name must be at least 3 characters long'), ) }) @@ -242,39 +208,12 @@ describe('Data validation', () => { spec: {}, } - await expect(otomiStack.createAplTeam(teamData, false)).rejects.toThrow( + await expect(otomiStack.createAplTeam(teamData)).rejects.toThrow( new ValidationError('Team name must not exceed 9 characters'), ) }) test('should not throw ValidationError when team name is exactly 9 characters', async () => { - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'ninechars', - labels: { - 'apl.io/teamId': 'ninechars', - }, - }, - spec: {}, - status: {}, - } - - const createItemSpy = jest.spyOn(otomiStack.repoService, 'createTeamConfig').mockReturnValue({ - agents: [], - builds: [], - codeRepos: [], - workloads: [], - services: [], - sealedsecrets: [], - netpols: [], - settings: teamSettings, - apps: [], - policies: [], - workloadValues: [], - knowledgeBases: [], - } as TeamConfig) - const teamData: AplTeamSettingsRequest = { kind: 'AplTeamSettingSet', metadata: { @@ -286,38 +225,15 @@ describe('Data validation', () => { spec: {}, } - await expect(otomiStack.createAplTeam(teamData, false)).resolves.not.toThrow() - createItemSpy.mockRestore() + await expect(otomiStack.createAplTeam(teamData)).resolves.not.toThrow() + + // Verify team was created in FileStore + const teamSettings = otomiStack.fileStore.getTeamResource('AplTeamSettingSet', 'ninechars', 'settings') + expect(teamSettings).toBeDefined() + expect(teamSettings?.metadata.name).toBe('ninechars') }) test('should not throw ValidationError when team name is less than 9 characters', async () => { - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'short', - labels: { - 'apl.io/teamId': 'short', - }, - }, - spec: {}, - status: {}, - } - - const createItemSpy = jest.spyOn(otomiStack.repoService, 'createTeamConfig').mockReturnValue({ - agents: [], - builds: [], - codeRepos: [], - workloads: [], - services: [], - sealedsecrets: [], - netpols: [], - settings: teamSettings, - apps: [], - policies: [], - workloadValues: [], - knowledgeBases: [], - } as TeamConfig) - const teamData: AplTeamSettingsRequest = { kind: 'AplTeamSettingSet', metadata: { @@ -329,8 +245,12 @@ describe('Data validation', () => { spec: {}, } - await expect(otomiStack.createAplTeam(teamData, false)).resolves.not.toThrow() - createItemSpy.mockRestore() + await expect(otomiStack.createAplTeam(teamData)).resolves.not.toThrow() + + // Verify team was created in FileStore + const teamSettings = otomiStack.fileStore.getTeamResource('AplTeamSettingSet', 'short', 'settings') + expect(teamSettings).toBeDefined() + expect(teamSettings?.metadata.name).toBe('short') }) }) @@ -338,12 +258,11 @@ describe('Work with values', () => { let otomiStack: OtomiStack beforeEach(async () => { otomiStack = new OtomiStack() - jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) await otomiStack.init() otomiStack.git = new Git('./test', undefined, 'someuser', 'some@ema.il', undefined, undefined) - jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() - jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeleteDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() }) test('can load from configuration to database and back', async () => { @@ -357,8 +276,8 @@ describe('Workload values', () => { otomiStack = new OtomiStack() await otomiStack.init() otomiStack.git = new Git('./test', undefined, 'someuser', 'some@ema.il', undefined, undefined) - jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() - jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeleteDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() }) test('returns filtered apps if App array is submitted isPreinstalled flag is true', () => { @@ -378,6 +297,7 @@ describe('Workload values', () => { describe('Users tests', () => { let otomiStack: OtomiStack + let mockGit: jest.Mocked const domainSuffix = 'dev.linode-apl.net' @@ -460,19 +380,29 @@ describe('Users tests', () => { beforeEach(async () => { otomiStack = new OtomiStack() await otomiStack.init() - otomiStack.git = mockDeep() + + // Initialize FileStore + const { FileStore } = require('./fileStore/file-store') + otomiStack.fileStore = new FileStore() + + // Mock Git operations + mockGit = mockDeep() + otomiStack.git = mockGit + + // Mock getSessionStack to return this otomiStack instance + const { getSessionStack } = require('src/middleware') + jest.mocked(getSessionStack).mockResolvedValue(otomiStack) jest.spyOn(otomiStack, 'getSettings').mockReturnValue({ cluster: { name: 'default-cluster', domainSuffix, provider: 'linode' }, }) - jest.spyOn(otomiStack, 'saveUser').mockResolvedValue() - jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() - jest.spyOn(otomiStack, 'doTeamDeployment').mockResolvedValue() - jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) + jest.spyOn(otomiStack, 'doDeleteDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() jest.spyOn(otomiStack, 'getApp').mockReturnValue({ id: 'keycloak' }) - await otomiStack.initRepo() - await otomiStack.createUser(defaultPlatformAdmin) - await otomiStack.createUser(anyPlatformAdmin) + + // Pre-create platform admin users in FileStore + createTestUser(otomiStack, defaultPlatformAdmin) + createTestUser(otomiStack, anyPlatformAdmin) }) afterEach(() => { @@ -482,7 +412,7 @@ describe('Users tests', () => { test('should not allow deleting the default platform admin user', async () => { await expect(otomiStack.deleteUser('1')).rejects.toMatchObject({ code: 403, - publicMessage: 'Forbidden', + publicMessage: 'Cannot delete the default platform admin user', }) }) @@ -494,9 +424,16 @@ describe('Users tests', () => { beforeEach(async () => { otomiStack = new OtomiStack() await otomiStack.init() - otomiStack.git = mockDeep() - await otomiStack.initRepo() - otomiStack.repoService.createUser(teamMember1) + + // Initialize FileStore + const { FileStore } = require('./fileStore/file-store') + otomiStack.fileStore = new FileStore() + + mockGit = mockDeep() + otomiStack.git = mockGit + + // Pre-create test user in FileStore + createTestUser(otomiStack, teamMember1) }) it('should return full user for platform admin', () => { @@ -634,16 +571,22 @@ describe('Users tests', () => { describe('editUser', () => { it('should allow platform admin to edit a user', async () => { const user = { ...defaultPlatformAdmin, id: '3', email: 'edit@dev.linode-apl.net' } - await otomiStack.createUser(user) + createTestUser(otomiStack, user) + const updated = { ...user, firstName: 'edited' } - jest.spyOn(otomiStack.repoService, 'updateUser').mockReturnValue(updated) const result = await otomiStack.editUser(user.id, updated, platformAdminSession) + expect(result.firstName).toBe('edited') + + // Verify FileStore was updated + const storedUser = otomiStack.fileStore.get(`env/users/${user.id}.yaml`) + expect(storedUser?.spec.firstName).toBe('edited') }) it('should not allow non-platform admin to edit a user', async () => { const user = { ...defaultPlatformAdmin, id: '4', email: 'edit2@dev.linode-apl.net' } - await otomiStack.createUser(user) + createTestUser(otomiStack, user) + await expect( otomiStack.editUser(user.id, user, { ...sessionUser, isPlatformAdmin: false }), ).rejects.toMatchObject({ @@ -704,12 +647,13 @@ describe('Users tests', () => { teamMember1.teams = ['team1'] teamMember2.teams = ['team2'] - otomiStack.repoService.createUser({ ...sessionUser, firstName: 'Session', lastName: 'User' }) - otomiStack.repoService.createUser(teamAdmin) - otomiStack.repoService.createUser(teamMember1) - otomiStack.repoService.createUser(teamMember2) - jest.spyOn(otomiStack, 'saveUser').mockResolvedValue() - jest.spyOn(otomiStack, 'doRepoDeployment').mockResolvedValue() + // Pre-create users in FileStore + createTestUser(otomiStack, { ...sessionUser, firstName: 'Session', lastName: 'User' }) + createTestUser(otomiStack, teamAdmin) + createTestUser(otomiStack, teamMember1) + createTestUser(otomiStack, teamMember2) + + jest.spyOn(otomiStack, 'doDeleteDeployment').mockResolvedValue() }) it('should allow platform admin to update any user teams', async () => { @@ -735,7 +679,8 @@ describe('Users tests', () => { const data = [{ ...teamMember2, teams: ['team3'] }] await expect(otomiStack.editTeamUsers(data, sessionUser)).rejects.toMatchObject({ code: 403, - publicMessage: 'Forbidden', + publicMessage: + 'Team admins are permitted to add or remove users only within the teams they manage. However, they cannot remove themselves or other team admins from those teams.', }) }) @@ -763,11 +708,12 @@ describe('Users tests', () => { authz: {}, teams: ['team1'], roles: [], + sub: 'regular-user', } const data = [{ ...teamMember2, teams: ['team1'] }] await expect(otomiStack.editTeamUsers(data, regularUser)).rejects.toMatchObject({ code: 403, - publicMessage: 'Forbidden', + publicMessage: "Only platform admins or team admins can modify a user's team memberships.", }) }) }) @@ -987,33 +933,27 @@ describe('PodService', () => { describe('Code repositories tests', () => { let otomiStack: OtomiStack - let teamConfigService: TeamConfigService + let mockGit: jest.Mocked beforeEach(async () => { otomiStack = new OtomiStack() - jest.spyOn(otomiStack, 'transformApps').mockReturnValue([]) await otomiStack.init() - await otomiStack.initRepo() - otomiStack.git = mockDeep() + + // Initialize FileStore + const { FileStore } = require('./fileStore/file-store') + otomiStack.fileStore = new FileStore() + + // Mock Git operations + mockGit = mockDeep() + otomiStack.git = mockGit + const { getSessionStack } = require('src/middleware') jest.mocked(getSessionStack).mockResolvedValue(otomiStack) - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'demo', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: {}, - status: {}, - } as AplTeamSettingsRequest - try { - otomiStack.repoService.createTeamConfig(teamSettings) - } catch { - // ignore - } - teamConfigService = otomiStack.repoService.getTeamConfigService('demo') + + // Pre-create team in FileStore + createTestTeam(otomiStack, 'demo', {}) + + // Pre-create a code repo for testing get/edit/delete operations const codeRepo: AplCodeRepoResponse = { kind: 'AplTeamCodeRepo', metadata: { @@ -1025,9 +965,11 @@ describe('Code repositories tests', () => { spec: { gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, status: {}, } + otomiStack.fileStore.setTeamResource(codeRepo) - jest.spyOn(teamConfigService, 'getCodeRepo').mockReturnValue(codeRepo) - jest.spyOn(otomiStack.git, 'deleteConfig').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeployment').mockResolvedValue() + jest.spyOn(otomiStack, 'doDeleteDeployment').mockResolvedValue() + jest.spyOn(mockGit, 'removeFile').mockResolvedValue() }) afterEach(() => { @@ -1035,71 +977,39 @@ describe('Code repositories tests', () => { }) test('should create an internal code repository', async () => { - const createItemSpy = jest.spyOn(teamConfigService, 'createCodeRepo').mockReturnValue({ - kind: 'AplTeamCodeRepo', - metadata: { - labels: { - 'apl.io/teamId': 'demo', - }, - name: 'code-1', - }, - spec: { gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, - status: {}, - }) - - const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamConfigItem').mockResolvedValue() - const codeRepo = await otomiStack.createCodeRepo('demo', { - name: 'code-1', + name: 'code-2', gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', + repositoryUrl: 'https://gitea-new.test.com', }) + // Verify return value expect(codeRepo).toEqual({ teamId: 'demo', - name: 'code-1', + name: 'code-2', gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', - }) - expect(createItemSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1', - }, - spec: { name: 'code-1', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, - }) - expect(saveTeamCodeReposSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: { gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, - status: {}, + repositoryUrl: 'https://gitea-new.test.com', }) - createItemSpy.mockRestore() - saveTeamCodeReposSpy.mockRestore() + // Verify FileStore state + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'code-2') + expect(stored).toBeDefined() + expect(stored?.metadata.name).toBe('code-2') + expect(stored?.spec.gitService).toBe('gitea') + expect(stored?.spec.repositoryUrl).toBe('https://gitea-new.test.com') + + // Verify Git operations + expect(mockGit.writeFile).toHaveBeenCalledWith( + 'env/teams/demo/codeRepos/code-2.yaml', + expect.objectContaining({ kind: 'AplTeamCodeRepo' }), + ) + expect(otomiStack.doDeployment).toHaveBeenCalled() }) test('should get an existing internal code repository', () => { - const codeRepo: AplCodeRepoResponse = { - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: { gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, - status: {}, - } - - jest.spyOn(teamConfigService, 'getCodeRepo').mockReturnValue(codeRepo) + // code-1 is already pre-created in beforeEach + const result = otomiStack.getCodeRepo('demo', 'code-1') - const result = otomiStack.getCodeRepo('demo', '1') expect(result).toEqual({ teamId: 'demo', name: 'code-1', @@ -1109,140 +1019,77 @@ describe('Code repositories tests', () => { }) test('should edit an existing internal code repository', async () => { - const updateItemSpy = jest.spyOn(teamConfigService, 'updateCodeRepo').mockReturnValue({ - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1-updated', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: { - gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', - }, - status: {}, - }) - - const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamConfigItem').mockResolvedValue() - - const codeRepo = await otomiStack.editCodeRepo('demo', '1', { + const codeRepo = await otomiStack.editCodeRepo('demo', 'code-1', { teamId: 'demo', - name: 'code-1-updated', + name: 'code-1', gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', + repositoryUrl: 'https://gitea-updated.test.com', }) + // Verify return value expect(codeRepo).toEqual({ teamId: 'demo', - name: 'code-1-updated', + name: 'code-1', gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', - }) - expect(updateItemSpy).toHaveBeenCalledWith('1', { - metadata: { name: 'code-1-updated' }, - spec: { name: 'code-1-updated', gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, - }) - expect(saveTeamCodeReposSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1-updated', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: { gitService: 'gitea', repositoryUrl: 'https://gitea.test.com' }, - status: {}, + repositoryUrl: 'https://gitea-updated.test.com', }) - updateItemSpy.mockRestore() - saveTeamCodeReposSpy.mockRestore() + // Verify FileStore was updated + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'code-1') + expect(stored?.spec.repositoryUrl).toBe('https://gitea-updated.test.com') + + // Verify Git operations + expect(mockGit.writeFile).toHaveBeenCalled() + expect(otomiStack.doDeployment).toHaveBeenCalled() }) test('should delete an existing internal code repository', async () => { - const codeRepo = { - teamId: 'demo', - name: 'code-1', - gitService: 'gitea', - repositoryUrl: 'https://gitea.test.com', - } as CodeRepo + // Verify code-1 exists in FileStore + let stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'code-1') + expect(stored).toBeDefined() - jest.spyOn(otomiStack, 'getCodeRepo').mockReturnValue(codeRepo) - const deleteItemSpy = jest.spyOn(teamConfigService, 'deleteCodeRepo').mockResolvedValue({} as never) + await otomiStack.deleteCodeRepo('demo', 'code-1') - await otomiStack.deleteCodeRepo('demo', '1') + // Verify it was deleted from FileStore + stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'code-1') + expect(stored).toBeUndefined() - expect(deleteItemSpy).toHaveBeenCalledWith('1') - - deleteItemSpy.mockRestore() + // Verify Git operations + expect(mockGit.removeFile).toHaveBeenCalled() + expect(otomiStack.doDeleteDeployment).toHaveBeenCalled() }) test('should create an external public code repository', async () => { - const createItemSpy = jest.spyOn(teamConfigService, 'createCodeRepo').mockReturnValue({ - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: { - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - status: {}, - }) - - const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamConfigItem').mockResolvedValue() - const codeRepo = await otomiStack.createCodeRepo('demo', { - name: 'code-1', + name: 'ext-pub-1', gitService: 'github', repositoryUrl: 'https://github.test.com', }) + // Verify return value expect(codeRepo).toEqual({ teamId: 'demo', - name: 'code-1', + name: 'ext-pub-1', gitService: 'github', repositoryUrl: 'https://github.test.com', }) - expect(createItemSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { name: 'code-1' }, - spec: { - name: 'code-1', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - }) - expect(saveTeamCodeReposSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { - name: 'code-1', - labels: { - 'apl.io/teamId': 'demo', - }, - }, - spec: { - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - status: {}, - }) - createItemSpy.mockRestore() - saveTeamCodeReposSpy.mockRestore() + // Verify FileStore state + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'ext-pub-1') + expect(stored).toBeDefined() + expect(stored?.spec.gitService).toBe('github') + + // Verify Git operations + expect(mockGit.writeFile).toHaveBeenCalled() }) test('should get an existing external public code repository', () => { - const codeRepo: AplCodeRepoResponse = { + // Create external public repo in FileStore + const extPubRepo: AplCodeRepoResponse = { kind: 'AplTeamCodeRepo', metadata: { - name: 'code-1', - labels: { - 'apl.io/teamId': 'demo', - }, + name: 'ext-pub-get', + labels: { 'apl.io/teamId': 'demo' }, }, spec: { gitService: 'github', @@ -1250,160 +1097,109 @@ describe('Code repositories tests', () => { }, status: {}, } + otomiStack.fileStore.setTeamResource(extPubRepo) - jest.spyOn(teamConfigService, 'getCodeRepo').mockReturnValue(codeRepo) - - const result = otomiStack.getCodeRepo('demo', '1') + const result = otomiStack.getCodeRepo('demo', 'ext-pub-get') expect(result).toEqual({ teamId: 'demo', - name: 'code-1', + name: 'ext-pub-get', gitService: 'github', repositoryUrl: 'https://github.test.com', }) }) test('should edit an existing external public code repository', async () => { - const updateItemSpy = jest.spyOn(teamConfigService, 'updateCodeRepo').mockReturnValue({ + // Create repo to edit + const extPubRepo: AplCodeRepoResponse = { kind: 'AplTeamCodeRepo', metadata: { - name: 'code-1-updated', - labels: { - 'apl.io/teamId': 'demo', - }, + name: 'ext-pub-edit', + labels: { 'apl.io/teamId': 'demo' }, }, spec: { gitService: 'github', repositoryUrl: 'https://github.test.com', }, status: {}, - }) - - const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamConfigItem').mockResolvedValue() + } + otomiStack.fileStore.setTeamResource(extPubRepo) - const codeRepo = await otomiStack.editCodeRepo('demo', '1', { + const codeRepo = await otomiStack.editCodeRepo('demo', 'ext-pub-edit', { teamId: 'demo', - name: 'code-1-updated', + name: 'ext-pub-edit', gitService: 'github', - repositoryUrl: 'https://github.test.com', + repositoryUrl: 'https://github-updated.test.com', }) + // Verify return value expect(codeRepo).toEqual({ teamId: 'demo', - name: 'code-1-updated', + name: 'ext-pub-edit', gitService: 'github', - repositoryUrl: 'https://github.test.com', - }) - expect(updateItemSpy).toHaveBeenCalledWith('1', { - metadata: { name: 'code-1-updated' }, - spec: { - name: 'code-1-updated', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - }) - expect(saveTeamCodeReposSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { name: 'code-1-updated', labels: { 'apl.io/teamId': 'demo' } }, - spec: { - gitService: 'github', - repositoryUrl: 'https://github.test.com', - }, - status: {}, + repositoryUrl: 'https://github-updated.test.com', }) - updateItemSpy.mockRestore() - saveTeamCodeReposSpy.mockRestore() + // Verify FileStore was updated + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'ext-pub-edit') + expect(stored?.spec.repositoryUrl).toBe('https://github-updated.test.com') }) test('should delete an existing external public code repository', async () => { - const codeRepo = { - teamId: 'demo', - name: 'code-1', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - } as CodeRepo - - jest.spyOn(otomiStack, 'getCodeRepo').mockReturnValue(codeRepo) - const deleteItemSpy = jest.spyOn(teamConfigService, 'deleteCodeRepo').mockResolvedValue({} as never) - - await otomiStack.deleteCodeRepo('demo', '1') - - expect(deleteItemSpy).toHaveBeenCalledWith('1') - - deleteItemSpy.mockRestore() - }) - - test('should create an external private code repository', async () => { - const createItemSpy = jest.spyOn(teamConfigService, 'createCodeRepo').mockReturnValue({ + // Create repo to delete + const extPubRepo: AplCodeRepoResponse = { kind: 'AplTeamCodeRepo', metadata: { - name: 'code-1', - labels: { - 'apl.io/teamId': 'demo', - }, + name: 'ext-pub-del', + labels: { 'apl.io/teamId': 'demo' }, }, spec: { gitService: 'github', repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', }, status: {}, - }) + } + otomiStack.fileStore.setTeamResource(extPubRepo) - const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamConfigItem').mockResolvedValue() + await otomiStack.deleteCodeRepo('demo', 'ext-pub-del') + // Verify it was deleted + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'ext-pub-del') + expect(stored).toBeUndefined() + }) + + test('should create an external private code repository', async () => { const codeRepo = await otomiStack.createCodeRepo('demo', { - name: 'code-1', + name: 'ext-priv-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', }) + // Verify return value expect(codeRepo).toEqual({ teamId: 'demo', - name: 'code-1', + name: 'ext-priv-1', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, secret: 'test', }) - expect(createItemSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { name: 'code-1' }, - spec: { - name: 'code-1', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', - }, - }) - expect(saveTeamCodeReposSpy).toHaveBeenCalledWith({ - kind: 'AplTeamCodeRepo', - metadata: { name: 'code-1', labels: { 'apl.io/teamId': 'demo' } }, - spec: { - gitService: 'github', - repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', - }, - status: {}, - }) - createItemSpy.mockRestore() - saveTeamCodeReposSpy.mockRestore() + // Verify FileStore state + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'ext-priv-1') + expect(stored).toBeDefined() + expect(stored?.spec.private).toBe(true) + expect(stored?.spec.secret).toBe('test') }) test('should edit an existing external private code repository', async () => { - const updateItemSpy = jest.spyOn(teamConfigService, 'updateCodeRepo').mockReturnValue({ + // Create repo to edit + const extPrivRepo: AplCodeRepoResponse = { kind: 'AplTeamCodeRepo', metadata: { - name: 'code-1-updated', - labels: { - 'apl.io/teamId': 'demo', - }, + name: 'ext-priv-edit', + labels: { 'apl.io/teamId': 'demo' }, }, spec: { gitService: 'github', @@ -1412,46 +1208,40 @@ describe('Code repositories tests', () => { secret: 'test', }, status: {}, - }) - - const saveTeamCodeReposSpy = jest.spyOn(otomiStack, 'saveTeamConfigItem').mockResolvedValue() + } + otomiStack.fileStore.setTeamResource(extPrivRepo) - const codeRepo = await otomiStack.editCodeRepo('demo', '1', { + const codeRepo = await otomiStack.editCodeRepo('demo', 'ext-priv-edit', { teamId: 'demo', - name: 'code-1-updated', + name: 'ext-priv-edit', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, - secret: 'test', + secret: 'test-updated', }) + // Verify return value expect(codeRepo).toEqual({ teamId: 'demo', - name: 'code-1-updated', + name: 'ext-priv-edit', gitService: 'github', repositoryUrl: 'https://github.test.com', private: true, - secret: 'test', - }) - expect(updateItemSpy).toHaveBeenCalledWith('1', { - metadata: { - name: 'code-1-updated', - }, - spec: { - name: 'code-1-updated', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', - }, + secret: 'test-updated', }) - expect(saveTeamCodeReposSpy).toHaveBeenCalledWith({ + + // Verify FileStore was updated + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'ext-priv-edit') + expect(stored?.spec.secret).toBe('test-updated') + }) + + test('should delete an existing external private code repository', async () => { + // Create repo to delete + const extPrivRepo: AplCodeRepoResponse = { kind: 'AplTeamCodeRepo', metadata: { - name: 'code-1-updated', - labels: { - 'apl.io/teamId': 'demo', - }, + name: 'ext-priv-del', + labels: { 'apl.io/teamId': 'demo' }, }, spec: { gitService: 'github', @@ -1460,29 +1250,13 @@ describe('Code repositories tests', () => { secret: 'test', }, status: {}, - }) - - updateItemSpy.mockRestore() - saveTeamCodeReposSpy.mockRestore() - }) - - test('should delete an existing external private code repository', async () => { - const codeRepo = { - teamId: 'demo', - name: 'code-1', - gitService: 'github', - repositoryUrl: 'https://github.test.com', - private: true, - secret: 'test', - } as CodeRepo - - jest.spyOn(otomiStack, 'getCodeRepo').mockReturnValue(codeRepo) - const deleteItemSpy = jest.spyOn(teamConfigService, 'deleteCodeRepo').mockResolvedValue({} as never) - - await otomiStack.deleteCodeRepo('demo', '1') + } + otomiStack.fileStore.setTeamResource(extPrivRepo) - expect(deleteItemSpy).toHaveBeenCalledWith('1') + await otomiStack.deleteCodeRepo('demo', 'ext-priv-del') - deleteItemSpy.mockRestore() + // Verify it was deleted + const stored = otomiStack.fileStore.getTeamResource('AplTeamCodeRepo', 'demo', 'ext-priv-del') + expect(stored).toBeUndefined() }) }) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index f2829d1b0..ab6f9a17d 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -1,4 +1,4 @@ -import { CoreV1Api, User as k8sUser, KubeConfig, V1ObjectReference } from '@kubernetes/client-node' +import { CoreV1Api, KubeConfig, User as k8sUser, V1ObjectReference } from '@kubernetes/client-node' import Debug from 'debug' import { getRegions, ObjectStorageKeyRegions } from '@linode/api-v4' @@ -6,10 +6,21 @@ import { existsSync, rmSync } from 'fs' import { pathExists, unlink } from 'fs-extra' import { readdir, readFile, writeFile } from 'fs/promises' import { generate as generatePassword } from 'generate-password' -import { cloneDeep, filter, isEmpty, map, mapValues, merge, omit, pick, set, unset } from 'lodash' -import { getAppList, getAppSchema, getSpec } from 'src/app' -import { AlreadyExists, ForbiddenError, HttpError, OtomiError, PublicUrlExists, ValidationError } from 'src/error' +import { cloneDeep, filter, get, isEmpty, map, merge, omit, pick, set, unset } from 'lodash' +import { getAppList, getAppSchema, getSecretPaths } from 'src/app' +import { + AlreadyExists, + ForbiddenError, + HttpError, + NotExistError, + OtomiError, + PublicUrlExists, + ValidationError, +} from 'src/error' import getRepo, { getWorktreeRepo, Git } from 'src/git' +import { getFileMaps } from 'src/repo' +import { FileStore } from 'src/fileStore/file-store' +import { getSettingsFileMaps } from 'src/fileStore/file-map' import { cleanSession, getSessionStack } from 'src/middleware' import { AplAgentRequest, @@ -24,19 +35,24 @@ import { AplKnowledgeBaseResponse, AplNetpolRequest, AplNetpolResponse, + AplObject, + AplPlatformObject, AplPolicyRequest, AplPolicyResponse, - AplResponseObject, + AplRecord, AplSecretRequest, AplSecretResponse, AplServiceRequest, AplServiceResponse, + AplTeamObject, AplTeamSettingsRequest, AplTeamSettingsResponse, AplWorkloadRequest, AplWorkloadResponse, App, Build, + buildPlatformObject, + buildTeamObject, Cloudtty, CodeRepo, Core, @@ -46,7 +62,6 @@ import { ObjWizard, Policies, Policy, - Repo, SealedSecret, Service, ServiceSpec, @@ -57,6 +72,8 @@ import { Team, TeamSelfService, TestRepoConnect, + toPlatformObject, + toTeamObject, User, Workload, WorkloadName, @@ -92,8 +109,6 @@ import { import { v4 as uuidv4 } from 'uuid' import { parse as parseYaml, stringify as stringifyYaml } from 'yaml' import { getAIModels } from './ai/aiModelHandler' -import { AkamaiAgentCR } from './ai/AkamaiAgentCR' -import { AkamaiKnowledgeBaseCR } from './ai/AkamaiKnowledgeBaseCR' import { DatabaseCR } from './ai/DatabaseCR' import { apply, @@ -105,9 +120,6 @@ import { k8sdelete, watchPodUntilRunning, } from './k8s_operations' -import { getFileMaps, loadValues } from './repo' -import { RepoService } from './services/RepoService' -import { TeamConfigService } from './services/TeamConfigService' import { getGiteaRepoUrls, getPrivateRepoBranches, @@ -117,10 +129,11 @@ import { testPublicRepoConnect, } from './utils/codeRepoUtils' import { getAplObjectFromV1, getV1MergeObject, getV1ObjectFromApl } from './utils/manifests' -import { getSealedSecretsPEM, sealedSecretManifest, SealedSecretManifestType } from './utils/sealedSecretUtils' +import { getSealedSecretsPEM, sealedSecretManifest } from './utils/sealedSecretUtils' import { getKeycloakUsers, isValidUsername } from './utils/userUtils' import { ObjectStorageClient } from './utils/wizardUtils' import { fetchChartYaml, fetchWorkloadCatalog, NewHelmChartValues, sparseCloneChart } from './utils/workloadUtils' +import { getResourceFilePath, getSecretFilePath } from './fileStore/file-map' interface ExcludedApp extends App { managed: boolean @@ -148,7 +161,6 @@ const env = cleanEnv({ }) export const rootPath = '/tmp/otomi/values' -//TODO Move this to the repo.ts const clusterSettingsFilePath = 'env/settings/cluster.yaml' function getTeamSealedSecretsValuesFilePath(teamId: string, sealedSecretsName: string): string { @@ -167,10 +179,6 @@ function getTeamKnowledgeBaseValuesFilePath(teamId: string, knowledgeBaseName: s return `env/teams/${teamId}/knowledgebases/${knowledgeBaseName}` } -function getTeamAgentValuesFilePath(teamId: string, agentName: string): string { - return `env/teams/${teamId}/agents/${agentName}` -} - function getTeamDatabaseValuesFilePath(teamId: string, databaseName: string): string { return `env/teams/${teamId}/databases/${databaseName}` } @@ -181,7 +189,7 @@ export default class OtomiStack { sessionId?: string isLoaded = false git: Git - repoService: RepoService + fileStore: FileStore constructor(editor?: string, sessionId?: string) { this.editor = editor @@ -218,109 +226,13 @@ export default class OtomiStack { } } - transformApps(appsObj: Record): App[] { - if (!appsObj || typeof appsObj !== 'object') return [] - - return Object.entries(appsObj).map(([appId, appData]) => { - // Retrieve schema to check if the `enabled` flag should be considered - const appSchema = getAppSchema(appId) - const isEnabled = appSchema?.properties?.enabled ? !!appData.enabled : undefined - - return { - id: appId, - enabled: isEnabled, - values: omit(appData, ['enabled']), - rawValues: {}, - } - }) - } - - transformPolicies(teamId: string, policies: Record): AplPolicyResponse[] { - return Object.entries(policies).map(([name, policy]) => ({ - kind: 'AplTeamPolicy', - metadata: { - name, - labels: { - 'apl.io/teamId': teamId, - }, - }, - spec: policy, - status: {}, - })) - } - - transformSecrets(teamId: string, secrets: SealedSecretManifestType[]): AplSecretResponse[] { - return secrets.map((secret) => { - const { annotations, labels, finalizers } = secret.spec.template?.metadata || {} - return { - kind: 'AplTeamSecret', - metadata: { - name: secret.metadata.name, - labels: { - 'apl.io/teamId': teamId, - }, - }, - spec: { - encryptedData: secret.spec.encryptedData, - type: secret.spec.template?.type || 'kubernetes.io/opaque', - immutable: secret.spec.template?.immutable ?? false, - namespace: secret.spec.template?.metadata?.namespace, - metadata: { - ...(!isEmpty(annotations) && { annotations }), - ...(!isEmpty(labels) && { labels }), - ...(!isEmpty(finalizers) && { finalizers }), - }, - }, - status: {}, - } - }) - } - - transformWorkloads(workloads: AplWorkloadResponse[], files: string[]): AplWorkloadResponse[] { - return workloads.map((workload) => { - const workloadName = workload.metadata.name - const teamId = workload.metadata.labels?.['apl.io/teamId'] - - const filePath = getTeamWorkloadValuesFilePath(teamId, workloadName) - return merge(workload, { spec: { values: files[filePath] } }) - }) - } - - transformTeamSettings(teamSettings: Team) { - if (teamSettings.id && !teamSettings.name) { - // eslint-disable-next-line no-param-reassign - teamSettings.name = teamSettings.id - } - // Always allow Alertmanager and Grafana for team Admin - if (teamSettings.name === 'admin' && teamSettings.managedMonitoring) { - // eslint-disable-next-line no-param-reassign - teamSettings.managedMonitoring.alertmanager = true - // eslint-disable-next-line no-param-reassign - teamSettings.managedMonitoring.grafana = true - } - return teamSettings - } - - async initRepo(repoService?: RepoService): Promise { - if (repoService) { - this.repoService = repoService + async initRepo(existingStore?: FileStore): Promise { + if (existingStore) { + this.fileStore = existingStore return - } else { - // We need to map the app values, so it adheres the App interface - const rawRepo = await loadValues(this.getRepoPath()) - - rawRepo.apps = this.transformApps(rawRepo.apps) - rawRepo.teamConfig = mapValues(rawRepo.teamConfig, (teamConfig, teamName) => ({ - ...omit(teamConfig, 'workloadValues'), - apps: this.transformApps(teamConfig.apps), - policies: this.transformPolicies(teamName, teamConfig.policies || {}), - sealedsecrets: this.transformSecrets(teamName, teamConfig.sealedsecrets || []), - workloads: this.transformWorkloads(teamConfig.workloads || [], rawRepo.files || {}), - settings: this.transformTeamSettings(teamConfig.settings), - })) - const repo = rawRepo as Repo - this.repoService = new RepoService(repo) } + // Load all files into the in-memory store + this.fileStore = await FileStore.load(this.getRepoPath()) } async initGit(inflateValues = true): Promise { @@ -368,43 +280,19 @@ export default class OtomiStack { const worktreePath = this.getRepoPath() this.git = await getWorktreeRepo(mainRepo, worktreePath, env.GIT_BRANCH) + this.fileStore = new FileStore() debug(`Worktree created for ${this.editor} in ${this.sessionId}`) } - getSecretPaths(): string[] { - // we split secrets from plain data, but have to overcome teams using patternproperties - const teamProp = 'teamConfig.patternProperties.^[a-z0-9]([-a-z0-9]*[a-z0-9])+$' - const teams = this.getTeams().map(({ name }) => name) - const cleanSecretPaths: string[] = [] - const { secretPaths } = getSpec() - secretPaths.map((p) => { - if (p.indexOf(teamProp) === -1 && !cleanSecretPaths.includes(p)) { - cleanSecretPaths.push(p) - } else { - teams.forEach((teamId: string) => { - if (p.indexOf(teamProp) === 0) { - cleanSecretPaths.push( - p - .replace(teamProp, `teamConfig.${teamId}`) - // add spec to the path for v2 endpoints - .replace(`teamConfig.${teamId}.settings`, `teamConfig.${teamId}.settings.spec`), - ) - } - }) - } - }) - // debug('secretPaths: ', cleanSecretPaths) - return cleanSecretPaths - } - getSettingsInfo(): SettingsInfo { + const settings = this.getSettings(['cluster', 'dns', 'otomi', 'smtp', 'ingress']) return { - cluster: pick(this.repoService.getCluster(), ['name', 'domainSuffix', 'apiServer', 'provider', 'linode']), - dns: pick(this.repoService.getDns(), ['zones']), - otomi: pick(this.repoService.getOtomi(), ['hasExternalDNS', 'hasExternalIDP', 'isPreInstalled', 'aiEnabled']), - smtp: pick(this.repoService.getSmtp(), ['smarthost']), - ingressClassNames: map(this.repoService.getIngress()?.classes, 'className') ?? [], + cluster: pick(settings.cluster, ['name', 'domainSuffix', 'apiServer', 'provider', 'linode']), + dns: pick(settings.dns, ['zones']), + otomi: pick(settings.otomi, ['hasExternalDNS', 'hasExternalIDP', 'isPreInstalled', 'aiEnabled']), + smtp: pick(settings.smtp, ['smarthost']), + ingressClassNames: map(settings.ingress?.classes, 'className') ?? [], } as SettingsInfo } @@ -487,22 +375,52 @@ export default class OtomiStack { } getSettings(keys?: string[]): Settings { - const settings = this.repoService.getSettings() - - if (keys?.includes('otomi')) { - const nodeSelector = settings.otomi?.nodeSelector - // convert otomi.nodeSelector to array of objects - if (!Array.isArray(nodeSelector)) { - const nodeSelectorArray = Object.entries(nodeSelector || {}).map(([name, value]) => ({ - name, - value, - })) - set(settings, 'otomi.nodeSelector', nodeSelectorArray) + const settings: Settings = {} + const settingsFileMaps = getSettingsFileMaps(this.getRepoPath()) + + // Early return: if specific keys requested, only fetch those + if (keys && keys.length > 0) { + keys.forEach((key) => { + const fileMap = settingsFileMaps.get(key) + if (!fileMap) return // Skip unknown keys + + const files = this.fileStore.getByKind(fileMap.kind) + for (const [, content] of files) { + settings[key] = content?.spec || content + } + }) + + // Apply otomi nodeSelector transformation if needed + if (keys.includes('otomi')) { + this.transformOtomiNodeSelector(settings) } + + return settings } - if (!keys) return settings - return pick(settings, keys) as Settings + // No keys specified: fetch all settings + for (const [name, fileMap] of settingsFileMaps.entries()) { + const files = this.fileStore.getByKind(fileMap.kind) + for (const [, content] of files) { + settings[name] = content?.spec || content + } + } + + // Apply otomi nodeSelector transformation + this.transformOtomiNodeSelector(settings) + + return settings + } + + private transformOtomiNodeSelector(settings: Settings): void { + const nodeSelector = settings.otomi?.nodeSelector + if (!Array.isArray(nodeSelector)) { + const nodeSelectorArray = Object.entries(nodeSelector || {}).map(([name, value]) => ({ + name, + value, + })) + set(settings, 'otomi.nodeSelector', nodeSelectorArray) + } } async loadIngressApps(id: string): Promise { @@ -510,8 +428,11 @@ export default class OtomiStack { debug(`Loading ingress apps for ${id}`) const content = await this.git.loadConfig('env/apps/ingress-nginx.yaml', 'env/apps/secrets.ingress-nginx.yaml') const values = content?.apps?.['ingress-nginx'] ?? {} - const teamId = 'admin' - this.repoService.getTeamConfigService(teamId).createApp({ enabled: true, values, rawValues: {}, id }) + + const filePath = getResourceFilePath('AplApp', id) + const aplApp = toPlatformObject('AplApp', id, { enabled: true, rawValues: {}, ...values }) + this.fileStore.set(filePath, aplApp) + debug(`Ingress app loaded for ${id}`) } catch (error) { debug(`Failed to load ingress apps for ${id}:`) @@ -521,10 +442,11 @@ export default class OtomiStack { async removeIngressApps(id: string): Promise { try { debug(`Removing ingress apps for ${id}`) - const path = `env/apps/${id}.yaml` + const filePath = `env/apps/${id}.yaml` const secretsPath = `env/apps/secrets.${id}.yaml` - this.repoService.deleteApp(id) - await this.git.removeFile(path) + + this.fileStore.delete(filePath) + await this.git.removeFile(filePath) await this.git.removeFile(secretsPath) debug(`Ingress app removed for ${id}`) } catch (error) { @@ -555,7 +477,7 @@ export default class OtomiStack { } async editSettings(data: Settings, settingId: string): Promise { - const settings = this.repoService.getSettings() + const settings = this.getSettings() await this.editIngressApps(settings, data, settingId) const updatedSettingsData: any = { ...data } // Preserve the otomi.adminPassword when editing otomi settings @@ -575,11 +497,20 @@ export default class OtomiStack { } settings[settingId] = removeBlankAttributes(updatedSettingsData[settingId] as Record) - this.repoService.updateSettings(settings) + + const settingKindMap = getSettingsFileMaps(this.getRepoPath()) + const kind = settingKindMap.get(settingId) + if (!kind) { + throw new Error(`Unknown settingId ${settingId}`) + } + const filePath = getResourceFilePath(kind.kind, settingId) + const spec = settings[settingId] + const aplObject = toPlatformObject(kind.kind, settingId, spec) + + this.fileStore.set(filePath, aplObject) + await this.saveSettings() - await this.doRepoDeployment((repoService) => { - repoService.updateSettings(settings) - }) + await this.doDeployment({ filePath, content: aplObject }) return settings } @@ -613,17 +544,27 @@ export default class OtomiStack { } getApp(name: string): App { - return this.repoService.getApp(name) + const filePath = getResourceFilePath('AplApp', name) + const content = this.fileStore.get(filePath) + + if (!content) { + throw new Error(`App ${name} not found`) + } + + return { values: content.spec, id: content.metadata.name } as App } getApps(teamId: string, picks?: string[]): Array { const appList = this.getAppList() - const apps = this.repoService.getApps().filter((app) => appList.includes(app.id)) - const providerSpecificApps = this.filterExcludedApp(apps) as App[] + + const allApps = appList.map((id) => { + return this.getApp(id) + }) + + const providerSpecificApps = this.filterExcludedApp(allApps) as App[] if (teamId === 'admin') return providerSpecificApps - // If not team admin load available teamApps const core = this.getCore() let teamApps = providerSpecificApps .map((app: App) => { @@ -636,9 +577,7 @@ export default class OtomiStack { if (!picks) return teamApps if (picks.includes('enabled')) { - const adminApps = this.repoService.getApps() - - teamApps = adminApps.map((adminApp) => { + teamApps = allApps.map((adminApp) => { const teamApp = teamApps.find((app) => app.id === adminApp.id) return teamApp || { id: adminApp.id, enabled: adminApp.enabled } }) @@ -647,16 +586,18 @@ export default class OtomiStack { return teamApps.map((app) => pick(app, picks)) as Array } - async editApp(teamId: string, id: string, data: App): Promise { - let app: App = this.repoService.getApp(id) + async editTeamApp(teamId: string, id: string, data: App): Promise { + let app: App = this.getApp(id) // Shallow merge, so only first level attributes can be replaced (values, rawValues, etc.) app = { ...app, ...data } - app = this.repoService.updateApp(id, app) + + const filePath = getResourceFilePath('AplApp', id) + const aplApp = toPlatformObject('AplApp', id, { enabled: app.enabled, ...app.values }) + this.fileStore.set(filePath, aplApp) + await this.saveAdminApp(app) - await this.doRepoDeployment((repoService) => { - repoService.updateApp(id, app) - }) - return this.repoService.getApp(id) + await this.doDeployment({ filePath, content: aplApp }) + return this.getApp(id) } canToggleApp(id: string): boolean { @@ -665,38 +606,73 @@ export default class OtomiStack { } async toggleApps(teamId: string, ids: string[], enabled: boolean): Promise { - await Promise.all( - ids.map(async (id) => { - const orig = this.repoService.getApp(id) - if (orig && this.canToggleApp(id)) { - const app = this.repoService.updateApp(id, { enabled }) - await this.saveAdminApp(app) - } - }), - ) + const aplRecords = ( + await Promise.all( + ids.map(async (id) => { + const orig = this.getApp(id) + if (orig && this.canToggleApp(id)) { + const filePath = getResourceFilePath('AplApp', id) + const aplApp = toPlatformObject('AplApp', id, { enabled, ...orig.values }) + this.fileStore.set(filePath, aplApp) + + const app = { ...orig, enabled } + await this.saveAdminApp(app) + return { filePath, content: aplApp } as AplRecord + } + return undefined + }), + ) + ).filter((record): record is AplRecord => record !== undefined) - await this.doRepoDeployment((repoService) => { - ids.map((id) => { - const orig = repoService.getApp(id) - if (orig && this.canToggleApp(id)) { - repoService.updateApp(id, { enabled }) - } - }) - }) + if (aplRecords.length === 0) { + throw new Error(`Failed toggling apps ${ids.toString()}`) + } + await this.doDeployments(aplRecords) } getTeams(): Array { - return this.repoService.getAllTeamSettings().map((team) => getV1ObjectFromApl(team) as Team) + const teams: Team[] = [] + const teamIds = this.fileStore.getTeamIds() + + for (const teamId of teamIds) { + const settingsFiles = this.fileStore.getByKind('AplTeamSettingSet', teamId) + for (const [, content] of settingsFiles) { + // v1 format: return spec directly + const team = getV1ObjectFromApl(content as AplTeamSettingsResponse) as Team + if (team) { + team.name = team.name || teamId + teams.push(team as Team) + } + } + } + + return teams + } + + getTeamIds(): string[] { + return this.fileStore.getTeamIds() } getAplTeams(): AplTeamSettingsResponse[] { - return this.repoService - .getAllTeamSettings() - .filter((t) => t.metadata.name !== 'admin') - .map(({ spec, ...rest }) => ({ - ...rest, - spec: { ...spec, password: undefined }, - })) + const teams: AplTeamSettingsResponse[] = [] + const teamIds = this.fileStore.getTeamIds() + + for (const teamId of teamIds) { + if (teamId === 'admin') continue + const settingsFiles = this.fileStore.getByKind('AplTeamSettingSet', teamId) + for (const [, content] of settingsFiles) { + if (content) { + // Return full v2 object with password removed + const team = { ...content } + if (team.spec) { + team.spec = { ...team.spec, password: undefined } + } + teams.push(team as AplTeamSettingsResponse) + } + } + } + + return teams } getTeamSelfServiceFlags(id: string): TeamSelfService { @@ -709,26 +685,35 @@ export default class OtomiStack { } getTeam(name: string): Team { - const team = getV1ObjectFromApl(this.repoService.getTeamConfigService(name).getSettings()) as Team + const settingsResponse = this.fileStore.getTeamResource('AplTeamSettingSet', name, 'settings') + if (!settingsResponse) { + throw new Error(`Team ${name} not found`) + } + const team = getV1ObjectFromApl(settingsResponse as AplTeamSettingsResponse) as Team + team.name = team.name || name unset(team, 'password') // Remove password from the response return team } getAplTeam(name: string): AplTeamSettingsResponse { - const team = this.repoService.getTeamConfigService(name).getSettings() - unset(team, 'spec.password') // Remove password from the response - return team + const settingsResponse = this.fileStore.getTeamResource( + 'AplTeamSettingSet', + name, + 'settings', + ) as AplTeamSettingsResponse + if (!settingsResponse) { + throw new Error(`Team ${name} not found`) + } + unset(settingsResponse, 'spec.password') // Remove password from the response + return settingsResponse } - async createTeam(data: Team, deploy = true): Promise { - const newTeam = await this.createAplTeam( - getAplObjectFromV1('AplTeamSettingSet', data) as AplTeamSettingsRequest, - deploy, - ) + async createTeam(data: Team): Promise { + const newTeam = await this.createAplTeam(getAplObjectFromV1('AplTeamSettingSet', data) as AplTeamSettingsRequest) return getV1ObjectFromApl(newTeam) as Team } - async createAplTeam(data: AplTeamSettingsRequest, deploy = true): Promise { + async createAplTeam(data: AplTeamSettingsRequest): Promise { const teamName = data.metadata.name if (teamName.length < 3) throw new ValidationError('Team name must be at least 3 characters long') if (teamName.length > 9) throw new ValidationError('Team name must not exceed 9 characters') @@ -746,30 +731,10 @@ export default class OtomiStack { }) } - const teamConfig = this.repoService.createTeamConfig(data) - const team = teamConfig.settings - const apps = getAppList() - const core = this.getCore() - const teamApps = apps.flatMap((appId) => { - const isShared = !!core.adminApps.find((a) => a.name === appId)?.isShared - const inTeamApps = !!core.teamApps.find((a) => a.name === appId) - return teamName !== 'admin' && (isShared || inTeamApps) - ? [this.repoService.getTeamConfigService(teamName).createApp({ id: appId })] - : [] // Empty array removes `undefined` entries - }) - - if (deploy) { - await this.saveTeam(team) - await this.doRepoDeployment( - (repoService) => { - repoService.createTeamConfig(data) - repoService.getTeamConfigService(teamName).setApps(teamApps) - }, - true, - [`${this.getRepoPath()}/env/teams/${teamName}/secrets.settings.yaml`], - ) - } - return team + const teamObject = toTeamObject(teamName, data) + const team = await this.saveTeam(teamObject) + await this.doDeployment(team, true, [`${this.getRepoPath()}/env/teams/${teamName}/secrets.settings.yaml`]) + return team.content as AplTeamSettingsResponse } async editTeam(name: string, data: Team): Promise { @@ -783,109 +748,102 @@ export default class OtomiStack { data: AplTeamSettingsRequest | DeepPartial, patch = false, ): Promise { - const team = patch - ? this.repoService.getTeamConfigService(name).patchSettings(data) - : this.repoService.getTeamConfigService(name).updateSettings(data) - await this.saveTeam(team) - await this.doTeamDeployment( - name, - (teamService) => { - teamService.updateSettings(team) - }, - true, - [`${this.getRepoPath()}/env/teams/${name}/secrets.settings.yaml`], - ) - return team + const currentTeam = this.getAplTeam(name) + + const updatedSpec = patch ? merge(cloneDeep(currentTeam.spec), data.spec) : { ...currentTeam.spec, ...data.spec } + + const teamObject = buildTeamObject(currentTeam, updatedSpec) + const team = await this.saveTeam(teamObject) + await this.doDeployment(team, true, [`${this.getRepoPath()}/env/teams/${name}/secrets.settings.yaml`]) + return team.content as AplTeamSettingsResponse } async deleteTeam(id: string): Promise { - await this.deleteTeamConfig(id) - await this.doRepoDeployment((repoService) => { - repoService.deleteTeamConfig(id) - }, false) + const filePaths = await this.deleteTeamObjects(id) + await this.doDeleteDeployment(filePaths) } - private getConfigKey(kind: AplKind): string { - return getFileMaps('').find((fm) => fm.kind === kind)!.resourceDir - } + async saveTeamConfigItem(aplTeamObject: AplTeamObject): Promise { + debug( + `Saving ${aplTeamObject.kind} ${aplTeamObject.metadata.name} for team ${aplTeamObject.metadata.labels['apl.io/teamId']}`, + ) + + const filePath = this.fileStore.setTeamResource(aplTeamObject) + await this.git.writeFile(filePath, aplTeamObject) - async saveTeamConfigItem(data: AplResponseObject): Promise { - const { kind, metadata } = data - const teamId = metadata.labels['apl.io/teamId']! - const configKey = this.getConfigKey(kind) - debug(`Saving ${kind} ${metadata.name} for team ${teamId}`) - const repo = this.createTeamConfigInRepo(teamId, configKey, [data]) - const fileMap = getFileMaps('').find((fm) => fm.kind === kind)! - await this.git.saveConfig(repo, fileMap) + return { filePath, content: aplTeamObject } } - async saveTeamWorkload(data: AplWorkloadResponse) { - const { metadata } = data - const teamId = metadata.labels['apl.io/teamId']! - debug(`Saving AplTeamWorkload ${metadata.name} for team ${teamId}`) + async saveTeamWorkload(aplTeamObject: AplTeamObject): Promise { + const teamId = aplTeamObject.metadata.labels['apl.io/teamId'] + debug(`Saving AplTeamWorkload ${aplTeamObject.metadata.name} for team ${teamId}`) + + // Create workload object without values for file storage const workload = { - kind: 'AplTeamWorkload', - metadata: data.metadata, - spec: omit(data.spec, 'values'), - } - const configKey = this.getConfigKey('AplTeamWorkload') - const repo = this.createTeamConfigInRepo(teamId, configKey, [workload]) - const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkload')! - await this.git.saveConfig(repo, fileMap) + ...aplTeamObject, + spec: omit(aplTeamObject.spec, 'values'), + } as AplTeamObject + + const filePath = this.fileStore.setTeamResource(workload) + await this.git.writeFile(filePath, workload) + + return { filePath, content: workload } } - async saveTeamWorkloadValues(data: AplWorkloadResponse, createManagedFile = false) { - const { metadata } = data - const teamId = metadata.labels['apl.io/teamId']! - debug(`Saving AplTeamWorkloadValues ${metadata.name} for team ${teamId}`) - const filePath = getTeamWorkloadValuesFilePath(teamId, metadata.name) - await this.git.writeTextFile(filePath, data.spec.values || '{}') + async saveTeamWorkloadValues( + teamId: string, + name: string, + values: string, + createManagedFile = false, + ): Promise { + debug(`Saving AplTeamWorkloadValues ${name} for team ${teamId}`) + //AplTeamWorkloadValues does not adhere the AplObject structure so we set it as any + const aplRecord = this.fileStore.set(getTeamWorkloadValuesFilePath(teamId, name), values as any) + await this.git.writeTextFile(aplRecord.filePath, values) if (createManagedFile) { - const filePathValuesManaged = getTeamWorkloadValuesManagedFilePath(teamId, metadata.name) + const filePathValuesManaged = getTeamWorkloadValuesManagedFilePath(teamId, name) await this.git.writeTextFile(filePathValuesManaged, '') } + return aplRecord } - async saveTeamPolicy(teamId: string, data: AplPolicyResponse): Promise { - debug(`Saving AplTeamPolicy for team ${teamId}`) - const configKey = data.metadata.name - const repo = this.createTeamConfigInRepo(teamId, configKey, data) - const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamPolicy')! - await this.git.saveConfig(repo, fileMap) - } - - async saveTeamSealedSecret(data: AplSecretResponse): Promise { - const { metadata } = data - const teamId = metadata.labels['apl.io/teamId']! - const sealedSecretChartValues = sealedSecretManifest(data) - const relativePath = getTeamSealedSecretsValuesFilePath(teamId, `${metadata.name}.yaml`) + async saveTeamSealedSecret(teamId: string, data: AplSecretRequest): Promise { debug(`Saving sealed secrets of team: ${teamId}`) - // @ts-ignore - await this.git.writeFile(relativePath, sealedSecretChartValues) + const { metadata } = data + const aplObject = toTeamObject(teamId, data) as AplSecretResponse + const sealedSecretChartValues = sealedSecretManifest(aplObject) + const aplRecord = this.fileStore.set( + getTeamSealedSecretsValuesFilePath(teamId, metadata.name), + sealedSecretChartValues, + ) + await this.git.writeFile(aplRecord.filePath, sealedSecretChartValues as any) + + return aplRecord } - async deleteTeamConfigItem(data: AplResponseObject): Promise { - const { kind, metadata } = data - const teamId = metadata.labels['apl.io/teamId']! - const configKey = this.getConfigKey(data.kind) - debug(`Removing ${kind} ${metadata.name} for team ${teamId}`) - const repo = this.createTeamConfigInRepo(teamId, configKey, [data]) - const fileMap = getFileMaps('').find((fm) => fm.kind === kind)! - await this.git.deleteConfig(repo, fileMap) + async deleteTeamConfigItem(kind: AplKind, teamId: string, name: string): Promise { + debug(`Removing ${kind} ${name} for team ${teamId}`) + + const filePath = this.fileStore.deleteTeamResource(kind, teamId, name) + await this.git.removeFile(filePath) + return filePath } - async deleteTeamWorkload(data: AplWorkloadResponse): Promise { - const { metadata } = data - const teamId = metadata.labels['apl.io/teamId']! - debug(`Removing AplWorkload ${metadata.name} for team ${teamId}`) - const repoWorkload = this.createTeamConfigInRepo(teamId, 'workloads', [data]) - const fileMapWorkload = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkload')! - const repoValues = this.createTeamConfigInRepo(teamId, 'workloads', [data]) - const fileMapValues = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkloadValues')! - const filePathValuesManaged = getTeamWorkloadValuesManagedFilePath(teamId, metadata.name) - await this.git.deleteConfig(repoWorkload, fileMapWorkload) - await this.git.deleteConfig(repoValues, fileMapValues) - await this.git.removeFile(filePathValuesManaged) + async deleteTeamWorkload(kind: AplKind, teamId: string, name: string): Promise { + debug(`Removing AplWorkload ${name} for team ${teamId}`) + + // Delete workload file + const workloadFilePath = this.fileStore.deleteTeamResource(kind, teamId, name) + await this.git.removeFile(workloadFilePath) + + // Delete workload values file + const valuesFilePath = getTeamWorkloadValuesFilePath(teamId, name) + await this.git.removeFile(valuesFilePath) + + // Delete managed values file + const managedFilePath = getTeamWorkloadValuesManagedFilePath(teamId, name) + await this.git.removeFile(managedFilePath) + return workloadFilePath } getTeamNetpols(teamId: string): Netpol[] { @@ -893,7 +851,8 @@ export default class OtomiStack { } getTeamAplNetpols(teamId: string): AplNetpolResponse[] { - return this.repoService.getTeamConfigService(teamId).getNetpols() + const files = this.fileStore.getByKind('AplTeamNetworkControl', teamId) + return Array.from(files.values()) as AplNetpolResponse[] } getAllNetpols(): Netpol[] { @@ -901,7 +860,8 @@ export default class OtomiStack { } getAllAplNetpols(): AplNetpolResponse[] { - return this.repoService.getAllNetpols() + const files = this.fileStore.getByKind('AplTeamNetworkControl') + return Array.from(files.values()) as AplNetpolResponse[] } async createNetpol(teamId: string, data: Netpol): Promise { @@ -915,21 +875,14 @@ export default class OtomiStack { async createAplNetpol(teamId: string, data: AplNetpolRequest): Promise { if (data.metadata.name.length < 2) throw new ValidationError('Network policy name must be at least 2 characters long') - try { - const netpol = this.repoService.getTeamConfigService(teamId).createNetpol(data) - await this.saveTeamConfigItem(netpol) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createNetpol(netpol) - }, - false, - ) - return netpol - } catch (err) { - if (err.code === 409) err.publicMessage = 'Network policy name already exists' - throw err + if (this.fileStore.getTeamResource('AplTeamNetworkControl', teamId, data.metadata.name)) { + throw new AlreadyExists('Network policy name already exists') } + + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplNetpolResponse } getNetpol(teamId: string, name: string): Netpol { @@ -938,7 +891,11 @@ export default class OtomiStack { } getAplNetpol(teamId: string, name: string): AplNetpolResponse { - return this.repoService.getTeamConfigService(teamId).getNetpol(name) + const netpol = this.fileStore.getTeamResource('AplTeamNetworkControl', teamId, name) + if (!netpol) { + throw new NotExistError(`Network policy ${name} not found in team ${teamId}`) + } + return netpol as AplNetpolResponse } async editNetpol(teamId: string, name: string, data: Netpol): Promise { @@ -953,35 +910,29 @@ export default class OtomiStack { data: AplNetpolRequest | DeepPartial, patch = false, ): Promise { - const netpol = patch - ? this.repoService.getTeamConfigService(teamId).patchNetpol(name, data) - : this.repoService.getTeamConfigService(teamId).updateNetpol(name, data as AplNetpolRequest) - await this.saveTeamConfigItem(netpol) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateNetpol(name, netpol) - }, - false, - ) - return netpol + const existing = this.getAplNetpol(teamId, name) + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), data.spec) + : ({ ...existing.spec, ...data.spec } as Netpol) + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplNetpolResponse } async deleteNetpol(teamId: string, name: string): Promise { - const netpol = this.repoService.getTeamConfigService(teamId).getNetpol(name) - this.repoService.getTeamConfigService(teamId).deleteNetpol(name) - await this.deleteTeamConfigItem(netpol) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteNetpol(name) - }, - false, - ) + const filePath = await this.deleteTeamConfigItem('AplTeamNetworkControl', teamId, name) + await this.doDeleteDeployment([filePath], false) } getAllUsers(sessionUser: SessionUser): Array { - const users = this.repoService.getUsers() + const files = this.fileStore.getByKind('AplUser') + const aplObjects = Array.from(files.values()) as AplObject[] + const users = aplObjects.map((aplObject) => { + return { ...aplObject.spec, id: aplObject.metadata.name } as User + }) if (sessionUser.isPlatformAdmin) { return users } else if (sessionUser.isTeamAdmin) { @@ -1007,8 +958,13 @@ export default class OtomiStack { uppercase: true, strict: true, }) - const user = { ...data, initialPassword } - let existingUsersEmail = this.repoService.getUsersEmail() + const userId = uuidv4() + const user: User = { ...data, id: userId, initialPassword } + + // Get existing users' emails + const files = this.fileStore.getByKind('AplUser') + let existingUsersEmail = Array.from(files.values()).map((aplObject: AplObject) => aplObject.spec.email) + if (!env.isDev) { const { otomi, cluster } = this.getSettings(['otomi', 'cluster']) const keycloak = this.getApp('keycloak') @@ -1018,34 +974,28 @@ export default class OtomiStack { const password = otomi?.adminPassword as string existingUsersEmail = await getKeycloakUsers(keycloakBaseUrl, realm, username, password) } - try { - if (existingUsersEmail.some((existingUser) => existingUser === user.email)) { - throw new AlreadyExists('User email already exists') - } - const createdUser = this.repoService.createUser(user) - await this.saveUser(createdUser) - await this.doRepoDeployment( - (repoService) => { - repoService.createUser(createdUser) - }, - true, - [`${this.getRepoPath()}/env/users/secrets.${createdUser.id}.yaml`], - ) - return createdUser - } catch (err) { - if (err.code === 409) err.publicMessage = 'User email already exists' - throw err + if (existingUsersEmail.some((existingUser) => existingUser === user.email)) { + throw new AlreadyExists('User email already exists') } + + const aplRecord = await this.saveUser(user) + await this.doDeployment(aplRecord, true, [`${this.getRepoPath()}/env/users/secrets.${userId}.yaml`]) + return user } getUser(id: string, sessionUser: SessionUser): User { - const user = this.repoService.getUser(id) + const filePath = getResourceFilePath('AplUser', id) + const user = this.fileStore.get(filePath) + if (!user) { + throw new NotExistError(`User ${id} not found`) + } + if (sessionUser.isPlatformAdmin) { - return user + return { ...user.spec, id } as User } if (sessionUser.isTeamAdmin) { - const { id: userId, email, isPlatformAdmin, isTeamAdmin, teams } = user - return { id: userId, email, isPlatformAdmin, isTeamAdmin, teams } as User + const { email, isPlatformAdmin, isTeamAdmin, teams } = user.spec + return { id, email, isPlatformAdmin, isTeamAdmin, teams } as User } throw new ForbiddenError() } @@ -1054,27 +1004,33 @@ export default class OtomiStack { if (!sessionUser.isPlatformAdmin) { throw new ForbiddenError('Only platform admins can modify user details.') } - const user = this.repoService.updateUser(id, data) - await this.saveUser(user) - await this.doRepoDeployment( - (repoService) => { - repoService.updateUser(id, user) - }, - true, - [`${this.getRepoPath()}/env/users/secrets.${user.id}.yaml`], - ) + + const filePath = getResourceFilePath('AplUser', id) + const existing = this.fileStore.get(filePath) + if (!existing) { + throw new NotExistError(`User ${id} not found`) + } + + const user: User = { ...existing, ...data, id } + + const aplRecord = await this.saveUser(user) + await this.doDeployment(aplRecord, true, [`${this.getRepoPath()}/env/users/secrets.${id}.yaml`]) return user } async deleteUser(id: string): Promise { - const user = this.repoService.getUser(id) + const filePath = getResourceFilePath('AplUser', id) + const aplObject = this.fileStore.get(filePath) + if (!aplObject) { + throw new NotExistError(`User ${id} not found`) + } + const user = aplObject.spec as User if (user.email === env.DEFAULT_PLATFORM_ADMIN_EMAIL) { throw new ForbiddenError('Cannot delete the default platform admin user') } + await this.deleteUserFile(user) - await this.doRepoDeployment((repoService) => { - repoService.deleteUser(user.email) - }, false) + await this.doDeleteDeployment([filePath], false) } private canTeamAdminUpdateUserTeams(sessionUser: SessionUser, existingUser: User, updatedUserTeams: string[]) { @@ -1112,32 +1068,42 @@ export default class OtomiStack { if (!sessionUser.isPlatformAdmin && !sessionUser.isTeamAdmin) { throw new ForbiddenError("Only platform admins or team admins can modify a user's team memberships.") } - for (const user of data) { - const existingUser = this.repoService.getUser(user.id!) + + const secretFiles: string[] = [] + const aplRecords: AplRecord[] = [] + + for (const userData of data) { + if (!userData.id) { + throw new NotExistError(`User ${userData.id} not found`) + } + const filePath = getResourceFilePath('AplUser', userData.id) + const aplObject = this.fileStore.get(filePath) + if (!aplObject) { + throw new NotExistError(`User ${userData.id} not found`) + } + const existingUser = aplObject.spec as User + if ( !sessionUser.isPlatformAdmin && - !this.canTeamAdminUpdateUserTeams(sessionUser, existingUser, user.teams as string[]) + !this.canTeamAdminUpdateUserTeams(sessionUser, existingUser, userData.teams as string[]) ) { throw new ForbiddenError( 'Team admins are permitted to add or remove users only within the teams they manage. However, they cannot remove themselves or other team admins from those teams.', ) } - const updateUser = this.repoService.updateUser(user.id!, { ...existingUser, teams: user.teams }) - await this.saveUser(updateUser) - } - const repoUsers = this.repoService.getUsers() - const files = repoUsers.map((user) => `${this.getRepoPath()}/env/users/secrets.${user.id}.yaml`) - await this.doRepoDeployment( - (repoService) => { - for (const user of data) { - const existingUser = repoService.getUser(user.id!) - repoService.updateUser(user.id!, { ...existingUser, teams: user.teams }) - } - }, - true, - files, - ) - const users = repoUsers.map((user) => ({ id: user.id, teams: user.teams || [] })) + + const updatedUser: User = { ...existingUser, teams: userData.teams } + const aplRecord = await this.saveUser(updatedUser) + secretFiles.push(`${this.getRepoPath()}/env/users/secrets.${userData.id}.yaml`) + aplRecords.push(aplRecord) + } + + await this.doDeployments(aplRecords, true, secretFiles) + + const users = aplRecords.map((aplRecord: AplRecord) => ({ + id: aplRecord.content.spec.id, + teams: aplRecord.content.spec.teams || [], + })) return users } @@ -1146,7 +1112,8 @@ export default class OtomiStack { } getTeamAplCodeRepos(teamId: string): AplCodeRepoResponse[] { - return this.repoService.getTeamConfigService(teamId).getCodeRepos() + const files = this.fileStore.getByKind('AplTeamCodeRepo', teamId) + return Array.from(files.values()) as AplCodeRepoResponse[] } getAllCodeRepos(): CodeRepo[] { @@ -1154,7 +1121,8 @@ export default class OtomiStack { } getAllAplCodeRepos(): AplCodeRepoResponse[] { - return this.repoService.getAllCodeRepos() + const files = this.fileStore.getByKind('AplTeamCodeRepo') + return Array.from(files.values()) as AplCodeRepoResponse[] } async createCodeRepo(teamId: string, data: CodeRepo): Promise { @@ -1166,31 +1134,19 @@ export default class OtomiStack { } async createAplCodeRepo(teamId: string, data: AplCodeRepoRequest): Promise { - const allRepoUrls = - this.repoService - .getTeamConfigService(teamId) - .getCodeRepos() - .map((repo) => repo.spec.repositoryUrl) || [] + // Check if URL already exists + const existingRepos = this.getTeamAplCodeRepos(teamId) + const allRepoUrls = existingRepos.map((repo) => repo.spec.repositoryUrl) || [] if (allRepoUrls.includes(data.spec.repositoryUrl)) throw new AlreadyExists('Code repository URL already exists') + const allNames = existingRepos.map((repo) => repo.metadata.name) || [] + if (allNames.includes(data.metadata.name)) throw new AlreadyExists('Code repo name already exists') if (!data.spec.private) unset(data.spec, 'secret') if (data.spec.gitService === 'gitea') unset(data.spec, 'private') - try { - const codeRepo = this.repoService.getTeamConfigService(teamId).createCodeRepo(data) - await this.saveTeamConfigItem(codeRepo) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createCodeRepo(codeRepo) - }, - false, - ) - return codeRepo - } catch (err) { - if (err.code === 409) { - err.publicMessage = 'Code repo name already exists' - } - throw err - } + + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplCodeRepoResponse } getCodeRepo(teamId: string, name: string): CodeRepo { @@ -1198,7 +1154,11 @@ export default class OtomiStack { } getAplCodeRepo(teamId: string, name: string): AplCodeRepoResponse { - return this.repoService.getTeamConfigService(teamId).getCodeRepo(name) + const codeRepo = this.fileStore.getTeamResource('AplTeamCodeRepo', teamId, name) + if (!codeRepo) { + throw new NotExistError(`Code repo ${name} not found in team ${teamId}`) + } + return codeRepo as AplCodeRepoResponse } async editCodeRepo(teamId: string, name: string, data: CodeRepo): Promise { @@ -1215,31 +1175,20 @@ export default class OtomiStack { ): Promise { if (!data.spec?.private) unset(data.spec, 'secret') if (data.spec?.gitService === 'gitea') unset(data.spec, 'private') - const codeRepo = patch - ? this.repoService.getTeamConfigService(teamId).patchCodeRepo(name, data) - : this.repoService.getTeamConfigService(teamId).updateCodeRepo(name, data as AplCodeRepoRequest) - await this.saveTeamConfigItem(codeRepo) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateCodeRepo(name, codeRepo) - }, - false, - ) - return codeRepo + + const existing = this.getAplCodeRepo(teamId, name) + const updatedSpec = patch ? merge(cloneDeep(existing.spec), data.spec) : { ...existing.spec, ...data.spec } + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplCodeRepoResponse } async deleteCodeRepo(teamId: string, name: string): Promise { - const codeRepo = this.repoService.getTeamConfigService(teamId).getCodeRepo(name) - this.repoService.getTeamConfigService(teamId).deleteCodeRepo(name) - await this.deleteTeamConfigItem(codeRepo) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteCodeRepo(name) - }, - false, - ) + const filePath = await this.deleteTeamConfigItem('AplTeamCodeRepo', teamId, name) + await this.doDeleteDeployment([filePath], false) } async getRepoBranches(codeRepoName: string, teamId: string): Promise { @@ -1310,8 +1259,8 @@ export default class OtomiStack { if (env.isDev || !teamId || teamId === 'admin') return [] const { cluster, otomi } = this.getSettings(['cluster', 'otomi']) const gitea = this.getApp('gitea') - const username = gitea?.values?.adminUsername as string - const password = (gitea?.values?.adminPassword as string) || (otomi?.adminPassword as string) + const username = (gitea?.values?.adminUsername ?? '') as string + const password = (gitea?.values?.adminPassword ?? otomi?.adminPassword ?? '') as string const orgName = `team-${teamId}` const domainSuffix = cluster?.domainSuffix const internalRepoUrls = (await getGiteaRepoUrls(username, password, orgName, domainSuffix)) || [] @@ -1319,14 +1268,12 @@ export default class OtomiStack { } getDashboard(teamName: string): Array { - const codeRepos = teamName ? this.repoService.getTeamConfigService(teamName).getCodeRepos() : this.getAllCodeRepos() - const builds = teamName ? this.repoService.getTeamConfigService(teamName).getBuilds() : this.getAllBuilds() - const workloads = teamName ? this.repoService.getTeamConfigService(teamName).getWorkloads() : this.getAllWorkloads() - const services = teamName ? this.repoService.getTeamConfigService(teamName).getServices() : this.getAllServices() - const secrets = teamName - ? this.repoService.getTeamConfigService(teamName).getSealedSecrets() - : this.getAllSealedSecrets() - const netpols = teamName ? this.repoService.getTeamConfigService(teamName).getNetpols() : this.getAllNetpols() + const codeRepos = teamName ? this.getTeamAplCodeRepos(teamName) : this.getAllCodeRepos() + const builds = teamName ? this.getTeamAplBuilds(teamName) : this.getAllBuilds() + const workloads = teamName ? this.getTeamAplWorkloads(teamName) : this.getAllWorkloads() + const services = teamName ? this.getTeamAplServices(teamName) : this.getAllServices() + const secrets = teamName ? this.getAplSealedSecrets(teamName) : this.getAllAplSealedSecrets() + const netpols = teamName ? this.getTeamAplNetpols(teamName) : this.getAllNetpols() return [ { name: 'code-repositories', count: codeRepos?.length }, @@ -1343,7 +1290,8 @@ export default class OtomiStack { } getTeamAplBuilds(teamId: string): AplBuildResponse[] { - return this.repoService.getTeamConfigService(teamId).getBuilds() + const files = this.fileStore.getByKind('AplTeamBuild', teamId) + return Array.from(files.values()) as AplBuildResponse[] } getAllBuilds(): Build[] { @@ -1351,7 +1299,8 @@ export default class OtomiStack { } getAllAplBuilds(): AplBuildResponse[] { - return this.repoService.getAllBuilds() + const files = this.fileStore.getByKind('AplTeamBuild') + return Array.from(files.values()) as AplBuildResponse[] } async createBuild(teamId: string, data: Build): Promise { @@ -1369,23 +1318,14 @@ export default class OtomiStack { 'Invalid container image name, the combined image name and tag must not exceed 128 characters.', ) } - try { - const build = this.repoService.getTeamConfigService(teamId).createBuild(data) - await this.saveTeamConfigItem(build) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createBuild(build) - }, - false, - ) - return build - } catch (err) { - if (err.code === 409) { - err.publicMessage = 'Container image name already exists, the combined image name and tag must be unique.' - } - throw err + if (this.fileStore.getTeamResource('AplTeamBuild', teamId, data.metadata.name)) { + throw new AlreadyExists('Container image name already exists') } + + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplBuildResponse } getBuild(teamId: string, name: string): Build { @@ -1393,7 +1333,11 @@ export default class OtomiStack { } getAplBuild(teamId: string, name: string): AplBuildResponse { - return this.repoService.getTeamConfigService(teamId).getBuild(name) + const build = this.fileStore.getTeamResource('AplTeamBuild', teamId, name) + if (!build) { + throw new NotExistError(`Build ${name} not found in team ${teamId}`) + } + return build as AplBuildResponse } async editBuild(teamId: string, name: string, data: Build): Promise { @@ -1408,31 +1352,22 @@ export default class OtomiStack { data: AplBuildRequest | DeepPartial, patch = false, ): Promise { - const build = patch - ? this.repoService.getTeamConfigService(teamId).patchBuild(name, data) - : this.repoService.getTeamConfigService(teamId).updateBuild(name, data as AplBuildRequest) - await this.saveTeamConfigItem(build) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateBuild(name, build) - }, - false, - ) - return build + const existing = this.getAplBuild(teamId, name) + + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), data.spec) + : ({ ...existing.spec, ...data.spec } as Build) + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplBuildResponse } async deleteBuild(teamId: string, name: string): Promise { - const build = this.repoService.getTeamConfigService(teamId).getBuild(name) - this.repoService.getTeamConfigService(teamId).deleteBuild(name) - await this.deleteTeamConfigItem(build) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteBuild(name) - }, - false, - ) + const filePath = await this.deleteTeamConfigItem('AplTeamBuild', teamId, name) + await this.doDeleteDeployment([filePath], false) } getTeamPolicies(teamId: string): Policies { @@ -1444,19 +1379,22 @@ export default class OtomiStack { } getTeamAplPolicies(teamId: string): AplPolicyResponse[] { - return this.repoService.getTeamConfigService(teamId).getPolicies() + const files = this.fileStore.getByKind('AplTeamPolicy', teamId) + return Array.from(files.values()) as AplPolicyResponse[] } getAllPolicies(): Record { const teamPolicies: Record = {} - this.repoService.getTeamIds().forEach((teamId) => { + const teamIds = this.fileStore.getTeamIds() + teamIds.forEach((teamId) => { teamPolicies[teamId] = this.getTeamPolicies(teamId) }) return teamPolicies } getAllAplPolicies(): AplPolicyResponse[] { - return this.repoService.getAllPolicies() + const files = this.fileStore.getByKind('AplTeamPolicy') + return Array.from(files.values()) as AplPolicyResponse[] } getPolicy(teamId: string, id: string): Policy { @@ -1464,7 +1402,11 @@ export default class OtomiStack { } getAplPolicy(teamId: string, id: string): AplPolicyResponse { - return this.repoService.getTeamConfigService(teamId).getPolicy(id) + const policy = this.fileStore.getTeamResource('AplTeamPolicy', teamId, id) + if (!policy) { + throw new NotExistError(`Policy ${id} not found in team ${teamId}`) + } + return policy as AplPolicyResponse } async editPolicy(teamId: string, policyId: string, data: Policy): Promise { @@ -1482,18 +1424,16 @@ export default class OtomiStack { data: AplPolicyRequest | DeepPartial, patch = false, ): Promise { - const policy = patch - ? this.repoService.getTeamConfigService(teamId).patchPolicies(policyId, data) - : this.repoService.getTeamConfigService(teamId).updatePolicies(policyId, data as AplPolicyRequest) - await this.saveTeamPolicy(teamId, policy) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updatePolicies(policyId, policy) - }, - false, - ) - return policy + const existing = this.getAplPolicy(teamId, policyId) + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), data.spec) + : ({ ...existing.spec, ...data.spec } as Policy) + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplPolicyResponse } async getK8sVersion(): Promise { @@ -1662,7 +1602,8 @@ export default class OtomiStack { } getTeamAplWorkloads(teamId: string): AplWorkloadResponse[] { - return this.repoService.getTeamConfigService(teamId).getWorkloads() + const files = this.fileStore.getByKind('AplTeamWorkload', teamId) + return Array.from(files.values()) as AplWorkloadResponse[] } getAllWorkloads(): Workload[] { @@ -1683,7 +1624,8 @@ export default class OtomiStack { } getAllAplWorkloads(): AplWorkloadResponse[] { - return this.repoService.getAllWorkloads() + const files = this.fileStore.getByKind('AplTeamWorkload') + return Array.from(files.values()) as AplWorkloadResponse[] } async createWorkload(teamId: string, data: Workload): Promise { @@ -1695,22 +1637,16 @@ export default class OtomiStack { } async createAplWorkload(teamId: string, data: AplWorkloadRequest): Promise { - try { - const workload = this.repoService.getTeamConfigService(teamId).createWorkload(data) - await this.saveTeamWorkload(workload) - await this.saveTeamWorkloadValues(workload, true) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createWorkload(workload) - }, - false, - ) - return workload - } catch (err) { - if (err.code === 409) err.publicMessage = 'Workload name already exists' - throw err + if (this.fileStore.getTeamResource('AplTeamWorkload', teamId, data.metadata.name)) { + throw new AlreadyExists('Workload name already exists') } + + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamWorkload(teamObject) + + await this.saveTeamWorkloadValues(teamId, data.metadata.name, data.spec.values || '{}', true) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplWorkloadResponse } getWorkload(teamId: string, name: string): Workload { @@ -1719,7 +1655,11 @@ export default class OtomiStack { } getAplWorkload(teamId: string, name: string): AplWorkloadResponse { - return this.repoService.getTeamConfigService(teamId).getWorkload(name) + const workload = this.fileStore.getTeamResource('AplTeamWorkload', teamId, name) + if (!workload) { + throw new NotExistError(`Workload ${name} not found in team ${teamId}`) + } + return workload as AplWorkloadResponse } async editWorkload(teamId: string, name: string, data: Workload): Promise { @@ -1734,60 +1674,48 @@ export default class OtomiStack { data: AplWorkloadRequest | DeepPartial, patch = false, ): Promise { - const workload = patch - ? this.repoService.getTeamConfigService(teamId).patchWorkload(name, data) - : this.repoService.getTeamConfigService(teamId).updateWorkload(name, data as AplWorkloadRequest) - await this.saveTeamWorkload(workload) + const existing = this.getAplWorkload(teamId, name) + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), data.spec) + : ({ ...existing.spec, ...data.spec } as Workload) + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamWorkload(teamObject) + const workloadResponse = aplRecord.content as AplWorkloadResponse if (data.spec && 'values' in data.spec) { - await this.saveTeamWorkloadValues(workload) + await this.saveTeamWorkloadValues(teamId, name, data.spec.values!) } - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateWorkload(name, workload) - }, - false, - ) - return workload + await this.doDeployment(aplRecord, false) + return workloadResponse } async deleteWorkload(teamId: string, name: string): Promise { - const workload = this.repoService.getTeamConfigService(teamId).getWorkload(name) - this.repoService.getTeamConfigService(teamId).deleteWorkload(name) - await this.deleteTeamWorkload(workload) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteWorkload(name) - }, - false, - ) + const filePath = await this.deleteTeamWorkload('AplTeamWorkload', teamId, name) + await this.doDeleteDeployment([filePath], false) } async editWorkloadValues(teamId: string, name: string, data: WorkloadValues): Promise { - const workload = this.repoService.getTeamConfigService(teamId).patchWorkload(name, { - spec: { - values: stringifyYaml(deepQuote(data.values)), - }, - }) - await this.saveTeamWorkloadValues(workload) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateWorkload(name, workload) - }, - false, - ) + const existing = this.getAplWorkload(teamId, name) + const updatedSpec = { + ...existing.spec, + values: stringifyYaml(deepQuote(data.values)), + } + + const workload: AplWorkloadResponse = { + ...existing, + spec: updatedSpec as AplWorkloadResponse['spec'], + } + const aplRecord = await this.saveTeamWorkloadValues(teamId, name, updatedSpec.values) + await this.doDeployment(aplRecord, false) return merge(pick(getV1ObjectFromApl(workload), ['id', 'teamId', 'name']), { values: data.values || undefined, }) as WorkloadValues } getWorkloadValues(teamId: string, name: string): WorkloadValues { - const workload = this.getAplWorkload(teamId, name) - return merge(pick(getV1ObjectFromApl(workload), ['id', 'teamId', 'name']), { - values: workload.spec.values || undefined, - }) as WorkloadValues + const workload = this.fileStore.getTeamResource('AplTeamWorkloadValues', teamId, name) + return { teamId, name, values: workload as any } } getAllServices(): Service[] { @@ -1795,7 +1723,8 @@ export default class OtomiStack { } getAllAplServices(): AplServiceResponse[] { - return this.repoService.getAllServices() + const files = this.fileStore.getByKind('AplTeamService') + return Array.from(files.values()) as AplServiceResponse[] } getTeamServices(teamId: string): Service[] { @@ -1803,7 +1732,8 @@ export default class OtomiStack { } getTeamAplServices(teamId: string): AplServiceResponse[] { - return this.repoService.getTeamConfigService(teamId).getServices() + const files = this.fileStore.getByKind('AplTeamService', teamId) + return Array.from(files.values()) as AplServiceResponse[] } async createService(teamId: string, data: Service): Promise { @@ -1818,21 +1748,13 @@ export default class OtomiStack { if (data.metadata.name.length < 2) throw new ValidationError('Service name must be at least 2 characters long') if (data.spec.cname?.tlsSecretName && data.spec.cname?.tlsSecretName.length < 2) throw new ValidationError('Secret name must be at least 2 characters long') - try { - const service = this.repoService.getTeamConfigService(teamId).createService(data) - await this.saveTeamConfigItem(service) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createService(service) - }, - false, - ) - return service - } catch (err) { - if (err.code === 409) err.publicMessage = 'Service name already exists' - throw err + if (this.fileStore.getTeamResource('AplTeamService', teamId, data.metadata.name)) { + throw new AlreadyExists('Service name already exists') } + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplServiceResponse } getService(teamId: string, name: string): Service { @@ -1841,7 +1763,11 @@ export default class OtomiStack { } getAplService(teamId: string, name: string): AplServiceResponse { - return this.repoService.getTeamConfigService(teamId).getService(name) + const service = this.fileStore.getTeamResource('AplTeamService', teamId, name) + if (!service) { + throw new NotExistError(`Service ${name} not found in team ${teamId}`) + } + return service as AplServiceResponse } async editService(teamId: string, name: string, data: Service): Promise { @@ -1856,36 +1782,25 @@ export default class OtomiStack { data: DeepPartial, patch = false, ): Promise { - const service = patch - ? this.repoService.getTeamConfigService(teamId).patchService(name, data) - : this.repoService.getTeamConfigService(teamId).updateService(name, data as AplServiceRequest) - await this.saveTeamConfigItem(service) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateService(name, service) - }, - false, - ) - return service + const existing = this.getAplService(teamId, name) + const updatedSpec = patch ? merge(cloneDeep(existing.spec), data.spec) : { ...existing.spec, ...data.spec } + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplServiceResponse } async deleteService(teamId: string, name: string): Promise { - const service = this.getAplService(teamId, name) - await this.deleteTeamConfigItem(service) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteService(name) - }, - false, - ) + const filePath = await this.deleteTeamConfigItem('AplTeamService', teamId, name) + await this.doDeleteDeployment([filePath], false) } checkPublicUrlInUse(teamId: string, service: AplServiceRequest): void { // skip when editing or when svc is of type "cluster" as it has no url const newSvc = service.spec - const services = this.repoService.getTeamConfigService(teamId).getServices() + const services = this.getTeamAplServices(teamId) const servicesFiltered = filter(services, (svc) => { const { domain, paths } = svc.spec @@ -1903,13 +1818,30 @@ export default class OtomiStack { }) if (servicesFiltered.length > 0) throw new PublicUrlExists() } + async doDeployments(aplRecords: AplRecord[], encryptSecrets = true, files?: string[]): Promise { + const rootStack = await getSessionStack() - async doTeamDeployment( - teamId: string, - action: (teamService: TeamConfigService) => void, - encryptSecrets = true, - files?: string[], - ): Promise { + try { + // Commit and push Git changes + await this.git.save(this.editor!, encryptSecrets, files) + // Pull the latest changes to ensure we have the most recent state + await rootStack.git.git.pull() + + for (const aplRecord of aplRecords) { + rootStack.fileStore.set(aplRecord.filePath, aplRecord.content) + } + + debug(`Updated root stack values with ${this.sessionId} changes`) + } catch (e) { + e.message = getSanitizedErrorMessage(e) + throw e + } finally { + // Clean up the session + await cleanSession(this.sessionId!) + } + } + + async doDeployment(aplRecord: AplRecord, encryptSecrets = true, files?: string[]): Promise { const rootStack = await getSessionStack() try { @@ -1918,8 +1850,7 @@ export default class OtomiStack { // Pull the latest changes to ensure we have the most recent state await rootStack.git.git.pull() - // Update the team configuration of the root stack - action(rootStack.repoService.getTeamConfigService(teamId)) + rootStack.fileStore.set(aplRecord.filePath, aplRecord.content) debug(`Updated root stack values with ${this.sessionId} changes`) } catch (e) { @@ -1931,11 +1862,7 @@ export default class OtomiStack { } } - async doRepoDeployment( - action: (repoService: RepoService) => void, - encryptSecrets = true, - files?: string[], - ): Promise { + async doDeleteDeployment(filePaths: string[], encryptSecrets = true, files?: string[]): Promise { const rootStack = await getSessionStack() try { @@ -1943,8 +1870,10 @@ export default class OtomiStack { await this.git.save(this.editor!, encryptSecrets, files) // Pull the latest changes to ensure we have the most recent state await rootStack.git.git.pull() - // update the repo configuration of the root stack - action(rootStack.repoService) + + for (const filePath of filePaths) { + rootStack.fileStore.delete(filePath) + } debug(`Updated root stack values with ${this.sessionId} changes`) } catch (e) { @@ -2055,24 +1984,10 @@ export default class OtomiStack { async getK8sServices(teamId: string): Promise> { if (env.isDev) return [] - // const teams = user.teams.map((name) => { - // return `team-${name}` - // }) const client = this.getApiClient() const collection: K8sService[] = [] - // if (user.isAdmin) { - // const svcList = await client.listServiceForAllNamespaces() - // svcList.body.items.map((item) => { - // collection.push({ - // name: item.metadata!.name ?? 'unknown', - // ports: item.spec?.ports?.map((portItem) => portItem.port) ?? [], - // }) - // }) - // return collection - // } - const svcList = await client.listNamespacedService({ namespace: `team-${teamId}` }) svcList.items.map((item) => { let name = item.metadata!.name ?? 'unknown' @@ -2156,21 +2071,12 @@ export default class OtomiStack { async createAplSealedSecret(teamId: string, data: AplSecretRequest): Promise { if (data.metadata.name.length < 2) throw new ValidationError('Secret name must be at least 2 characters long') - try { - const sealedSecret = this.repoService.getTeamConfigService(teamId).createSealedSecret(data) - await this.saveTeamSealedSecret(sealedSecret) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createSealedSecret(sealedSecret) - }, - false, - ) - return sealedSecret - } catch (err) { - if (err.code === 409) err.publicMessage = 'SealedSecret name already exists' - throw err + if (this.fileStore.getTeamResource(data.kind, teamId, data.metadata.name)) { + throw new AlreadyExists('SealedSecret name already exists') } + const aplRecord = await this.saveTeamSealedSecret(teamId, data) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplSecretResponse } async editSealedSecret(teamId: string, name: string, data: SealedSecret): Promise { @@ -2185,50 +2091,35 @@ export default class OtomiStack { data: DeepPartial, patch = false, ): Promise { - const existingSecret = this.repoService.getTeamConfigService(teamId).getSealedSecret(name) - const namespace = data.spec?.namespace ?? existingSecret.spec.namespace ?? `team-${teamId}` - const sealedSecret = patch - ? this.repoService.getTeamConfigService(teamId).patchSealedSecret(name, { - metadata: data.metadata, - spec: { - encryptedData: data.spec?.encryptedData, - namespace, - }, + const existing = await this.getAplSealedSecret(teamId, name) + const namespace = data.spec?.namespace ?? existing.spec.namespace ?? `team-${teamId}` + + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), { + encryptedData: data.spec?.encryptedData, + namespace, }) - : this.repoService.getTeamConfigService(teamId).updateSealedSecret(name, { - kind: 'AplTeamSecret', - metadata: data.metadata, - spec: { - ...existingSecret.spec, - encryptedData: data.spec?.encryptedData, - namespace, - immutable: data.spec?.immutable ?? existingSecret.spec.immutable, - metadata: data.spec?.metadata ?? existingSecret.spec.metadata, - }, - } as AplSecretRequest) - await this.saveTeamSealedSecret(sealedSecret) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateSealedSecret(name, sealedSecret) - }, - false, - ) - return sealedSecret + : ({ + ...existing.spec, + encryptedData: data.spec?.encryptedData, + namespace, + immutable: data.spec?.immutable ?? existing.spec.immutable, + metadata: data.spec?.metadata ?? existing.spec.metadata, + } as SealedSecret) + + const aplRecord = await this.saveTeamSealedSecret(teamId, { + kind: existing.kind, + metadata: existing.metadata, + spec: updatedSpec, + }) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplSecretResponse } async deleteSealedSecret(teamId: string, name: string): Promise { - const sealedSecret = await this.getAplSealedSecret(teamId, name) - this.repoService.getTeamConfigService(teamId).deleteSealedSecret(sealedSecret.metadata.name) - const relativePath = getTeamSealedSecretsValuesFilePath(teamId, `${name}.yaml`) - await this.git.removeFile(relativePath) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteSealedSecret(name) - }, - false, - ) + const filePath = this.fileStore.deleteTeamResource('AplTeamSecret', teamId, name) + await this.git.removeFile(filePath) + await this.doDeleteDeployment([filePath], false) } async getSealedSecret(teamId: string, name: string): Promise { @@ -2237,8 +2128,11 @@ export default class OtomiStack { } async getAplSealedSecret(teamId: string, name: string): Promise { - const sealedSecret = this.repoService.getTeamConfigService(teamId).getSealedSecret(name) - return sealedSecret + const sealedSecret = this.fileStore.getTeamResource('AplTeamSecret', teamId, name) + if (!sealedSecret) { + throw new NotExistError(`SealedSecret ${name} not found in team ${teamId}`) + } + return sealedSecret as AplSecretResponse } getAllSealedSecrets(): SealedSecret[] { @@ -2246,15 +2140,20 @@ export default class OtomiStack { } getAllAplSealedSecrets(): AplSecretResponse[] { - return this.repoService.getAllSealedSecrets() + const files = this.fileStore.getByKind('AplTeamSecret') + return Array.from(files.values()) as AplSecretResponse[] } getSealedSecrets(teamId: string): SealedSecret[] { - return this.getAplSealedSecrets(teamId).map(getV1ObjectFromApl) as SealedSecret[] + return this.getAplSealedSecrets(teamId).map((secret) => ({ + ...getV1ObjectFromApl(secret), + teamId, + })) as SealedSecret[] } getAplSealedSecrets(teamId: string): AplSecretResponse[] { - return this.repoService.getTeamConfigService(teamId).getSealedSecrets() + const files = this.fileStore.getByKind('AplTeamSecret', teamId) + return Array.from(files.values()) as AplSecretResponse[] } async getSecretsFromK8s(teamId: string): Promise> { @@ -2265,21 +2164,14 @@ export default class OtomiStack { async createAplKnowledgeBase(teamId: string, data: AplKnowledgeBaseRequest): Promise { if (data.metadata.name.length < 2) throw new ValidationError('Knowledge base name must be at least 2 characters long') - try { - const knowledgeBase = this.repoService.getTeamConfigService(teamId).createKnowledgeBase(data) - await this.saveTeamKnowledgeBase(teamId, knowledgeBase) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createKnowledgeBase(knowledgeBase) - }, - false, - ) - return knowledgeBase - } catch (err) { - if (err.code === 409) err.publicMessage = 'Knowledge base name already exists' - throw err + if (this.fileStore.getTeamResource('AkamaiKnowledgeBase', teamId, data.metadata.name)) { + throw new AlreadyExists('Knowledge base name already exists') } + + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamKnowledgeBase(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplKnowledgeBaseResponse } async editAplKnowledgeBase( @@ -2288,98 +2180,65 @@ export default class OtomiStack { data: DeepPartial, patch = false, ): Promise { - const existingKB = this.repoService.getTeamConfigService(teamId).getKnowledgeBase(name) - const knowledgeBase = patch - ? this.repoService.getTeamConfigService(teamId).patchKnowledgeBase(name, { - metadata: data.metadata, - spec: data.spec, - }) - : this.repoService.getTeamConfigService(teamId).updateKnowledgeBase(name, { - kind: 'AkamaiKnowledgeBase', - metadata: { - name: data.metadata?.name ?? existingKB.metadata.name, - }, - spec: { - modelName: data.spec?.modelName ?? existingKB.spec.modelName, - sourceUrl: data.spec?.sourceUrl ?? existingKB.spec.sourceUrl, - }, - }) + const existing = await this.getAplKnowledgeBase(teamId, name) - await this.saveTeamKnowledgeBase(teamId, knowledgeBase) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateKnowledgeBase(name, knowledgeBase) - }, - false, - ) - return knowledgeBase + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), data.spec) + : { + modelName: data.spec?.modelName ?? existing.spec.modelName, + sourceUrl: data.spec?.sourceUrl ?? existing.spec.sourceUrl, + } + + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamKnowledgeBase(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplKnowledgeBaseResponse } async deleteAplKnowledgeBase(teamId: string, name: string): Promise { - const knowledgeBase = await this.getAplKnowledgeBase(teamId, name) - this.repoService.getTeamConfigService(teamId).deleteKnowledgeBase(knowledgeBase.metadata.name) + const filePath = this.fileStore.deleteTeamResource('AkamaiKnowledgeBase', teamId, name) const relativePath = getTeamKnowledgeBaseValuesFilePath(teamId, `${name}.yaml`) const databasePath = getTeamDatabaseValuesFilePath(teamId, `${name}.yaml`) await this.git.removeFile(relativePath) await this.git.removeFile(databasePath) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteKnowledgeBase(name) - }, - false, - ) + await this.doDeleteDeployment([filePath], false) } async getAplKnowledgeBase(teamId: string, name: string): Promise { - const knowledgeBase = this.repoService.getTeamConfigService(teamId).getKnowledgeBase(name) - return knowledgeBase + const knowledgeBase = this.fileStore.getTeamResource('AkamaiKnowledgeBase', teamId, name) + if (!knowledgeBase) { + throw new NotExistError(`Knowledge base ${name} not found in team ${teamId}`) + } + return knowledgeBase as AplKnowledgeBaseResponse } getAplKnowledgeBases(teamId: string): AplKnowledgeBaseResponse[] { - return this.repoService.getTeamConfigService(teamId).getKnowledgeBases() + const files = this.fileStore.getByKind('AkamaiKnowledgeBase', teamId) + return Array.from(files.values()) as AplKnowledgeBaseResponse[] } - getAllAplKnowledgeBases(): AplKnowledgeBaseResponse[] { - return this.repoService.getAllKnowledgeBases() - } - - private async saveTeamKnowledgeBase(teamId: string, knowledgeBase: AplKnowledgeBaseResponse): Promise { - const databaseCR = await DatabaseCR.create(teamId, knowledgeBase.metadata.name) - const knowledgeBaseCR = await AkamaiKnowledgeBaseCR.create( - teamId, - knowledgeBase.metadata.name, - databaseCR.spec.cluster.name, - { - kind: 'AkamaiKnowledgeBase', - metadata: knowledgeBase.metadata, - spec: knowledgeBase.spec, - }, - ) + private async saveTeamKnowledgeBase(aplTeamObject: AplTeamObject): Promise { + const { metadata } = aplTeamObject + const teamId = metadata.labels['apl.io/teamId'] + const databaseCR = await DatabaseCR.create(teamId, metadata.name) - await this.saveKnowledgeBaseCR(teamId, knowledgeBaseCR) + const filePath = this.fileStore.setTeamResource(aplTeamObject) + await this.git.writeFile(filePath, aplTeamObject) await this.saveDatabaseCR(teamId, databaseCR) + + return { filePath, content: aplTeamObject } } - // Agent methods - following the same patterns as knowledge base methods async createAplAgent(teamId: string, data: AplAgentRequest): Promise { if (data.metadata.name.length < 2) throw new ValidationError('Agent name must be at least 2 characters long') - try { - const agent = this.repoService.getTeamConfigService(teamId).createAgent(data) - await this.saveTeamAgent(teamId, agent) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.createAgent(agent) - }, - false, - ) - return agent - } catch (err) { - if (err.code === 409) err.publicMessage = 'Agent name already exists' - throw err + if (this.fileStore.getTeamResource('AkamaiAgent', teamId, data.metadata.name)) { + throw new AlreadyExists('Agent name already exists') } + const teamObject = toTeamObject(teamId, data) + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplAgentResponse } async editAplAgent( @@ -2388,66 +2247,45 @@ export default class OtomiStack { data: DeepPartial, patch = false, ): Promise { - const existingAgent = this.repoService.getTeamConfigService(teamId).getAgent(name) - const agent = patch - ? this.repoService.getTeamConfigService(teamId).patchAgent(name, { - metadata: data.metadata, - spec: data.spec, - }) - : this.repoService.getTeamConfigService(teamId).updateAgent(name, { - kind: 'AkamaiAgent', - metadata: { - name: data.metadata?.name ?? existingAgent.metadata.name, - }, - spec: { - foundationModel: data.spec?.foundationModel ?? existingAgent.spec.foundationModel, - agentInstructions: data.spec?.agentInstructions ?? existingAgent.spec.agentInstructions, - tools: (data.spec?.tools ?? existingAgent.spec.tools) as typeof existingAgent.spec.tools, - }, - }) + const existing = this.getAplAgent(teamId, name) + const updatedSpec = patch + ? merge(cloneDeep(existing.spec), data.spec) + : { + foundationModel: data.spec?.foundationModel ?? existing.spec.foundationModel, + agentInstructions: data.spec?.agentInstructions ?? existing.spec.agentInstructions, + tools: (data.spec?.tools ?? existing.spec.tools) as typeof existing.spec.tools, + } - await this.saveTeamAgent(teamId, agent) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.updateAgent(name, agent) - }, - false, - ) - return agent + const teamObject = buildTeamObject(existing, updatedSpec) + + const aplRecord = await this.saveTeamConfigItem(teamObject) + await this.doDeployment(aplRecord, false) + return aplRecord.content as AplAgentResponse } async deleteAplAgent(teamId: string, name: string): Promise { - const agent = await this.getAplAgent(teamId, name) - this.repoService.getTeamConfigService(teamId).deleteAgent(agent.metadata.name) - const relativePath = getTeamAgentValuesFilePath(teamId, `${name}.yaml`) - await this.git.removeFile(relativePath) - await this.doTeamDeployment( - teamId, - (teamService) => { - teamService.deleteAgent(name) - }, - false, - ) + const filePath = this.fileStore.deleteTeamResource('AkamaiAgent', teamId, name) + + await this.git.removeFile(filePath) + await this.doDeleteDeployment([filePath], false) } - async getAplAgent(teamId: string, name: string): Promise { - const agent = this.repoService.getTeamConfigService(teamId).getAgent(name) - return agent + getAplAgent(teamId: string, name: string): AplAgentResponse { + const agent = this.fileStore.getTeamResource('AkamaiAgent', teamId, name) + if (!agent) { + throw new NotExistError(`Agent ${name} not found in team ${teamId}`) + } + return agent as AplAgentResponse } getAplAgents(teamId: string): AplAgentResponse[] { - return this.repoService.getTeamConfigService(teamId).getAgents() + const files = this.fileStore.getByKind('AkamaiAgent', teamId) + return Array.from(files.values()) as AplAgentResponse[] } - private async saveTeamAgent(teamId: string, agent: AplAgentResponse): Promise { - const agentCR = await AkamaiAgentCR.create(teamId, agent.metadata.name, { - kind: 'AkamaiAgent', - metadata: agent.metadata, - spec: agent.spec, - }) - - await this.saveAgentCR(teamId, agentCR) + getAllAplAgents(): AplAgentResponse[] { + const files = this.fileStore.getByKind('AkamaiAgent') + return Array.from(files.values()) as AplAgentResponse[] } private async saveDatabaseCR(teamId: string, databaseCR: DatabaseCR) { @@ -2455,16 +2293,6 @@ export default class OtomiStack { await this.git.writeFile(dbPath, databaseCR.toRecord()) } - private async saveKnowledgeBaseCR(teamId: string, knowledgeBaseCR: AkamaiKnowledgeBaseCR) { - const kbPath = getTeamKnowledgeBaseValuesFilePath(teamId, `${knowledgeBaseCR.metadata.name}.yaml`) - await this.git.writeFile(kbPath, knowledgeBaseCR.toRecord()) - } - - private async saveAgentCR(teamId: string, agentCR: AkamaiAgentCR) { - const agentPath = getTeamAgentValuesFilePath(teamId, `${agentCR.metadata.name}.yaml`) - await this.git.writeFile(agentPath, agentCR.toRecord()) - } - async loadValues(): Promise>>>> { debug('Loading values') await this.git.initSops() @@ -2472,65 +2300,174 @@ export default class OtomiStack { this.isLoaded = true } + private buildSecretObject(aplObject: AplTeamObject | AplPlatformObject, secretSpec: Record): AplObject { + return { + kind: aplObject.kind, + metadata: aplObject.metadata, + spec: omit(secretSpec, ['id', 'teamId', 'name']), + } + } + + private extractAppSecretPaths(appName: string, globalPaths: string[]): string[] { + const appPrefix = `apps.${appName}.` + return globalPaths.filter((path) => path.startsWith(appPrefix)).map((path) => path.replace(appPrefix, '')) + } + + private extractSettingsSecretPaths(kind: AplKind, globalPaths: string[]): string[] { + const settingsPrefixMap: Record = { + AplDns: 'dns.', + AplKms: 'kms.', + AplSmtp: 'smtp.', + AplIdentityProvider: 'oidc.', + AplCapabilitySet: 'otomi.', + AplAlertSet: 'alerts.', + AplObjectStorage: 'obj.', + } + + const prefix = settingsPrefixMap[kind] + if (!prefix) return [] + + return globalPaths.filter((path) => path.startsWith(prefix)).map((path) => path.replace(prefix, '')) + } + + private extractTeamSecretPaths(globalPaths: string[]): string[] { + // Team paths use pattern: teamConfig.patternProperties.^[a-z0-9]([-a-z0-9]*[a-z0-9])+$.settings.{field} + const teamPattern = 'teamConfig.patternProperties.^[a-z0-9]([-a-z0-9]*[a-z0-9])+$.settings.' + + return globalPaths.filter((path) => path.startsWith(teamPattern)).map((path) => path.replace(teamPattern, '')) + } + + private async saveWithSecrets( + aplObject: AplTeamObject | AplPlatformObject, + secretPaths: string[], + ): Promise { + const secretData = {} + const specWithoutSecrets = cloneDeep(aplObject.spec) + secretPaths.forEach((secretPath) => { + const secretValue = get(aplObject.spec, secretPath) + if (secretValue) { + set(secretData, secretPath, secretValue) + unset(specWithoutSecrets, secretPath) + } + }) + + // Determine file path and save using appropriate FileStore method + let filePath: string + if ('labels' in aplObject.metadata && 'apl.io/teamId' in aplObject.metadata.labels) { + // Store full object with secrets. + // TODO check if this is needed. + filePath = this.fileStore.setTeamResource(aplObject as AplTeamObject) + } else { + // Store full object with secrets. + // TODO check if this is needed. + filePath = this.fileStore.setPlatformResource(aplObject as AplPlatformObject) + } + + // Write main file + await this.git.writeFile(filePath, { ...aplObject, spec: specWithoutSecrets }) + + // Write secrets file if there are any secrets + if (Object.keys(secretData).length > 0) { + const secretFilePath = getSecretFilePath(filePath) + // Build proper AplObject structure for secret file + const secretObject = this.buildSecretObject(aplObject, secretData) + await this.git.writeFile(secretFilePath, secretObject) + } + + return { filePath, content: aplObject } + } + async saveAdminApp(app: App, secretPaths?: string[]): Promise { const { id, enabled, values, rawValues } = app - const apps = { - [id]: { - ...(values || {}), - }, + const spec: Record = { + ...(values || {}), } + if (!isEmpty(rawValues)) { - apps[id]._rawValues = rawValues + spec._rawValues = rawValues } if (this.canToggleApp(id)) { - apps[id].enabled = !!enabled - } else { - delete apps[id].enabled + spec.enabled = !!enabled } - const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplApp')! - await this.git.saveConfigWithSecrets({ apps }, secretPaths ?? this.getSecretPaths(), fileMap) + const aplPlatformObject = buildPlatformObject('AplApp', id, spec) + + const globalPaths = secretPaths ?? getSecretPaths() + const appSecretPaths = this.extractAppSecretPaths(id, globalPaths) + + await this.saveWithSecrets(aplPlatformObject, appSecretPaths) } async saveSettings(secretPaths?: string[]): Promise { const settings = cloneDeep(this.getSettings()) as Record> settings.otomi.nodeSelector = arrayToObject(settings.otomi.nodeSelector as []) - const fileMaps = getFileMaps('').filter((fm) => fm.resourceDir === 'settings')! - for (const fileMap of fileMaps) { - await this.git.saveConfigWithSecrets(settings, secretPaths ?? this.getSecretPaths(), fileMap) + + // Get all settings file maps + const settingsFileMaps = getSettingsFileMaps('') + const globalPaths = secretPaths ?? getSecretPaths() + + // Save each setting as a separate AplPlatformObject + for (const [settingName, fileMap] of settingsFileMaps.entries()) { + const settingValue = settings[settingName] + if (settingValue) { + const aplPlatformObject = buildPlatformObject(fileMap.kind, settingName, settingValue) + const settingsSecretPaths = this.extractSettingsSecretPaths(fileMap.kind, globalPaths) + await this.saveWithSecrets(aplPlatformObject, settingsSecretPaths) + } } } - async saveUser(user: User): Promise { + async saveUser(user: User): Promise { debug(`Saving user ${user.email}`) - const users: User[] = [] - users.push(user) - const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplUser')! - await this.git.saveSecretConfig({ users }, fileMap, false) + + if (!user.id) { + throw new Error('User id not set') + } + const aplPlatformObject = buildPlatformObject('AplUser', user.id, user as unknown as Record) + const filePath = this.fileStore.setPlatformResource(aplPlatformObject) + + // Save all values to secrets files as users do not have main file + const secretObject = this.buildSecretObject(aplPlatformObject, user as unknown as Record) + const secretFilePath = getSecretFilePath(filePath) + await this.git.writeFile(secretFilePath, secretObject) + + return { filePath, content: aplPlatformObject } } async deleteUserFile(user: User): Promise { debug(`Deleting user ${user.email}`) - this.repoService.deleteUser(user.email) + const filePath = getResourceFilePath('AplUser', user.email) + this.fileStore.delete(filePath) const users: User[] = [] users.push(user) const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplUser')! await this.git.deleteConfig({ users }, fileMap, 'secrets.') } - async saveTeam(team: AplTeamSettingsResponse, secretPaths?: string[]): Promise { - const { kind, metadata } = team - debug(`Saving team ${metadata.name}`) - const repo = this.createTeamConfigInRepo(team.metadata.name, 'settings', team) - const fileMap = getFileMaps('').find((fm) => fm.kind === kind)! - await this.git.saveConfigWithSecrets(repo, secretPaths ?? this.getSecretPaths(), fileMap) + async saveTeam(aplTeamObject: AplTeamObject, secretPaths?: string[]): Promise { + const teamId = aplTeamObject.metadata.labels['apl.io/teamId'] + debug(`Saving team ${teamId}`) + + const globalPaths = secretPaths ?? getSecretPaths() + const teamSecretPaths = this.extractTeamSecretPaths(globalPaths) + + return await this.saveWithSecrets(aplTeamObject, teamSecretPaths) } - async deleteTeamConfig(name: string): Promise { - this.repoService.deleteTeamConfig(name) + async deleteTeamObjects(name: string): Promise { + // Delete all files for this team from file store + const teamPrefix = `env/teams/${name}/` + const filePaths: string[] = [] + for (const key of this.fileStore.keys()) { + if (key.startsWith(teamPrefix)) { + this.fileStore.delete(key) + filePaths.push(key) + } + } const teamDir = `env/teams/${name}` await this.git.removeDir(teamDir) + return filePaths } transformService(service: AplServiceResponse): Record { @@ -2608,16 +2545,6 @@ export default class OtomiStack { } } - private createTeamConfigInRepo(teamId: string, key: string, value: T): Record { - return { - teamConfig: { - [teamId]: { - [key]: value, - }, - }, - } - } - private getVersions(currentSha: string): Record { const { otomi } = this.getSettings(['otomi']) return { diff --git a/src/services/RepoService.test.ts b/src/services/RepoService.test.ts deleted file mode 100644 index 8f2d8f0ca..000000000 --- a/src/services/RepoService.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { AlreadyExists } from '../error' -import { AplTeamSettingsRequest, App, Repo, User } from '../otomi-models' -import { RepoService } from './RepoService' -import { TeamConfigService } from './TeamConfigService' - -jest.mock('uuid', () => ({ - v4: jest.fn(() => 'mocked-uuid'), -})) - -describe('RepoService', () => { - let service: RepoService - let repo: Repo - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'team1', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - } as AplTeamSettingsRequest - - beforeEach(() => { - repo = { - apps: [], - users: [], - teamConfig: {}, - cluster: { name: 'Test Cluster', provider: 'linode' }, - dns: {}, - ingress: {}, - otomi: { version: '1.0.0' }, - smtp: { smarthost: 'smtp.mailtrap.io' }, - platformBackups: {}, - alerts: {}, - databases: {}, - kms: {}, - obj: {}, - oidc: { issuer: 'https://issuer.com', clientID: 'client-id', clientSecret: 'client-secret' }, - versions: { version: '1.0.0' }, - files: {}, - } as Repo - service = new RepoService(repo) - service.createTeamConfig(teamSettings) - }) - - describe('getTeamConfigService', () => { - test('should throw an error if team config does not exist', () => { - expect(() => service.getTeamConfigService('nonexistent-team')).toThrow( - 'TeamConfig for nonexistent-team does not exist.', - ) - }) - - test('should return an instance of TeamConfigService when team config exists', () => { - const teamConfigService = service.getTeamConfigService('team1') - - expect(teamConfigService).toBeInstanceOf(TeamConfigService) - expect(service.getTeamConfig('team1')).toBeDefined() - }) - }) - - describe('Users', () => { - const user: User = { email: 'user@test.com', firstName: 'user', lastName: 'test' } - - test('should create a user', () => { - const createdUser = service.createUser(user) - expect(createdUser).toEqual({ email: 'user@test.com', id: 'mocked-uuid', firstName: 'user', lastName: 'test' }) - expect(service.getUsers()).toHaveLength(1) - }) - - test('should throw an error if user already exists', () => { - service.createUser(user) - expect(() => service.createUser(user)).toThrow(AlreadyExists) - }) - - test('should retrieve a user by ID', () => { - const createdUser = service.createUser(user) - expect(service.getUser(createdUser.id!)).toEqual(createdUser) - }) - - test('should delete a user', () => { - const createdUser = service.createUser(user) - service.deleteUser(createdUser.email) - expect(service.getUsers()).toHaveLength(0) - }) - }) - - describe('Apps', () => { - const app: App = { id: 'app1', enabled: true } - - test('should retrieve all apps', () => { - service.getRepo().apps.push(app) - expect(service.getApps()).toContain(app) - }) - - test('should retrieve a specific app', () => { - service.getRepo().apps.push(app) - expect(service.getApp('app1')).toEqual(app) - }) - - test('should throw an error when retrieving a non-existent app', () => { - expect(() => service.getApp('nonexistent')).toThrow('App[nonexistent] does not exist.') - }) - - test('should update an app', () => { - service.getRepo().apps.push(app) - const updatedApp = service.updateApp('app1', { enabled: false }) - expect(updatedApp.enabled).toBe(false) - }) - - test('should delete an app', () => { - service.getRepo().apps.push(app) - service.deleteApp('app1') - expect(service.getApps()).toHaveLength(0) - }) - }) - - describe('Team Config', () => { - test('should create a team config', () => { - const teamConfig = service.getTeamConfig('team1') - expect(teamConfig?.settings).toEqual({ - kind: 'AplTeamSettingSet', - metadata: { - labels: { - 'apl.io/teamId': 'team1', - }, - name: 'team1', - }, - spec: {}, - status: {}, - }) - expect(service.getTeamConfig('team1')).toBeDefined() - }) - - test('should throw an error if team config already exists', () => { - expect(() => service.createTeamConfig(teamSettings)).toThrow(AlreadyExists) - }) - - test('should delete a team config', () => { - service.deleteTeamConfig('team1') - expect(service.getTeamConfig('team1')).toBeUndefined() - }) - }) - - describe('Collection Functions', () => { - test('should retrieve a collection', () => { - service.getRepo().cluster = { name: 'Test Cluster', provider: 'linode' } - expect(service.getCollection('cluster')).toEqual({ name: 'Test Cluster', provider: 'linode' }) - }) - - test('should throw an error for non-existent collection', () => { - expect(() => service.getCollection('nonexistent')).toThrow( - 'Getting repo collection [nonexistent] does not exist.', - ) - }) - - test('should update an existing collection', () => { - service.getRepo().cluster = { name: 'Old Cluster', provider: 'linode' } - service.updateCollection('cluster', { name: 'Updated Cluster' }) - expect(service.getCollection('cluster')).toEqual({ name: 'Updated Cluster' }) - }) - - test('should throw an error when updating a non-existent collection', () => { - expect(() => service.updateCollection('nonexistent', { key: 'value' })).toThrow( - 'Updating repo collection [nonexistent] does not exist.', - ) - }) - }) - - describe('Settings', () => { - test('should retrieve settings', () => { - service.getRepo().cluster = { name: 'Cluster A', provider: 'linode' } - service.getRepo().dns = { provider: { linode: { apiToken: 'test' } } } - const settings = service.getSettings() - - expect(settings.cluster).toEqual({ name: 'Cluster A', provider: 'linode' }) - expect(settings.dns).toEqual({ provider: { linode: { apiToken: 'test' } } }) - }) - - test('should update settings', () => { - service.updateSettings({ cluster: { name: 'Updated Cluster', provider: 'linode' } }) - expect(service.getSettings().cluster).toEqual({ name: 'Updated Cluster', provider: 'linode' }) - }) - }) - - describe('Global Retrieval Functions', () => { - test('should return all users emails', () => { - service.createUser({ email: 'user1@test.com', firstName: 'user', lastName: 'test' }) - service.createUser({ email: 'user2@test.com', firstName: 'user', lastName: 'test' }) - expect(service.getUsersEmail()).toEqual(['user1@test.com', 'user2@test.com']) - }) - - test('should return all builds', () => { - service.getTeamConfigService('team1').createBuild({ - kind: 'AplTeamBuild', - metadata: { name: 'Build1' }, - spec: {}, - }) - expect(service.getAllBuilds()).toHaveLength(1) - }) - }) -}) diff --git a/src/services/RepoService.ts b/src/services/RepoService.ts deleted file mode 100644 index 2ffe25fe4..000000000 --- a/src/services/RepoService.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { find, has, map, mergeWith, remove, set } from 'lodash' -import { v4 as uuidv4 } from 'uuid' -import { AlreadyExists } from '../error' -import { - Alerts, - AplBuildResponse, - AplCodeRepoResponse, - AplKnowledgeBaseResponse, - AplNetpolResponse, - AplPolicyResponse, - AplSecretResponse, - AplServiceResponse, - AplTeamSettingsRequest, - AplTeamSettingsResponse, - AplWorkloadResponse, - App, - Cluster, - Dns, - Ingress, - Kms, - Otomi, - Repo, - Settings, - Smtp, - TeamConfig, - User, - Versions, -} from '../otomi-models' -import { createAplObject } from '../utils/manifests' -import { TeamConfigService } from './TeamConfigService' - -function mergeCustomizer(prev, next) { - return next -} - -export class RepoService { - // We can create an LRU cache if needed with a lot of teams. - private teamConfigServiceCache = new Map() - - constructor(private repo: Repo) { - this.repo.apps ??= [] - this.repo.alerts ??= {} as Alerts - this.repo.cluster ??= {} as Cluster - this.repo.databases ??= {} - this.repo.dns ??= {} as Dns - this.repo.ingress ??= {} as Ingress - this.repo.kms ??= {} as Kms - this.repo.obj ??= {} - this.repo.otomi ??= {} as Otomi - this.repo.platformBackups ??= {} - this.repo.users ??= [] - this.repo.versions ??= {} as Versions - this.repo.teamConfig ??= {} - this.repo.files ??= {} - } - - public getTeamConfigService(teamName: string): TeamConfigService { - if (!this.repo.teamConfig[teamName]) { - throw new Error(`TeamConfig for ${teamName} does not exist.`) - } - - // Check if we already have an instance cached - if (!this.teamConfigServiceCache.has(teamName)) { - // If not, create a new one and store it in the cache - this.teamConfigServiceCache.set(teamName, new TeamConfigService(this.repo.teamConfig[teamName])) - } - - // Return the cached instance - return this.teamConfigServiceCache.get(teamName)! - } - - public getApp(id: string): App { - const app = find(this.repo.apps, { id }) - if (!app) { - throw new Error(`App[${id}] does not exist.`) - } - return app - } - - public getApps(): App[] { - return this.repo.apps ?? [] - } - - public updateApp(id: string, updates: Partial): App { - const app = find(this.repo.apps, { id }) - if (!app) { - throw new Error(`App[${id}] does not exist.`) - } - return mergeWith(app, updates, mergeCustomizer) - } - - public deleteApp(id: string): void { - remove(this.repo.apps, { id }) - } - - public createUser(user: User): User { - const newUser = { ...user, id: user.id ?? uuidv4() } - if (find(this.repo.users, { email: newUser.email })) { - throw new AlreadyExists(`User[${user.email}] already exists.`) - } - this.repo.users.push(newUser) - return newUser - } - - public getUser(id: string): User { - const user = find(this.repo.users, { id }) - if (!user) { - throw new Error(`User[${id}] does not exist.`) - } - return user - } - - public getUsers(): User[] { - return this.repo.users ?? [] - } - - public getUsersEmail(): string[] { - return map(this.repo.users, 'email') - } - - public updateUser(id: string, updates: Partial): User { - const user = find(this.repo.users, { id }) - if (!user) throw new Error(`User[${id}] does not exist.`) - return mergeWith(user, updates, mergeCustomizer) - } - - public deleteUser(email: string): void { - remove(this.repo.users, { email }) - } - - private getDefaultTeamConfig(): TeamConfig { - return { - builds: [], - codeRepos: [], - workloads: [], - services: [], - sealedsecrets: [], - knowledgeBases: [], - agents: [], - netpols: [], - settings: {} as AplTeamSettingsResponse, - apps: [], - policies: [], - } - } - - public createTeamConfig(team: AplTeamSettingsRequest): TeamConfig { - const teamName = team.metadata.name - if (has(this.repo.teamConfig, teamName)) { - throw new AlreadyExists(`TeamConfig[${teamName}] already exists.`) - } - const newTeam = this.getDefaultTeamConfig() - newTeam.settings = createAplObject(teamName, team, teamName) as AplTeamSettingsResponse - this.repo.teamConfig[teamName] = newTeam - return this.repo.teamConfig[teamName] - } - - public getTeamConfig(teamName: string): TeamConfig | undefined { - return this.repo.teamConfig[teamName] - } - - public deleteTeamConfig(teamName: string): void { - if (!has(this.repo.teamConfig, teamName)) { - throw new Error(`TeamConfig[${teamName}] does not exist.`) - } - delete this.repo.teamConfig[teamName] - this.teamConfigServiceCache.delete(teamName) - } - - public getCluster(): Cluster { - return this.repo.cluster - } - - public getDns(): Dns { - return this.repo.dns - } - - public getIngress(): Ingress { - return this.repo.ingress - } - - public getOtomi(): Otomi { - return this.repo.otomi - } - - public getSmtp(): Smtp { - return this.repo.smtp - } - - public getObj(): any | undefined { - return this.repo.obj - } - - public getPlatformBackups(): any | undefined { - return this.repo.platformBackups - } - - public getSettings(): Settings { - const settings: Settings = { - alerts: this.repo.alerts, - cluster: this.repo.cluster, - dns: this.repo.dns, - ingress: this.repo.ingress, - kms: this.repo.kms, - obj: this.repo.obj, - otomi: this.repo.otomi, - platformBackups: this.repo.platformBackups, - } - - if (this.repo.smtp) { - settings.smtp = this.repo.smtp - } - if (this.repo.oidc) { - settings.oidc = this.repo.oidc - } - - return settings - } - - public updateSettings(updates: Partial): void { - mergeWith(this.repo, updates, mergeCustomizer) - } - - public getRepo(): Repo { - return this.repo - } - - public setRepo(repo: Repo): void { - this.repo = repo - } - - public getAllTeamSettings(): AplTeamSettingsResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getSettings()) - } - - public getTeamIds(): string[] { - return Object.keys(this.repo.teamConfig) - } - - public getAllNetpols(): AplNetpolResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getNetpols()) - } - - public getAllBuilds(): AplBuildResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getBuilds()) - } - - public getAllPolicies(): AplPolicyResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getPolicies()) - } - - public getAllWorkloads(): AplWorkloadResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getWorkloads()) - } - - public getAllServices(): AplServiceResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getServices()) - } - - public getAllSealedSecrets(): AplSecretResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getSealedSecrets()) - } - - public getAllKnowledgeBases(): AplKnowledgeBaseResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getKnowledgeBases()) - } - - public getAllCodeRepos(): AplCodeRepoResponse[] { - return this.getTeamIds().flatMap((teamName) => this.getTeamConfigService(teamName).getCodeRepos()) - } - - /** Retrieve a collection dynamically from the Repo */ - public getCollection(collectionId: string): any { - if (!has(this.repo, collectionId)) { - throw new Error(`Getting repo collection [${collectionId}] does not exist.`) - } - return this.repo[collectionId] - } - - /** Update a collection dynamically in the Repo */ - public updateCollection(collectionId: string, data: any): void { - if (!has(this.repo, collectionId)) { - throw new Error(`Updating repo collection [${collectionId}] does not exist.`) - } - set(this.repo, collectionId, data) - } -} diff --git a/src/services/TeamConfigService.test.ts b/src/services/TeamConfigService.test.ts deleted file mode 100644 index a22fd3209..000000000 --- a/src/services/TeamConfigService.test.ts +++ /dev/null @@ -1,606 +0,0 @@ -// Mock UUID to generate predictable values -import { AlreadyExists, NotExistError, ValidationError } from '../error' -import { - AplAgentRequest, - AplBuildRequest, - AplKnowledgeBaseRequest, - AplNetpolRequest, - AplSecretRequest, - AplServiceRequest, - AplWorkloadRequest, - App, - TeamConfig, -} from '../otomi-models' -import { TeamConfigService } from './TeamConfigService' - -jest.mock('uuid', () => ({ - v4: jest.fn(() => 'mocked-uuid'), -})) - -describe('TeamConfigService', () => { - let service: TeamConfigService - let teamConfig: TeamConfig - - beforeEach(() => { - const teamSettings = { - kind: 'AplTeamSettingSet', - metadata: { - name: 'team1', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - } - teamConfig = { - builds: [], - codeRepos: [], - workloads: [], - workloadValues: [], - services: [], - sealedsecrets: [], - netpols: [], - apps: [], - policies: [], - knowledgeBases: [], - agents: [], - settings: teamSettings, - } as TeamConfig - service = new TeamConfigService(teamConfig) - }) - - describe('Builds', () => { - const build: AplBuildRequest = { - kind: 'AplTeamBuild', - metadata: { name: 'TestBuild' }, - spec: {}, - } - test('should create a build', () => { - const createdBuild = service.createBuild(build) - - expect(createdBuild).toEqual({ - kind: 'AplTeamBuild', - metadata: { - name: 'TestBuild', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - }) - expect(service.getBuilds()).toHaveLength(1) - }) - - test('should throw an error if creating a duplicate build', () => { - service.createBuild(build) - - expect(() => service.createBuild(build)).toThrow(AlreadyExists) - }) - - test('should retrieve a build by id', () => { - const createdBuild = service.createBuild(build) - - expect(service.getBuild(createdBuild.metadata.name)).toEqual(createdBuild) - }) - - test('should throw an error when retrieving a non-existent build', () => { - expect(() => service.getBuild('non-existent')).toThrow(NotExistError) - }) - - test('should update a build', () => { - const createdBuild = service.createBuild(build) - - const updatedBuild = service.patchBuild(createdBuild.metadata.name, { - metadata: { name: 'UpdatedBuild' }, - }) - expect(updatedBuild.metadata.name).toBe('UpdatedBuild') - }) - - test('should delete a build', () => { - const createdBuild = service.createBuild(build) - - service.deleteBuild(createdBuild.metadata.name) - expect(service.getBuilds()).toHaveLength(0) - }) - }) - - describe('Workloads', () => { - const workload: AplWorkloadRequest = { - kind: 'AplTeamWorkload', - metadata: { - name: 'TestWorkload', - }, - spec: { - url: 'http://test.com', - values: '', - }, - } - test('should create a workload', () => { - const createdWorkload = service.createWorkload(workload) - - expect(createdWorkload).toEqual({ - kind: 'AplTeamWorkload', - metadata: { - name: 'TestWorkload', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: { - url: 'http://test.com', - values: '', - }, - status: {}, - }) - expect(service.getWorkloads()).toHaveLength(1) - }) - - test('should throw an error if creating a duplicate workload', () => { - service.createWorkload(workload) - - expect(() => service.createWorkload(workload)).toThrow(AlreadyExists) - }) - - test('should retrieve a workload by id', () => { - const createdWorkload = service.createWorkload(workload) - - expect(service.getWorkload(createdWorkload.metadata.name)).toEqual(createdWorkload) - }) - - test('should throw an error when retrieving a non-existent workload', () => { - expect(() => service.getWorkload('non-existent')).toThrow(NotExistError) - }) - - test('should delete a workload', () => { - const createdWorkload = service.createWorkload(workload) - - service.deleteWorkload(createdWorkload.metadata.name) - expect(service.getWorkloads()).toHaveLength(0) - }) - }) - - describe('Services', () => { - const serviceData: AplServiceRequest = { - kind: 'AplTeamService', - metadata: { name: 'TestService' }, - spec: {}, - } - test('should create a service', () => { - const createdService = service.createService(serviceData) - - expect(createdService).toEqual({ - kind: 'AplTeamService', - metadata: { - name: 'TestService', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - }) - expect(service.getServices()).toHaveLength(1) - }) - - test('should throw an error if creating a duplicate service', () => { - service.createService(serviceData) - - expect(() => service.createService(serviceData)).toThrow(AlreadyExists) - }) - - test('should retrieve a service by id', () => { - const createdService = service.createService(serviceData) - - expect(service.getService(createdService.metadata.name)).toEqual(createdService) - }) - - test('should throw an error when retrieving a non-existent service', () => { - expect(() => service.getService('non-existent')).toThrow(NotExistError) - }) - - test('should delete a service', () => { - const createdService = service.createService(serviceData) - - service.deleteService(createdService.metadata.name) - expect(service.getServices()).toHaveLength(0) - }) - }) - - describe('SealedSecrets', () => { - const secret: AplSecretRequest = { - kind: 'AplTeamSecret', - metadata: { name: 'TestSecret' }, - spec: { - type: 'kubernetes.io/opaque', - encryptedData: { key: 'value' }, - }, - } - test('should create a sealed secret', () => { - const createdSecret = service.createSealedSecret(secret) - - expect(createdSecret).toEqual({ - kind: 'AplTeamSecret', - metadata: { - name: 'TestSecret', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: { - type: 'kubernetes.io/opaque', - encryptedData: { key: 'value' }, - }, - status: {}, - }) - expect(service.getSealedSecrets()).toHaveLength(1) - }) - - test('should throw an error if creating a duplicate sealed secret', () => { - service.createSealedSecret(secret) - - expect(() => service.createSealedSecret(secret)).toThrow(AlreadyExists) - }) - - test('should retrieve a sealed secret by id', () => { - const createdSecret = service.createSealedSecret(secret) - - expect(service.getSealedSecret(createdSecret.metadata.name)).toEqual(createdSecret) - }) - - test('should throw an error when retrieving a non-existent sealed secret', () => { - expect(() => service.getSealedSecret('non-existent')).toThrow(NotExistError) - }) - - test('should delete a sealed secret', () => { - const createdSecret = service.createSealedSecret(secret) - - service.deleteSealedSecret(createdSecret.metadata.name) - expect(service.getSealedSecrets()).toHaveLength(0) - }) - }) - - describe('Netpols', () => { - const netpol: AplNetpolRequest = { - kind: 'AplTeamNetworkControl', - metadata: { name: 'TestNetpol' }, - spec: {}, - } - - test('should create a netpol', () => { - const created = service.createNetpol(netpol) - expect(created).toEqual({ - kind: 'AplTeamNetworkControl', - metadata: { - name: 'TestNetpol', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - }) - expect(service.getNetpol(created.metadata.name)).toEqual(created) - }) - - test('should throw an error when creating duplicate netpol', () => { - service.createNetpol(netpol) - expect(() => service.createNetpol(netpol)).toThrow(AlreadyExists) - }) - - test('should delete a netpol', () => { - const created = service.createNetpol(netpol) - service.deleteNetpol(created.metadata.name) - expect(() => service.getNetpol(created.metadata.name)).toThrow(NotExistError) - }) - }) - - describe('Apps', () => { - const app: App = { id: 'app1' } - - test('should create an app', () => { - const created = service.createApp(app) - expect(created).toEqual({ id: 'app1' }) - expect(service.getApp(created.id)).toEqual(created) - }) - - test('should throw an error when creating duplicate app', () => { - service.createApp(app) - expect(() => service.createApp(app)).toThrow(AlreadyExists) - }) - }) - - describe('Policies', () => { - test('should retrieve policies', () => { - expect(service.getPolicies()).toEqual([]) - }) - - test('should update policies', () => { - service.patchPolicies('require-limits', { - spec: { action: 'Audit', severity: 'medium' }, - }) - expect(service.getPolicies()).toEqual([ - { - kind: 'AplTeamPolicy', - metadata: { - name: 'require-limits', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: { - action: 'Audit', - severity: 'medium', - }, - status: {}, - }, - ]) - }) - - test('should retrieve a single policy', () => { - service.updatePolicies('require-limits', { - kind: 'AplTeamPolicy', - metadata: { name: 'require-limits' }, - spec: { action: 'Audit', severity: 'medium' }, - }) - expect(service.getPolicy('require-limits')).toEqual({ - kind: 'AplTeamPolicy', - metadata: { - name: 'require-limits', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: { action: 'Audit', severity: 'medium' }, - status: {}, - }) - }) - }) - - describe('Settings', () => { - test('should retrieve settings', () => { - expect(service.getSettings()).toEqual({ - kind: 'AplTeamSettingSet', - metadata: { - labels: { - 'apl.io/teamId': 'team1', - }, - name: 'team1', - }, - spec: {}, - status: {}, - }) - }) - - test('should update settings', () => { - const updated = service.updateSettings({ - kind: 'AplTeamSettingSet', - metadata: { name: 'team1' }, - spec: { networkPolicy: { egressPublic: true } }, - }) - expect(updated).toEqual({ - kind: 'AplTeamSettingSet', - metadata: { - labels: { - 'apl.io/teamId': 'team1', - }, - name: 'team1', - }, - spec: { - networkPolicy: { - egressPublic: true, - }, - }, - status: {}, - }) - expect(service.getSettings().spec.networkPolicy!.egressPublic).toBe(true) - }) - }) - - describe('getCollection', () => { - test('should retrieve an existing collection', () => { - service.getSettings().metadata.name = 'team1' - service.createBuild({ kind: 'AplTeamBuild', metadata: { name: 'TestBuild' }, spec: {} }) - expect(service.getCollection('builds')).toEqual([ - { - kind: 'AplTeamBuild', - metadata: { - name: 'TestBuild', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: {}, - status: {}, - }, - ]) - }) - - test('should throw an error when trying to retrieve a non-existent collection', () => { - expect(() => service.getCollection('nonExistentCollection')).toThrow( - 'Getting TeamConfig collection [nonExistentCollection] does not exist.', - ) - }) - }) - - describe('updateCollection', () => { - test('should update an existing collection', () => { - service.createBuild({ kind: 'AplTeamBuild', metadata: { name: 'Build1' }, spec: {} }) - service.updateCollection('builds', [{ kind: 'AplTeamBuild', metadata: { name: 'UpdatedBuild' } }]) - expect(service.getCollection('builds')).toEqual([{ kind: 'AplTeamBuild', metadata: { name: 'UpdatedBuild' } }]) - }) - - test('should create a new collection if it does not exist', () => { - service.updateCollection('customCollection', [{ key: 'value' }]) - expect(service.getCollection('customCollection')).toEqual([{ key: 'value' }]) - }) - }) - - describe('Agents', () => { - const knowledgeBase: AplKnowledgeBaseRequest = { - kind: 'AkamaiKnowledgeBase', - metadata: { name: 'test-kb' }, - spec: { - modelName: 'text-embedding-model', - sourceUrl: 'https://example.com/data.zip', - }, - } - - const agentWithTools: AplAgentRequest = { - kind: 'AkamaiAgent', - metadata: { name: 'test-agent' }, - spec: { - foundationModel: 'gpt-4', - agentInstructions: 'You are a helpful assistant', - tools: [ - { - type: 'knowledgeBase', - name: 'test-kb', - }, - ], - }, - } - - const agentWithoutTools: AplAgentRequest = { - kind: 'AkamaiAgent', - metadata: { name: 'simple-agent' }, - spec: { - foundationModel: 'gpt-4', - agentInstructions: 'You are a helpful assistant', - }, - } - - test('should create an agent with tools', () => { - // First create the knowledge base - service.createKnowledgeBase(knowledgeBase) - - const created = service.createAgent(agentWithTools) - expect(created).toEqual({ - kind: 'AkamaiAgent', - metadata: { - name: 'test-agent', - labels: { - 'apl.io/teamId': 'team1', - }, - }, - spec: { - foundationModel: 'gpt-4', - agentInstructions: 'You are a helpful assistant', - tools: [ - { - type: 'knowledgeBase', - name: 'test-kb', - }, - ], - }, - status: {}, - }) - expect(service.getAgents()).toHaveLength(1) - }) - - test('should create an agent without tools', () => { - const created = service.createAgent(agentWithoutTools) - expect(created.spec.tools).toBeUndefined() - expect(service.getAgents()).toHaveLength(1) - }) - - test('should throw validation error when knowledge base does not exist', () => { - expect(() => service.createAgent(agentWithTools)).toThrow(ValidationError) - expect(() => service.createAgent(agentWithTools)).toThrow('KnowledgeBase[test-kb] does not exist.') - }) - - test('should throw an error when creating duplicate agent', () => { - service.createKnowledgeBase(knowledgeBase) - service.createAgent(agentWithTools) - expect(() => service.createAgent(agentWithTools)).toThrow(AlreadyExists) - }) - - test('should retrieve an agent by name', () => { - service.createKnowledgeBase(knowledgeBase) - const created = service.createAgent(agentWithTools) - const retrieved = service.getAgent(created.metadata.name) - - // getAgent transforms the agent through AkamaiAgentCR, so we check key properties - expect(retrieved.metadata.name).toBe(created.metadata.name) - expect(retrieved.spec.foundationModel).toBe(created.spec.foundationModel) - expect(retrieved.spec.tools?.length).toBe(1) - expect(retrieved.spec.tools?.[0].name).toBe('test-kb') - }) - - test('should throw an error when retrieving a non-existent agent', () => { - expect(() => service.getAgent('non-existent')).toThrow(NotExistError) - }) - - test('should update an agent with new tools', () => { - service.createKnowledgeBase(knowledgeBase) - const created = service.createAgent(agentWithoutTools) - - const updated = service.updateAgent(created.metadata.name, { - kind: 'AkamaiAgent', - metadata: { name: 'simple-agent' }, - spec: { - foundationModel: 'gpt-4', - agentInstructions: 'Updated instructions', - tools: [ - { - type: 'knowledgeBase', - name: 'test-kb', - }, - ], - }, - }) - - expect(updated.spec.agentInstructions).toBe('Updated instructions') - expect(updated.spec.tools).toEqual([ - { - type: 'knowledgeBase', - name: 'test-kb', - }, - ]) - }) - - test('should throw validation error when updating with non-existent knowledge base', () => { - const created = service.createAgent(agentWithoutTools) - - expect(() => - service.updateAgent(created.metadata.name, { - kind: 'AkamaiAgent', - metadata: { name: 'simple-agent' }, - spec: { - foundationModel: 'gpt-4', - agentInstructions: 'Updated instructions', - tools: [ - { - type: 'knowledgeBase', - name: 'non-existent-kb', - }, - ], - }, - }), - ).toThrow(ValidationError) - }) - - test('should patch an agent', () => { - service.createKnowledgeBase(knowledgeBase) - const created = service.createAgent(agentWithTools) - - const patched = service.patchAgent(created.metadata.name, { - spec: { - agentInstructions: 'Patched instructions', - }, - }) - - expect(patched.spec.agentInstructions).toBe('Patched instructions') - expect(patched.spec.foundationModel).toBe('gpt-4') // Original value preserved - }) - - test('should delete an agent', () => { - service.createKnowledgeBase(knowledgeBase) - const created = service.createAgent(agentWithTools) - - service.deleteAgent(created.metadata.name) - expect(service.getAgents()).toHaveLength(0) - }) - }) -}) diff --git a/src/services/TeamConfigService.ts b/src/services/TeamConfigService.ts deleted file mode 100644 index 21e95d18d..000000000 --- a/src/services/TeamConfigService.ts +++ /dev/null @@ -1,567 +0,0 @@ -import { cloneDeep, find, has, merge, omit, remove, set } from 'lodash' -import { v4 as uuidv4 } from 'uuid' -import { AkamaiAgentCR } from '../ai/AkamaiAgentCR' -import { AkamaiKnowledgeBaseCR } from '../ai/AkamaiKnowledgeBaseCR' -import { AlreadyExists, NotExistError, ValidationError } from '../error' -import { - AplAgentRequest, - AplAgentResponse, - AplBuildRequest, - AplBuildResponse, - AplCodeRepoRequest, - AplCodeRepoResponse, - AplKnowledgeBaseRequest, - AplKnowledgeBaseResponse, - AplNetpolRequest, - AplNetpolResponse, - AplPolicyRequest, - AplPolicyResponse, - AplRequestObject, - AplResponseObject, - AplSecretRequest, - AplSecretResponse, - AplServiceRequest, - AplServiceResponse, - AplTeamSettingsRequest, - AplTeamSettingsResponse, - AplWorkloadRequest, - AplWorkloadResponse, - App, - DeepPartial, - TeamConfig, -} from '../otomi-models' -import { createAplObject, getAplMergeObject, updateAplObject } from '../utils/manifests' - -function mergeCustomizer(prev, next) { - return next -} - -export class TeamConfigService { - constructor(private teamConfig: TeamConfig) { - this.teamConfig.codeRepos ??= [] - this.teamConfig.builds ??= [] - this.teamConfig.workloads ??= [] - this.teamConfig.services ??= [] - this.teamConfig.sealedsecrets ??= [] - this.teamConfig.knowledgeBases ??= [] - this.teamConfig.agents ??= [] - this.teamConfig.netpols ??= [] - this.teamConfig.apps ??= [] - this.teamConfig.policies ??= [] - } - - private createAplObject(name: string, request: AplRequestObject): AplResponseObject { - return createAplObject(name, request, this.teamConfig.settings.metadata.name) - } - - // ===================================== - // == BUILDS CRUD == - // ===================================== - - public createBuild(build: AplBuildRequest): AplBuildResponse { - const { name } = build.metadata - if (find(this.teamConfig.builds, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`Build[${name}] already exists.`) - } - - const newBuild = this.createAplObject(name, build) as AplBuildResponse - this.teamConfig.builds.push(newBuild) - return newBuild - } - - public getBuild(name: string): AplBuildResponse { - const build = find(this.teamConfig.builds, (item) => item.metadata.name === name) - if (!build) { - throw new NotExistError(`Build[${name}] does not exist.`) - } - return build - } - - public getBuilds(): AplBuildResponse[] { - return this.teamConfig.builds ?? [] - } - - public updateBuild(name: string, updates: AplBuildRequest): AplBuildResponse { - const build = this.getBuild(name) - return updateAplObject(build, updates) as AplBuildResponse - } - - public patchBuild(name: string, updates: DeepPartial): AplBuildResponse { - const build = this.getBuild(name) - const mergeObj = getAplMergeObject(updates) - return merge(build, mergeObj) - } - - public deleteBuild(name: string): void { - remove(this.teamConfig.builds, (item) => item.metadata.name === name) - } - - // ===================================== - // == CODEREPOS CRUD == - // ===================================== - - public createCodeRepo(codeRepo: AplCodeRepoRequest): AplCodeRepoResponse { - const { name } = codeRepo.metadata - if (find(this.teamConfig.codeRepos, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`CodeRepo[${name}] already exists.`) - } - - const newCodeRepo = this.createAplObject(name, codeRepo) as AplCodeRepoResponse - this.teamConfig.codeRepos.push(newCodeRepo) - return newCodeRepo - } - - public getCodeRepo(name: string): AplCodeRepoResponse { - const codeRepo = find(this.teamConfig.codeRepos, (item) => item.metadata.name === name) - if (!codeRepo) { - throw new NotExistError(`CodeRepo[${name}] does not exist.`) - } - return codeRepo - } - - public getCodeRepos(): AplCodeRepoResponse[] { - return this.teamConfig.codeRepos ?? [] - } - - public updateCodeRepo(name: string, updates: AplCodeRepoRequest): AplCodeRepoResponse { - const codeRepo = this.getCodeRepo(name) - return updateAplObject(codeRepo, updates) as AplCodeRepoResponse - } - - public patchCodeRepo(name: string, updates: DeepPartial): AplCodeRepoResponse { - const codeRepo = this.getCodeRepo(name) - const mergeObj = getAplMergeObject(updates) - return merge(codeRepo, mergeObj) - } - - public deleteCodeRepo(name: string): void { - remove(this.teamConfig.codeRepos, (item) => item.metadata.name === name) - } - - // ===================================== - // == WORKLOADS CRUD == - // ===================================== - - public createWorkload(workload: AplWorkloadRequest): AplWorkloadResponse { - const { name } = workload.metadata - if (find(this.teamConfig.workloads, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`Workload[${name}] already exists.`) - } - - const newWorkload = this.createAplObject(name, workload) as AplWorkloadResponse - this.teamConfig.workloads.push(newWorkload) - return newWorkload - } - - public getWorkload(name: string): AplWorkloadResponse { - const workload = find(this.teamConfig.workloads, (item) => item.metadata.name === name) - if (!workload) { - throw new NotExistError(`Workload[${name}] does not exist.`) - } - return workload - } - - public getWorkloads(): AplWorkloadResponse[] { - return (this.teamConfig.workloads ?? []).map((workload) => omit(workload, 'spec.values')) - } - - public updateWorkload(name: string, updates: AplWorkloadRequest): AplWorkloadResponse { - const workload = this.getWorkload(name) - return updateAplObject(workload, updates) as AplWorkloadResponse - } - - public patchWorkload(name: string, updates: DeepPartial): AplWorkloadResponse { - const workload = this.getWorkload(name) - const mergeObj = getAplMergeObject(updates) - return merge(workload, mergeObj) - } - - public deleteWorkload(name: string): void { - remove(this.teamConfig.workloads, (item) => item.metadata.name === name) - } - - // ===================================== - // == SERVICES CRUD == - // ===================================== - - public createService(service: AplServiceRequest): AplServiceResponse { - const { name } = service.metadata - if (find(this.teamConfig.services, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`Service[${name}] already exists.`) - } - - const newService = this.createAplObject(name, service) as AplServiceResponse - this.teamConfig.services.push(newService) - return newService - } - - public getService(name: string): AplServiceResponse { - const service = find(this.teamConfig.services, (item) => item.metadata.name === name) - if (!service) { - throw new NotExistError(`Service[${name}] does not exist.`) - } - return service - } - - public getServices(): AplServiceResponse[] { - return this.teamConfig.services ?? [] - } - - public updateService(name: string, updates: AplServiceRequest): AplServiceResponse { - const service = this.getService(name) - return updateAplObject(service, updates) as AplServiceResponse - } - - public patchService(name: string, updates: DeepPartial): AplServiceResponse { - const service = this.getService(name) - const mergeObj = getAplMergeObject(updates) - return merge(service, mergeObj) - } - - public deleteService(name: string): void { - remove(this.teamConfig.services, (item) => item.metadata.name === name) - } - - // ===================================== - // == SEALED SECRETS CRUD == - // ===================================== - - public createSealedSecret(secret: AplSecretRequest): AplSecretResponse { - const { name } = secret.metadata - if (find(this.teamConfig.sealedsecrets, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`SealedSecret[${name}] already exists.`) - } - - const newSecret = this.createAplObject(name, secret) as AplSecretResponse - this.teamConfig.sealedsecrets.push(newSecret) - return newSecret - } - - public getSealedSecret(name: string): AplSecretResponse { - const secret = find(this.teamConfig.sealedsecrets, (item) => item.metadata.name === name) - if (!secret) { - throw new NotExistError(`SealedSecret[${name}] does not exist.`) - } - return secret - } - - public getSealedSecrets(): AplSecretResponse[] { - return this.teamConfig.sealedsecrets ?? [] - } - - public updateSealedSecret(name: string, updates: AplSecretRequest): AplSecretResponse { - const secret = this.getSealedSecret(name) - return updateAplObject(secret, updates) as AplSecretResponse - } - - public patchSealedSecret(name: string, updates: DeepPartial): AplSecretResponse { - const secret = this.getSealedSecret(name) - const mergeObj = getAplMergeObject(updates) - return merge(secret, mergeObj) - } - - public deleteSealedSecret(name: string): void { - remove(this.teamConfig.sealedsecrets, (item) => item.metadata.name === name) - } - - // ===================================== - // == KNOWLEDGE BASE CRUD == - // ===================================== - - public createKnowledgeBase(knowledgeBase: AplKnowledgeBaseRequest): AplKnowledgeBaseResponse { - const { name } = knowledgeBase.metadata - if (find(this.teamConfig.knowledgeBases, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`KnowledgeBase[${name}] already exists.`) - } - - const newKnowledgeBase = this.createAplObject(name, knowledgeBase) as AplKnowledgeBaseResponse - this.teamConfig.knowledgeBases.push(newKnowledgeBase) - return newKnowledgeBase - } - - public getKnowledgeBase(name: string): AplKnowledgeBaseResponse { - const knowledgeBase = find(this.teamConfig.knowledgeBases, (item) => item.metadata.name === name) - if (!knowledgeBase) { - throw new NotExistError(`KnowledgeBase[${name}] does not exist.`) - } - // If the knowledge base has pipeline parameters, it's a full CR that needs transformation - if (knowledgeBase.spec && 'pipelineParameters' in knowledgeBase.spec) { - return AkamaiKnowledgeBaseCR.fromCR(knowledgeBase as any).toApiResponse(this.teamConfig.settings.metadata.name) - } - return knowledgeBase - } - - public getKnowledgeBases(): AplKnowledgeBaseResponse[] { - const knowledgeBases = this.teamConfig.knowledgeBases ?? [] - return knowledgeBases.map((kb) => { - // If the knowledge base has pipeline parameters, it's a full CR that needs transformation - if (kb.spec && 'pipelineParameters' in kb.spec) { - return AkamaiKnowledgeBaseCR.fromCR(kb as any).toApiResponse(this.teamConfig.settings.metadata.name) - } - return kb - }) - } - - public updateKnowledgeBase(name: string, updates: AplKnowledgeBaseRequest): AplKnowledgeBaseResponse { - const knowledgeBase = this.getKnowledgeBase(name) - return updateAplObject(knowledgeBase, updates) as AplKnowledgeBaseResponse - } - - public patchKnowledgeBase(name: string, updates: DeepPartial): AplKnowledgeBaseResponse { - const knowledgeBase = this.getKnowledgeBase(name) - const mergeObj = getAplMergeObject(updates) - return merge(knowledgeBase, mergeObj) - } - - public deleteKnowledgeBase(name: string): void { - remove(this.teamConfig.knowledgeBases, (item) => item.metadata.name === name) - } - - public validateKnowledgeBaseExists(knowledgeBaseName: string): boolean { - const knowledgeBases = this.teamConfig.knowledgeBases ?? [] - return knowledgeBases.some((kb) => kb.metadata.name === knowledgeBaseName) - } - - // ===================================== - // == AGENT CRUD == - // ===================================== - - public createAgent(agent: AplAgentRequest): AplAgentResponse { - const { name } = agent.metadata - if (find(this.teamConfig.agents, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`Agent[${name}] already exists.`) - } - - // Validate that knowledgeBase exists if specified in tools - if (agent.spec.tools) { - for (const tool of agent.spec.tools) { - if (tool.type === 'knowledgeBase' && tool.name) { - const toolName = tool.name as string - if (!this.validateKnowledgeBaseExists(toolName)) { - throw new ValidationError(`KnowledgeBase[${toolName}] does not exist.`) - } - } - } - } - - const newAgent = this.createAplObject(name, agent) as AplAgentResponse - this.teamConfig.agents.push(newAgent) - return newAgent - } - - public getAgent(name: string): AplAgentResponse { - const agent = find(this.teamConfig.agents, (item) => item.metadata.name === name) - if (!agent) { - throw new NotExistError(`Agent[${name}] does not exist.`) - } - if (agent.spec && 'foundationModel' in agent.spec) { - return AkamaiAgentCR.fromCR(agent as any).toApiResponse(this.teamConfig.settings.metadata.name) - } - return agent - } - - public getAgents(): AplAgentResponse[] { - return this.teamConfig.agents.map((agent) => { - if (agent.spec && 'foundationModel' in agent.spec) { - return AkamaiAgentCR.fromCR(agent as any).toApiResponse(this.teamConfig.settings.metadata.name) - } - return agent - }) - } - - public updateAgent(name: string, updates: AplAgentRequest): AplAgentResponse { - const agent = this.getAgent(name) - // Validate that knowledgeBase exists if specified in tools - if (updates.spec.tools) { - for (const tool of updates.spec.tools) { - if (tool.type === 'knowledgeBase' && tool.name) { - const toolName = tool.name as string - if (!this.validateKnowledgeBaseExists(toolName)) { - throw new ValidationError(`KnowledgeBase[${toolName}] does not exist.`) - } - } - } - } - return updateAplObject(agent, updates) as AplAgentResponse - } - - public patchAgent(name: string, updates: DeepPartial): AplAgentResponse { - const agent = this.getAgent(name) - // Validate that knowledgeBase exists if specified in tools - if (updates.spec?.tools) { - for (const tool of updates.spec.tools) { - if (tool && tool.type === 'knowledgeBase' && typeof tool.name === 'string') { - const toolName = tool.name as string - if (!this.validateKnowledgeBaseExists(toolName)) { - throw new ValidationError(`KnowledgeBase[${toolName}] does not exist.`) - } - } - } - } - const mergeObj = getAplMergeObject(updates) - return merge(agent, mergeObj) - } - - public deleteAgent(name: string): void { - remove(this.teamConfig.agents, (item) => item.metadata.name === name) - } - - // ===================================== - // == NETPOLS CRUD == - // ===================================== - - public createNetpol(netpol: AplNetpolRequest): AplNetpolResponse { - const { name } = netpol.metadata - if (find(this.teamConfig.netpols, (item) => item.metadata.name === name)) { - throw new AlreadyExists(`Netpol[${name}] already exists.`) - } - - const newNetpol = this.createAplObject(name, netpol) as AplNetpolResponse - this.teamConfig.netpols.push(newNetpol) - return newNetpol - } - - public getNetpol(name: string): AplNetpolResponse { - const netpol = find(this.teamConfig.netpols, (item) => item.metadata.name === name) - if (!netpol) { - throw new NotExistError(`Netpol[${name}] does not exist.`) - } - return netpol - } - - public getNetpols(): AplNetpolResponse[] { - return this.teamConfig.netpols ?? [] - } - - public updateNetpol(name: string, updates: AplNetpolRequest): AplNetpolResponse { - const netpol = this.getNetpol(name) - return updateAplObject(netpol, updates) as AplNetpolResponse - } - - public patchNetpol(name: string, updates: DeepPartial): AplNetpolResponse { - const netpol = this.getNetpol(name) - const mergeObj = getAplMergeObject(updates) - return merge(netpol, mergeObj) - } - - public deleteNetpol(name: string): void { - remove(this.teamConfig.netpols, (item) => item.metadata.name === name) - } - - // ===================================== - // == SETTINGS CRUD == - // ===================================== - - public getSettings(): AplTeamSettingsResponse { - return this.teamConfig.settings - } - - public updateSettings(updates: DeepPartial): AplTeamSettingsResponse { - Object.assign(this.teamConfig.settings.spec, cloneDeep(updates.spec)) - return this.teamConfig.settings - } - - public patchSettings(updates: DeepPartial): AplTeamSettingsResponse { - const mergeObj = getAplMergeObject(updates) - return merge(this.teamConfig.settings, mergeObj) - } - - // ===================================== - // == APPS CRUD == - // ===================================== - - public createApp(app: App): App { - this.teamConfig.apps ??= [] - const newApp = { ...app, id: app.id ?? uuidv4() } - if (find(this.teamConfig.apps, { id: newApp.id })) { - throw new AlreadyExists(`App[${app.id}] already exists.`) - } - this.teamConfig.apps.push(newApp) - return newApp - } - - public getApp(id: string): App { - const app = find(this.teamConfig.apps, { id }) - if (!app) { - throw new NotExistError(`App[${id}] does not exist.`) - } - return app - } - - public getApps(): App[] { - const teamId = this.teamConfig.settings?.metadata?.name - return (this.teamConfig.apps ?? []).map((app) => ({ - ...app, - teamId, - })) - } - - public setApps(apps: App[]) { - this.teamConfig.apps = apps - } - - // ===================================== - // == POLICIES CRUD == - // ===================================== - - public getPolicy(name: string): AplPolicyResponse { - const policy = find(this.teamConfig.policies, (item) => item.metadata.name === name) - if (!policy) { - throw new NotExistError(`Policy[${name}] does not exist.`) - } - return policy - } - - public getPolicies(): AplPolicyResponse[] { - return this.teamConfig.policies ?? [] - } - - public updatePolicies(name: string, updates: AplPolicyRequest): AplPolicyResponse { - const policy = find(this.teamConfig.policies, (item) => item.metadata.name === name) - if (!policy) { - const newPolicy = this.createAplObject(name, { - metadata: { name }, - kind: 'AplTeamPolicy', - spec: updates.spec, - }) as AplPolicyResponse - this.teamConfig.policies.push(newPolicy) - return newPolicy - } else { - Object.assign(policy.spec, updates.spec) - return policy - } - } - - public patchPolicies(name: string, updates: DeepPartial): AplPolicyResponse { - const policy = find(this.teamConfig.policies, (item) => item.metadata.name === name) - if (!policy) { - const newPolicy = this.createAplObject(name, { - metadata: { name }, - kind: 'AplTeamPolicy', - spec: { - action: updates.spec?.action || 'Audit', - severity: updates.spec?.severity || 'medium', - }, - }) as AplPolicyResponse - this.teamConfig.policies.push(newPolicy) - return newPolicy - } else { - const mergeObj = getAplMergeObject(updates) - return merge(policy, mergeObj) - } - } - - /** Retrieve a collection dynamically from the Teamconfig - * Try not to use this function */ - public getCollection(collectionId: string): AplResponseObject[] { - if (!has(this.teamConfig, collectionId)) { - throw new Error(`Getting TeamConfig collection [${collectionId}] does not exist.`) - } - return this.teamConfig[collectionId] as any - } - - /** Update a collection dynamically in the Teamconfig */ - public updateCollection(collectionId: string, data: any): void { - set(this.teamConfig, collectionId, data) - } -} diff --git a/src/utils/manifests.ts b/src/utils/manifests.ts index 716d911ff..435fbbce0 100644 --- a/src/utils/manifests.ts +++ b/src/utils/manifests.ts @@ -1,5 +1,6 @@ import { cloneDeep, merge, omit } from 'lodash' import { AplKind, AplRequestObject, AplResponseObject, DeepPartial, ServiceSpec, V1ApiObject } from '../otomi-models' +import { App } from 'supertest/types' export function getAplObjectFromV1(kind: AplKind, spec: V1ApiObject | ServiceSpec): AplRequestObject { return { @@ -10,9 +11,18 @@ export function getAplObjectFromV1(kind: AplKind, spec: V1ApiObject | ServiceSpe spec, } as AplRequestObject } +export function getAplObjectFromApp(kind: AplKind, spec: App, name: string): AplRequestObject { + return { + kind, + metadata: { + name, + }, + spec, + } as AplRequestObject +} export function getV1ObjectFromApl(aplObject: AplResponseObject): V1ApiObject { - const teamId = aplObject.metadata.labels['apl.io/teamId'] + const teamId = aplObject.metadata.labels?.['apl.io/teamId'] return { name: aplObject.metadata.name, ...(teamId && { teamId }), From 26e65e076ea36c784e24d36cdf44b457788dd965 Mon Sep 17 00:00:00 2001 From: Cas Lubbers Date: Thu, 27 Nov 2025 13:28:47 +0100 Subject: [PATCH 2/5] feat: fix deleting user --- src/git.ts | 89 +------ src/otomi-stack.ts | 22 +- src/playground.ts | 18 -- src/repo.ts | 587 --------------------------------------------- 4 files changed, 14 insertions(+), 702 deletions(-) delete mode 100755 src/repo.ts diff --git a/src/git.ts b/src/git.ts index 519e166ea..dd6732494 100644 --- a/src/git.ts +++ b/src/git.ts @@ -5,8 +5,7 @@ import { rmSync } from 'fs' import { copy, ensureDir, pathExists, readFile, writeFile } from 'fs-extra' import { unlink } from 'fs/promises' import { glob } from 'glob' -import jsonpath from 'jsonpath' -import { cloneDeep, get, isEmpty, merge, set, unset } from 'lodash' +import { isEmpty, merge } from 'lodash' import { basename, dirname, join } from 'path' import simpleGit, { CheckRepoActions, CleanOptions, CommitResult, ResetMode, SimpleGit } from 'simple-git' import { @@ -23,7 +22,6 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml' import { BASEURL } from './constants' import { GitPullError, HttpError, ValidationError } from './error' import { Core } from './otomi-models' -import { FileMap, getFilePath, getResourceName, renderManifest, renderManifestForSecrets } from './repo' import { getSanitizedErrorMessage, removeBlankAttributes, sanitizeGitPassword } from './utils' const debug = Debug('otomi:repo') @@ -237,91 +235,6 @@ export class Git { return merge(data, secretData) as Core } - async saveConfig( - config: Record, - fileMap: FileMap, - unsetBlankAttributes?: boolean, - ): Promise> { - let jsonPathsValuesPublic - if (fileMap.kind === 'AplTeamPolicy') { - jsonPathsValuesPublic = jsonpath.nodes(config, '$.teamConfig.*.*') - } else { - jsonPathsValuesPublic = jsonpath.nodes(config, fileMap.jsonPathExpression) - } - await Promise.all( - jsonPathsValuesPublic.map(async (node) => { - const nodePath = node.path - const nodeValue = node.value - try { - const filePath = getFilePath(fileMap, nodePath, nodeValue, '') - const manifest = fileMap.v2 ? nodeValue : renderManifest(fileMap, nodePath, nodeValue) - await this.writeFile(filePath, manifest, unsetBlankAttributes) - } catch (e) { - console.log(nodePath) - console.log(fileMap) - throw e - } - }), - ) - } - - async saveConfigWithSecrets( - config: Record, - secretJsonPaths: string[], - fileMap: FileMap, - ): Promise> { - const secretData = {} - const plainData = cloneDeep(config) - secretJsonPaths.forEach((objectPath) => { - const val = get(config, objectPath) - if (val) { - set(secretData, fileMap.v2 ? objectPath.replace('.spec', '') : objectPath, val) - unset(plainData, objectPath) - } - }) - - await this.saveConfig(plainData, fileMap) - await this.saveSecretConfig(secretData, fileMap) - } - - async saveSecretConfig(secretConfig: Record, fileMap: FileMap, unsetBlankAttributes?: boolean) { - const jsonPathsValuesSecrets = jsonpath.nodes(secretConfig, fileMap.jsonPathExpression) - await Promise.all( - jsonPathsValuesSecrets.map(async (node) => { - const nodePath = node.path - const nodeValue = node.value - try { - const filePath = getFilePath(fileMap, nodePath, nodeValue, 'secrets.') - const resourceName = getResourceName(fileMap, nodePath, nodeValue) - const manifest = renderManifestForSecrets(fileMap, resourceName, nodeValue) - await this.writeFile(filePath, manifest, unsetBlankAttributes) - } catch (e) { - console.log(nodePath) - console.log(fileMap) - throw e - } - }), - ) - } - - async deleteConfig(config: Record, fileMap: FileMap, fileNamePrefix = '') { - const jsonPathsValuesSecrets = jsonpath.nodes(config, fileMap.jsonPathExpression) - await Promise.all( - jsonPathsValuesSecrets.map(async (node) => { - const nodePath = node.path - const nodeValue = node.value - try { - const filePath = getFilePath(fileMap, nodePath, nodeValue, fileNamePrefix) - await this.removeFile(filePath) - } catch (e) { - console.log(nodePath) - console.log(fileMap) - throw e - } - }), - ) - } - isRootClone(): boolean { return this.path === env.GIT_LOCAL_PATH } diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index ab6f9a17d..0e20ae0d2 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -18,7 +18,6 @@ import { ValidationError, } from 'src/error' import getRepo, { getWorktreeRepo, Git } from 'src/git' -import { getFileMaps } from 'src/repo' import { FileStore } from 'src/fileStore/file-store' import { getSettingsFileMaps } from 'src/fileStore/file-map' import { cleanSession, getSessionStack } from 'src/middleware' @@ -1029,7 +1028,7 @@ export default class OtomiStack { throw new ForbiddenError('Cannot delete the default platform admin user') } - await this.deleteUserFile(user) + await this.deleteUserFile(id) await this.doDeleteDeployment([filePath], false) } @@ -2435,14 +2434,19 @@ export default class OtomiStack { return { filePath, content: aplPlatformObject } } - async deleteUserFile(user: User): Promise { - debug(`Deleting user ${user.email}`) - const filePath = getResourceFilePath('AplUser', user.email) + async deleteUserFile(userId: string): Promise { + debug(`Deleting user ${userId}`) + const filePath = getResourceFilePath('AplUser', userId) + this.fileStore.delete(filePath) - const users: User[] = [] - users.push(user) - const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplUser')! - await this.git.deleteConfig({ users }, fileMap, 'secrets.') + + await this.git.removeFile(filePath) + + const secretFilePath = getSecretFilePath(filePath) + const secretExists = await this.git.fileExists(secretFilePath) + if (secretExists) { + await this.git.removeFile(secretFilePath) + } } async saveTeam(aplTeamObject: AplTeamObject, secretPaths?: string[]): Promise { diff --git a/src/playground.ts b/src/playground.ts index 7799644c5..8fd608742 100644 --- a/src/playground.ts +++ b/src/playground.ts @@ -1,26 +1,8 @@ #!/usr/bin/env node --nolazy -r ts-node/register -r tsconfig-paths/register -import { getFileMaps, getFilePath, loadValues } from './repo' -import { Repo } from './otomi-models' - async function play() { // Suppose your environment directory is "my-environment" const envDir = '/private/tmp/otomi-bootstrap-dev' - - const allSpecs = await loadValues(envDir) - - const repo = allSpecs as Repo - - const build = repo.teamConfig['demo'].builds[0] - const jsonPath = ['$', 'teamConfig', 'demo'] - const filePath = getFilePath( - getFileMaps(envDir).find((filemap) => filemap.kind === 'AplTeamBuild')!, - jsonPath, - build, - '', - ) - console.log(allSpecs) - console.log(filePath) } play() diff --git a/src/repo.ts b/src/repo.ts deleted file mode 100755 index 34ed93148..000000000 --- a/src/repo.ts +++ /dev/null @@ -1,587 +0,0 @@ -import { rmSync } from 'fs' -import { rm } from 'fs/promises' -import { globSync } from 'glob' -import jsonpath from 'jsonpath' -import { cloneDeep, get, merge, omit, set } from 'lodash' -import path from 'path' -import { AplKind } from './otomi-models' -import { getDirNames, loadYaml } from './utils' - -export async function getTeamNames(envDir: string): Promise> { - const teamsDir = path.join(envDir, 'env', 'teams') - return await getDirNames(teamsDir, { skipHidden: true }) -} - -export interface FileMap { - envDir: string - kind: AplKind - jsonPathExpression: string - pathGlob: string - processAs: 'arrayItem' | 'mapItem' - resourceGroup: 'team' | 'platformSettings' | 'platformApps' | 'platformDatabases' | 'platformBackups' | 'users' - resourceDir: string - loadToSpec: boolean - v2: boolean -} - -export function getResourceFileName(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record) { - let fileName = 'unknown' - if (fileMap.resourceGroup === 'team') { - if (fileMap.processAs === 'arrayItem') { - fileName = (fileMap.v2 ? data.metadata.name || data.metadata.id : data.name || data.id) || fileName - } else { - fileName = jsonPath[jsonPath.length - 1].toString() - } - } else { - if (fileMap.processAs === 'arrayItem') { - fileName = (fileMap.v2 ? data.metadata.name || data.metadata.id : data.name || data.id) || fileName - } else { - fileName = jsonPath[jsonPath.length - 1].toString() - } - } - return fileName -} - -export function getTeamNameFromJsonPath(jsonPath: jsonpath.PathComponent[]): string { - return jsonPath[2].toString() -} - -export function getResourceName(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record) { - let resourceName = 'unknown' - if (fileMap.processAs === 'arrayItem') { - resourceName = (fileMap.v2 ? data.metadata.name || data.metadata.id : data.name || data.id) || resourceName - return resourceName - } - - // Custom workaround for teamPolicy because it is a mapItem - if (fileMap.resourceGroup === 'team' && fileMap.kind !== 'AplTeamPolicy') { - resourceName = getTeamNameFromJsonPath(jsonPath) - return resourceName - } else { - resourceName = jsonPath[jsonPath.length - 1].toString() - return resourceName - } -} - -export function getFilePath( - fileMap: FileMap, - jsonPath: jsonpath.PathComponent[], - data: Record, - fileNamePrefix: string, -) { - let filePath = '' - const resourceName = getResourceFileName(fileMap, jsonPath, data) - if (fileMap.resourceGroup === 'team') { - const teamName = getTeamNameFromJsonPath(jsonPath) - filePath = `${fileMap.envDir}/env/teams/${teamName}/${fileMap.resourceDir}/${fileNamePrefix}${resourceName}.yaml` - } else { - filePath = `${fileMap.envDir}/env/${fileMap.resourceDir}/${fileNamePrefix}${resourceName}.yaml` - } - // normalize paths like /ab/c/./test/yaml - return path.normalize(filePath) -} - -export function extractTeamDirectory(filePath: string): string { - const match = filePath.match(/\/teams\/([^/]+)/) - if (match === null) throw new Error(`Cannot extract team name from ${filePath} string`) - return match[1] -} - -export function getJsonPath(fileMap: FileMap, filePath: string): string { - let { jsonPathExpression: jsonPath } = fileMap - - if (fileMap.resourceGroup === 'team') { - const teamName = extractTeamDirectory(filePath) - jsonPath = jsonPath.replace('teamConfig.*', `teamConfig.${teamName}`) - } - - if (jsonPath.includes('.*')) { - const fileName = path.basename(filePath, path.extname(filePath)) - const strippedFileName = fileName.replace(/^secrets\.|\.yaml|\.dec$/g, '') - jsonPath = jsonPath.replace('.*', `.${strippedFileName}`) - } - if (jsonPath.includes('[*]')) jsonPath = jsonPath.replace('[*]', '') - jsonPath = jsonPath.replace('$.', '') - return jsonPath -} - -export function getFileMaps(envDir: string): Array { - const maps: Array = [ - { - kind: 'AplApp', - envDir, - jsonPathExpression: '$.apps.*', - pathGlob: `${envDir}/env/apps/*.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformApps', - resourceDir: 'apps', - loadToSpec: true, - v2: false, - }, - { - envDir, - kind: 'AplAlertSet', - jsonPathExpression: '$.alerts', - pathGlob: `${envDir}/env/settings/*alerts.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplCluster', - envDir, - jsonPathExpression: '$.cluster', - pathGlob: `${envDir}/env/settings/cluster.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplDatabase', - envDir, - jsonPathExpression: '$.databases.*', - pathGlob: `${envDir}/env/databases/*.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformDatabases', - resourceDir: 'databases', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplDns', - envDir, - jsonPathExpression: '$.dns', - pathGlob: `${envDir}/env/settings/*dns.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplIngress', - envDir, - jsonPathExpression: '$.ingress', - pathGlob: `${envDir}/env/settings/ingress.yaml`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplKms', - envDir, - jsonPathExpression: '$.kms', - pathGlob: `${envDir}/env/settings/*kms.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplObjectStorage', - envDir, - jsonPathExpression: '$.obj', - pathGlob: `${envDir}/env/settings/*obj.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplIdentityProvider', - envDir, - jsonPathExpression: '$.oidc', - pathGlob: `${envDir}/env/settings/*oidc.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplCapabilitySet', - envDir, - jsonPathExpression: '$.otomi', - pathGlob: `${envDir}/env/settings/*otomi.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplBackupCollection', - envDir, - jsonPathExpression: '$.platformBackups', - pathGlob: `${envDir}/env/settings/*platformBackups.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformBackups', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplSmtp', - envDir, - jsonPathExpression: '$.smtp', - pathGlob: `${envDir}/env/settings/*smtp.{yaml,yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplUser', - envDir, - jsonPathExpression: '$.users[*]', - pathGlob: `${envDir}/env/users/*.{yaml,yaml.dec}`, - processAs: 'arrayItem', - resourceGroup: 'users', - resourceDir: 'users', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplVersion', - envDir, - jsonPathExpression: '$.versions', - pathGlob: `${envDir}/env/settings/versions.yaml`, - processAs: 'mapItem', - resourceGroup: 'platformSettings', - resourceDir: 'settings', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplTeamCodeRepo', - envDir, - jsonPathExpression: '$.teamConfig.*.codeRepos[*]', - pathGlob: `${envDir}/env/teams/*/codeRepos/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'codeRepos', - loadToSpec: false, - v2: true, - }, - { - kind: 'AplTeamBuild', - envDir, - jsonPathExpression: '$.teamConfig.*.builds[*]', - pathGlob: `${envDir}/env/teams/*/builds/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'builds', - loadToSpec: true, - v2: true, - }, - { - kind: 'AplTeamWorkload', - envDir, - jsonPathExpression: '$.teamConfig.*.workloads[*]', - pathGlob: `${envDir}/env/teams/*/workloads/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'workloads', - loadToSpec: true, - v2: true, - }, - { - kind: 'AplTeamWorkloadValues', - envDir, - jsonPathExpression: '$.teamConfig.*.workloadValues[*]', - pathGlob: `${envDir}/env/teams/*/workloadValues/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'workloadValues', - loadToSpec: false, - v2: false, - }, - { - kind: 'AplTeamService', - envDir, - jsonPathExpression: '$.teamConfig.*.services[*]', - pathGlob: `${envDir}/env/teams/*/services/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'services', - loadToSpec: true, - v2: true, - }, - { - kind: 'AplTeamSecret', - envDir, - jsonPathExpression: '$.teamConfig.*.sealedsecrets[*]', - pathGlob: `${envDir}/env/teams/*/sealedsecrets/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'sealedsecrets', - loadToSpec: false, - v2: true, - }, - { - kind: 'AkamaiKnowledgeBase', - envDir, - jsonPathExpression: '$.teamConfig.*.knowledgeBases[*]', - pathGlob: `${envDir}/env/teams/*/knowledgebases/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'knowledgebases', - loadToSpec: false, - v2: true, - }, - { - kind: 'AkamaiAgent', - envDir, - jsonPathExpression: '$.teamConfig.*.agents[*]', - pathGlob: `${envDir}/env/teams/*/agents/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'agents', - loadToSpec: false, - v2: true, - }, - { - kind: 'AplTeamNetworkControl', - envDir, - jsonPathExpression: '$.teamConfig.*.netpols[*]', - pathGlob: `${envDir}/env/teams/*/netpols/*.yaml`, - processAs: 'arrayItem', - resourceGroup: 'team', - resourceDir: 'netpols', - loadToSpec: true, - v2: true, - }, - { - kind: 'AplTeamSettingSet', - envDir, - jsonPathExpression: '$.teamConfig.*.settings', - pathGlob: `${envDir}/env/teams/*/*settings{.yaml,.yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'team', - resourceDir: '.', - loadToSpec: true, - v2: true, - }, - { - kind: 'AplTeamTool', - envDir, - jsonPathExpression: '$.teamConfig.*.apps', - pathGlob: `${envDir}/env/teams/*/*apps{.yaml,.yaml.dec}`, - processAs: 'mapItem', - resourceGroup: 'team', - resourceDir: '.', - loadToSpec: true, - v2: false, - }, - { - kind: 'AplTeamPolicy', - envDir, - jsonPathExpression: '$.teamConfig.*.policies[*]', - pathGlob: `${envDir}/env/teams/*/policies/*.yaml`, - processAs: 'mapItem', - resourceGroup: 'team', - resourceDir: 'policies', - loadToSpec: true, - v2: true, - }, - ] - return maps -} - -export function hasCorrespondingDecryptedFile(filePath: string, fileList: Array): boolean { - return fileList.includes(`${filePath}.dec`) -} - -export function getFileMap(kind: AplKind, envDir: string): FileMap { - const fileMaps = getFileMaps(envDir) - const fileMapFiltered = fileMaps.find((fileMap) => fileMap.kind === kind) - return fileMapFiltered! -} - -export function renderManifest(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record) { - let spec = data - const labels = {} - if (fileMap.resourceGroup === 'team') { - spec = omit(data, ['id', 'name', 'teamId']) - labels['apl.io/teamId'] = getTeamNameFromJsonPath(jsonPath) - } - return { - kind: fileMap.kind, - metadata: { - name: getResourceName(fileMap, jsonPath, data), - labels, - }, - spec, - } -} - -export function renderManifestForSecrets(fileMap: FileMap, resourceName: string, data: Record) { - return { - kind: fileMap.kind, - metadata: { - name: resourceName, - }, - spec: omit(data, ['id', 'teamId', 'name']), - } -} - -export async function unsetValuesFile(envDir: string): Promise { - const valuesPath = path.join(envDir, 'values-repo.yaml') - await rm(valuesPath, { force: true }) - return valuesPath -} - -export function unsetValuesFileSync(envDir: string): string { - const valuesPath = path.join(envDir, 'values-repo.yaml') - rmSync(valuesPath, { force: true }) - return valuesPath -} - -function isKindValid(kind: string | undefined, fileMap: FileMap): boolean { - return ( - kind === fileMap.kind || - (kind === 'SealedSecret' && fileMap.kind === 'AplTeamSecret') || - fileMap.kind === 'AplTeamWorkloadValues' - ) -} - -function isNameValid(data: Record, fileMap: FileMap, fileName: string | undefined): boolean { - //TODO Remove users exception once name has been set in metadata consistently - return ( - fileMap.resourceGroup === 'users' || fileMap.kind === 'AplTeamWorkloadValues' || data.metadata?.name === fileName - ) -} - -function isTeamValid(data: Record, fileMap: FileMap, teamName: string | undefined): boolean { - return ['AplTeamWorkloadValues', 'AplTeamSecret'].includes(fileMap.kind) || data?.metadata?.labels?.['apl.io/teamId'] -} - -export async function loadFileToSpec( - filePath: string, - fileMap: FileMap, - spec: Record, - deps = { loadYaml }, -): Promise { - const jsonPath = getJsonPath(fileMap, filePath) - try { - const data = (await deps.loadYaml(filePath)) || {} - // ensure that local path does not include envDir and the leading slash - const localFilePath = filePath.replace(fileMap.envDir, '').replace(/^\/+/, '') - // eslint-disable-next-line no-param-reassign - spec.files[localFilePath] = data - if (fileMap.processAs === 'arrayItem') { - const ref: Record[] = get(spec, jsonPath) - const name = filePath.match(/\/([^/]+)\.yaml$/)?.[1] - if (!isKindValid(data?.kind, fileMap)) { - console.error(`Unexpected manifest kind in ${filePath}: ${data?.kind}`) - return - } - if (!isNameValid(data, fileMap, name)) { - console.error(`Unexpected name in ${filePath}: ${data.metadata?.name}`) - return - } - if (fileMap.kind !== 'AplTeamWorkloadValues' && fileMap.resourceGroup === 'team') { - const teamName = filePath.match(/\/env\/teams\/([^/]+)\//)?.[1] - if (!isTeamValid(data, fileMap, teamName)) { - console.error(`Unexpected team in ${filePath}: ${data?.metadata?.labels?.['apl.io/teamId']}`) - return - } - } - // TODO: Remove workaround for User currently relying on id in console - if (fileMap.kind === 'AplUser') { - data.spec.id = data.metadata?.name - } - if (fileMap.kind === 'AplTeamWorkloadValues') { - //TODO remove this custom workaround for workloadValues as it has no spec - ref.push({ ...data, name }) - } else if (fileMap.v2) { - ref.push(data) - } else { - ref.push(data?.spec) - } - } else if (fileMap.kind === 'AplTeamPolicy') { - const ref: Record = get(spec, jsonPath) - const policy = { - [data?.metadata?.name]: data?.spec, - } - const newRef = merge(cloneDeep(ref), policy) - set(spec, jsonPath, newRef) - } else { - const ref: Record = get(spec, jsonPath) - let newRef - if (fileMap.v2) { - newRef = merge(cloneDeep(ref), data) - } else { - newRef = merge(cloneDeep(ref), data?.spec) - } - // Decrypted secrets may need to be merged with plain text specs - set(spec, jsonPath, newRef) - } - } catch (e) { - console.log(filePath) - console.log(fileMap) - throw e - } -} - -export function initSpec(fileMap: FileMap, jsonPath: string, spec: Record) { - if (fileMap.processAs === 'arrayItem') { - set(spec, jsonPath, []) - } else { - set(spec, jsonPath, {}) - } -} - -export async function loadToSpec( - spec: Record, - fileMap: FileMap, - deps = { loadFileToSpec }, -): Promise { - const globOptions = { - nodir: true, // Exclude directories - dot: false, - } - const files: string[] = globSync(fileMap.pathGlob, globOptions).sort() - const promises: Promise[] = [] - - files.forEach((filePath) => { - const jsonPath = getJsonPath(fileMap, filePath) - initSpec(fileMap, jsonPath, spec) - if (hasCorrespondingDecryptedFile(filePath, files)) return - promises.push(deps.loadFileToSpec(filePath, fileMap, spec)) - }) - - await Promise.all(promises) -} - -export async function loadValues(envDir: string, deps = { loadToSpec }): Promise> { - //We need everything to load to spec for the API - const fileMaps = getFileMaps(envDir) - const spec = { - files: {}, - } - - await Promise.all( - fileMaps.map(async (fileMap) => { - await deps.loadToSpec(spec, fileMap) - }), - ) - - return spec -} - -export async function getKmsSettings(envDir: string, deps = { loadToSpec }): Promise> { - const kmsFiles = getFileMap('AplKms', envDir) - const spec = {} - await deps.loadToSpec(spec, kmsFiles) - return spec -} From db5a7e452d6d3d915c1e807ceb6d858002113106 Mon Sep 17 00:00:00 2001 From: Cas Lubbers Date: Thu, 27 Nov 2025 14:56:01 +0100 Subject: [PATCH 3/5] feat: add Architecture.md --- ARCHITECTURE.md | 527 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 527 insertions(+) create mode 100644 ARCHITECTURE.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 000000000..99b57e9f0 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,527 @@ +# FileStore Architecture Documentation + +This document provides visual representations of the FileStore architecture and data flow patterns in the Otomi API. + +## Table of Contents +1. [High-Level Architecture](#1-high-level-architecture) +2. [Object Transformation Flow](#2-object-transformation-flow) +3. [CRUD Operation Flows](#3-crud-operation-flows) +4. [Secret File Handling](#4-secret-file-handling) +5. [FileStore Load Process](#5-filestore-load-process) +6. [File Path Generation](#6-file-path-generation) + +--- + +## 1. High-Level Architecture + +This diagram shows the relationship between key components and data types. + +```mermaid +classDiagram + class OtomiStack { + +FileStore fileStore + +Git git + +createCodeRepo(teamId, data) + +getCodeRepo(teamId, name) + +editCodeRepo(teamId, name, data) + +deleteCodeRepo(teamId, name) + } + + class FileStore { + -Map~string, AplObject~ store + +load(envDir) FileStore + +get(filePath) AplObject + +set(filePath, content) AplRecord + +delete(filePath) string + +getTeamResource(kind, teamId, name) AplObject + +setTeamResource(aplTeamObject) string + +deleteTeamResource(kind, teamId, name) string + +getByKind(kind, teamId) Map + } + + class Git { + +writeFile(path, content) + +removeFile(path) + +save(editor, encryptSecrets, files) + +fileExists(path) boolean + } + + class AplObject { + <> + +AplKind kind + +metadata + +spec Record + +status? Record + } + + class AplTeamObject { + <> + +AplKind kind + +metadata with teamId label + +spec Record + } + + class AplPlatformObject { + <> + +AplKind kind + +metadata + +spec Record + } + + class AplRequestObject { + <> + AplServiceRequest | AplCodeRepoRequest | ... + } + + class AplResponseObject { + <> + AplServiceResponse | AplCodeRepoResponse | ... + } + + OtomiStack --> FileStore : uses + OtomiStack --> Git : uses + FileStore --> AplObject : stores + AplTeamObject --|> AplObject : extends + AplPlatformObject --|> AplObject : extends + AplRequestObject ..> AplTeamObject : transforms to + AplResponseObject ..> AplObject : based on + + note for FileStore "In-Memory Cache\nMerged secrets\nFast access" + note for Git "On-Disk Storage\nSecrets split\nVersion controlled" +``` + +--- + +## 2. Object Transformation Flow + +This diagram shows how objects transform as they move through the system. + +```mermaid +flowchart TB + subgraph "V2 API Layer" + V2Request[AplRequestObject
AplCodeRepoRequest, etc.
kind + metadata + spec] + end + + subgraph "Request Transformation" + toTeam["toTeamObject()"] + end + + subgraph "Storage Layer" + AplTeam[AplTeamObject
+ teamId label] + FileStore[(FileStore
In-Memory)] + Git[(Git Repository
On-Disk)] + end + + subgraph "V2 API Response" + AplResp[AplResponseObject
AplCodeRepoResponse, etc.
kind + metadata + spec + status] + end + + %% CREATE/UPDATE Flow + V2Request -->|"1. Add teamId label"| toTeam + toTeam --> AplTeam + AplTeam -->|"2. Store (with secrets)"| FileStore + AplTeam -->|"3. Write (secrets split)"| Git + + %% READ Flow + FileStore -->|"4. Retrieve directly"| AplResp +``` + +--- + +## 3. CRUD Operation Flows + +### 3.1 CREATE Flow + +```mermaid +sequenceDiagram + participant API as API Endpoint + participant Stack as OtomiStack + participant FS as FileStore + participant Git as Git + participant Root as RootStack + + API->>Stack: createAplCodeRepo(teamId, aplRequest) + + Note over Stack: Add teamId label + Stack->>Stack: toTeamObject(teamId, aplRequest) + + Note over Stack: Save to storage + Stack->>FS: setTeamResource(aplTeamObject) + FS-->>Stack: filePath + + Stack->>Git: writeFile(filePath, aplTeamObject) + Git-->>Stack: success + + Note over Stack: Deploy changes + Stack->>Stack: doDeployment(aplRecord) + Stack->>Git: save(editor, encryptSecrets, files) + Stack->>Git: pull() + + Note over Stack: Update root FileStore + Stack->>Root: fileStore.set(filePath, content) + + Stack-->>API: AplCodeRepoResponse +``` + +### 3.2 READ Flow + +```mermaid +sequenceDiagram + participant API as API Endpoint + participant Stack as OtomiStack + participant FS as FileStore + + API->>Stack: getAplCodeRepo(teamId, name) + + Note over Stack: Generate file path + Stack->>Stack: getResourceFilePath(kind, name, teamId) + + Stack->>FS: getTeamResource(kind, teamId, name) + + Note over FS: Lookup in in-memory Map + FS->>FS: store.get(filePath) + FS-->>Stack: AplObject (with merged secrets) + + Stack-->>API: AplCodeRepoResponse +``` + +### 3.3 UPDATE Flow + +```mermaid +sequenceDiagram + participant API as API Endpoint + participant Stack as OtomiStack + participant FS as FileStore + participant Git as Git + participant Root as RootStack + + API->>Stack: editAplCodeRepo(teamId, name, aplRequest) + + Note over Stack: Get existing resource + Stack->>FS: getTeamResource(kind, teamId, name) + FS-->>Stack: existing AplObject + + Note over Stack: Merge specs + Stack->>Stack: merge(existing.spec, aplRequest.spec) + + Note over Stack: Build updated object + Stack->>Stack: buildTeamObject(existing, updatedSpec) + + Stack->>FS: setTeamResource(updatedObject) + FS-->>Stack: filePath + + Stack->>Git: writeFile(filePath, updatedObject) + + Note over Stack: Deploy + Stack->>Stack: doDeployment(aplRecord) + Stack->>Root: fileStore.set(filePath, content) + + Stack-->>API: AplCodeRepoResponse +``` + +### 3.4 DELETE Flow + +```mermaid +sequenceDiagram + participant API as API Endpoint + participant Stack as OtomiStack + participant FS as FileStore + participant Git as Git + participant Root as RootStack + + API->>Stack: deleteCodeRepo(teamId, name) + + Note over Stack: Delete from FileStore + Stack->>FS: deleteTeamResource(kind, teamId, name) + FS->>FS: store.delete(filePath) + FS-->>Stack: filePath + + Note over Stack: Delete from Git + Stack->>Git: removeFile(filePath) + Git-->>Stack: success + + Note over Stack: Check for secret file + Stack->>Stack: getSecretFilePath(filePath) + Stack->>Git: fileExists(secretFilePath) + + alt Secret file exists + Stack->>Git: removeFile(secretFilePath) + end + + Note over Stack: Deploy deletion + Stack->>Stack: doDeleteDeployment([filePath]) + Stack->>Git: save(editor, encryptSecrets) + Stack->>Root: fileStore.delete(filePath) + + Stack-->>API: void +``` + +--- + +## 4. Secret File Handling + +### 4.1 Secret Extraction and Storage + +```mermaid +flowchart TB + subgraph "Input" + AplObj[AplObject with secrets
spec: name, password, apiKey] + SecretPaths[Secret Paths
password, apiKey] + end + + subgraph "saveWithSecrets Process" + Extract[Extract secrets
from spec using paths] + Split{Split data} + MainSpec[Main Spec
name only] + SecretSpec[Secret Spec
password, apiKey] + end + + subgraph "FileStore Storage" + FSFull[Store FULL object
with secrets merged] + end + + subgraph "Git Storage" + GitMain[Write main file
env/teams/demo/services/api.yaml
spec: name] + GitSecret[Write secret file
env/teams/demo/secrets.services/api.yaml
spec: password, apiKey] + end + + AplObj --> Extract + SecretPaths --> Extract + Extract --> Split + + Split --> MainSpec + Split --> SecretSpec + + AplObj --> FSFull + + MainSpec --> GitMain + SecretSpec --> GitSecret +``` + +### 4.2 Secret Path Extraction Functions + +```mermaid +flowchart LR + subgraph "Global Secret Paths" + Global["apps.harbor.adminPassword
dns.provider
kms.accessKey
teamConfig...settings.apiToken"] + end + + subgraph "Extraction Functions" + ExtractApp["extractAppSecretPaths(appName)
Filter: apps.{appName}.*
Strip prefix"] + ExtractSettings["extractSettingsSecretPaths(kind)
Filter by prefix map
AplDns → dns.*
AplKms → kms.*"] + ExtractTeam["extractTeamSecretPaths()
Filter: teamConfig.pattern...*
Strip prefix"] + end + + subgraph "Resource-Specific Paths" + AppPaths["adminPassword"] + SettingsPaths["provider
accessKey"] + TeamPaths["apiToken"] + end + + Global --> ExtractApp + Global --> ExtractSettings + Global --> ExtractTeam + + ExtractApp --> AppPaths + ExtractSettings --> SettingsPaths + ExtractTeam --> TeamPaths +``` + +### 4.3 Secret Merge During Load + +```mermaid +flowchart TB + subgraph "On Disk - Git Repository" + MainFile["env/teams/demo/settings.yaml
kind: AplTeamSettingSet
spec: name='demo'"] + SecretFile["env/teams/demo/secrets.settings.yaml
kind: AplTeamSettingSet
spec: apiToken='ENC...'"] + end + + subgraph "FileStore.load() - PASS 1" + Load1[Load all YAML files
Validate with Zod] + TempMap["Temporary Map
settings.yaml → spec with name
secrets.settings.yaml → spec with apiToken"] + end + + subgraph "FileStore.load() - PASS 2" + Detect{Detect secret file
path includes '/secrets.'} + GetMain[Get main file
mainPath = path replace '/secrets.' with '/'] + Merge[Deep merge specs
lodash merge main and secret] + Delete[Delete secret entry
from temp map] + end + + subgraph "In-Memory - FileStore" + Final["settings.yaml
kind: AplTeamSettingSet
spec: name='demo', apiToken='ENC...'"] + end + + MainFile --> Load1 + SecretFile --> Load1 + Load1 --> TempMap + + TempMap --> Detect + Detect -->|"Yes"| GetMain + GetMain --> Merge + Merge --> Delete + Delete --> Final +``` + +--- + +## 5. FileStore Load Process + +Complete two-pass loading and merging process. + +```mermaid +flowchart TB + Start([Start: FileStore.load]) + + subgraph "PASS 1: Load All Files" + GetMaps[Get FileMaps for all AplKinds] + Glob[Glob for files matching patterns
.yaml and .yaml.dec] + LoadYAML[Load YAML content] + LogWarning[Warning logged failed validation] + + CheckSkip{Skip validation?
sealedsecrets/
workloadValues/} + Validate[Validate with Zod
AplObjectSchema] + ValidFail{Valid?} + + StoreTemp[Store in temporary Map
relativePath -> AplObject] + end + + subgraph "PASS 2: Merge Secrets" + Iterate[Iterate all temp files] + IsSecret{Path contains
'/secrets.'?} + + GetMainPath[mainPath = path
.replace '/secrets.', '/'] + MainExists{Main file
exists?} + + DeepMerge[Deep merge secret.spec
into main.spec
merge, main.spec, secret.spec] + + UseAsMain[Store secret as main
Special case: users] + + RemoveSecret[Remove secret file
from temp map] + end + + subgraph "Final Storage" + StoreAll[Store all merged files
in FileStore.store Map
'env/teams/builds/build-1.yaml' -> AplObject] + ReturnFS[Return FileStore instance] + end + + Start --> GetMaps + GetMaps --> Glob + Glob --> LoadYAML + + LoadYAML --> CheckSkip + CheckSkip -->|No| Validate + CheckSkip -->|Yes| StoreTemp + + Validate --> ValidFail + ValidFail -->|Yes| StoreTemp + ValidFail -->|No| LogWarning + + StoreTemp --> Iterate + + Iterate --> IsSecret + IsSecret -->|No| Iterate + IsSecret -->|Yes| GetMainPath + + GetMainPath --> MainExists + MainExists -->|Yes| DeepMerge + MainExists -->|No| UseAsMain + + DeepMerge --> RemoveSecret + UseAsMain --> RemoveSecret + + RemoveSecret --> Iterate + + Iterate --> StoreAll + StoreAll --> ReturnFS + ReturnFS --> End([End]) +``` + +--- + +## 6. File Path Generation + +### 6.1 Path Generation Process + +```mermaid +flowchart LR + subgraph "Input" + Int["kind: AplTeamService
name: 'api'
teamId: 'demo'"] + end + + subgraph "getResourceFilePath()" + GetMap["Get FileMap for kind
getFileMapForKind(kind)"] + Template["pathTemplate:
'env/teams/{teamId}/services/{name}.yaml'"] + Replace1["Replace {teamId}
→ 'demo'"] + Replace2["Replace {name}
→ 'api'"] + end + + subgraph "Output" + Out["'env/teams/demo/services/api.yaml'"] + end + + Input --> GetMap + GetMap --> Template + Template --> Replace1 + Replace1 --> Replace2 + Replace2 --> Output + +``` + +### 6.2 Secret File Path Derivation + +```mermaid +flowchart LR + subgraph "Main File Path" + Main["'env/teams/demo/settings.yaml'"] + end + + subgraph "getSecretFilePath()" + Split["Split by '/'
['env', 'teams', 'demo', 'settings.yaml']"] + ExtractDir["dir = parts.slice(0, -1)
['env', 'teams', 'demo']"] + ExtractFile["file = parts[parts.length-1]
'settings.yaml'"] + Prepend["Prepend 'secrets.'
'secrets.settings.yaml'"] + Join["Join with '/'"] + end + + subgraph "Secret File Path" + Secret["'env/teams/demo/secrets.settings.yaml'"] + end + + Main --> Split + Split --> ExtractDir + Split --> ExtractFile + ExtractFile --> Prepend + ExtractDir --> Join + Prepend --> Join + Join --> Secret +``` + +--- + +## Key Takeaways + +### In-Memory vs On-Disk +- **FileStore (In-Memory)**: Stores complete objects with secrets merged. Fast access for all operations. +- **Git (On-Disk)**: Stores main files without secrets + separate secret files. Version controlled and encrypted. + +### Object Transformation Pattern +``` +V2 API: AplRequest → AplTeamObject → FileStore → AplResponse + (add label) (store) (retrieve) +``` + +### Secret Handling Pattern +``` +Write: Split secrets → Main file + Secret file (on disk) +Load: Merge secrets → Complete object (in memory) +Access: Always use FileStore (has merged secrets) +``` + +### CRUD Pattern +``` +CREATE: Transform → FileStore.set → Git.writeFile → Deploy +READ: FileStore.get → Transform +UPDATE: Get → Merge → FileStore.set → Git.writeFile → Deploy +DELETE: FileStore.delete → Git.removeFile → Deploy +``` From 9cd873c4a032ae272e8bf794a98ce377bd54df15 Mon Sep 17 00:00:00 2001 From: Cas Lubbers Date: Mon, 1 Dec 2025 11:09:47 +0100 Subject: [PATCH 4/5] fix: getting enabled apps --- src/otomi-stack.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 0e20ae0d2..c9fff9535 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -576,9 +576,11 @@ export default class OtomiStack { if (!picks) return teamApps if (picks.includes('enabled')) { - teamApps = allApps.map((adminApp) => { - const teamApp = teamApps.find((app) => app.id === adminApp.id) - return teamApp || { id: adminApp.id, enabled: adminApp.enabled } + return teamApps.map((teamApp) => { + return { + id: teamApp?.id, + enabled: Boolean(teamApp?.values?.enabled ?? true), + } }) } From 3249e24e0b35799a89268cdaa8a83d88aa1fbdfb Mon Sep 17 00:00:00 2001 From: Cas Lubbers Date: Mon, 1 Dec 2025 12:46:02 +0100 Subject: [PATCH 5/5] fix: saving sealed secrets --- src/otomi-stack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index c9fff9535..c90506387 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -163,7 +163,7 @@ export const rootPath = '/tmp/otomi/values' const clusterSettingsFilePath = 'env/settings/cluster.yaml' function getTeamSealedSecretsValuesFilePath(teamId: string, sealedSecretsName: string): string { - return `env/teams/${teamId}/sealedsecrets/${sealedSecretsName}` + return `env/teams/${teamId}/sealedsecrets/${sealedSecretsName}.yaml` } function getTeamWorkloadValuesManagedFilePath(teamId: string, workloadName: string): string {