From c524277555d3cbfc8cb8dccf82ae05540509b9fc Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 2 Dec 2024 15:16:28 +0100 Subject: [PATCH 1/9] Factorize services loading --- src/collection-api/routes/index.js | 9 +- src/collection-api/routes/services.js | 196 +++++++++++++------------- src/collection-api/server.js | 2 +- 3 files changed, 105 insertions(+), 102 deletions(-) diff --git a/src/collection-api/routes/index.js b/src/collection-api/routes/index.js index 43cce838a..1e675127b 100644 --- a/src/collection-api/routes/index.js +++ b/src/collection-api/routes/index.js @@ -1,11 +1,14 @@ import express from 'express'; import helmet from 'helmet'; +import * as Services from '../../archivist/services/index.js'; + import docsRouter from './docs.js'; +import metadataRouter from './metadata.js'; import servicesRouter from './services.js'; import versionsRouter from './versions.js'; -export default function apiRouter(basePath) { +export default async function apiRouter(basePath) { const router = express.Router(); const defaultDirectives = helmet.contentSecurityPolicy.getDefaultDirectives(); @@ -27,7 +30,9 @@ export default function apiRouter(basePath) { res.json({ message: 'Welcome to an instance of the Open Terms Archive API. Documentation is available at /docs. Learn more on Open Terms Archive on https://opentermsarchive.org.' }); }); - router.use(servicesRouter); + const services = await Services.load(); + + router.use(servicesRouter(services)); router.use(versionsRouter); return router; diff --git a/src/collection-api/routes/services.js b/src/collection-api/routes/services.js index daa56ff3d..55ef658f9 100644 --- a/src/collection-api/routes/services.js +++ b/src/collection-api/routes/services.js @@ -1,9 +1,5 @@ import express from 'express'; -import * as Services from '../../archivist/services/index.js'; - -const services = await Services.load(); - /** * @swagger * tags: @@ -54,106 +50,108 @@ const services = await Services.load(); * items: * type: string */ -const router = express.Router(); +export default function servicesRouter(services) { + const router = express.Router(); -/** - * @swagger - * /services: - * get: - * summary: Enumerate all services. - * tags: [Services] - * produces: - * - application/json - * responses: - * 200: - * description: A JSON array of all services. - * content: - * application/json: - * schema: - * type: array - * items: - * type: object - * properties: - * id: - * type: string - * description: The ID of the service. - * name: - * type: string - * description: The name of the service. - * terms: - * type: array - * description: The declared terms types for this service. - * items: - * type: object - * properties: - * type: - * type: string - * description: The type of terms. - */ -router.get('/services', (req, res) => { - res.status(200).json(Object.values(services).map(service => ({ - id: service.id, - name: service.name, - terms: service.getTermsTypes().map(type => ({ type })), - }))); -}); + /** + * @swagger + * /services: + * get: + * summary: Enumerate all services. + * tags: [Services] + * produces: + * - application/json + * responses: + * 200: + * description: A JSON array of all services. + * content: + * application/json: + * schema: + * type: array + * items: + * type: object + * properties: + * id: + * type: string + * description: The ID of the service. + * name: + * type: string + * description: The name of the service. + * terms: + * type: array + * description: The declared terms types for this service. + * items: + * type: object + * properties: + * type: + * type: string + * description: The type of terms. + */ + router.get('/services', (req, res) => { + res.status(200).json(Object.values(services).map(service => ({ + id: service.id, + name: service.name, + terms: service.getTermsTypes().map(type => ({ type })), + }))); + }); -/** - * @swagger - * /service/{serviceId}: - * get: - * summary: Retrieve the declaration of a specific service through its ID. - * tags: [Services] - * produces: - * - application/json - * parameters: - * - in: path - * name: serviceId - * description: The ID of the service. - * schema: - * type: string - * required: true - * examples: - * service-1: - * value: service-1 - * summary: Simple service ID - * service-2: - * value: Service 2! - * summary: Service ID with spaces and special characters - * responses: - * 200: - * description: The full JSON declaration of the service with the given ID. - * content: - * application/json: - * schema: - * $ref: '#/components/schemas/Service' - * 404: - * description: No service matching the provided ID is found. - */ -router.get('/service/:serviceId', (req, res) => { - const matchedServiceID = Object.keys(services).find(key => key.toLowerCase() === req.params.serviceId?.toLowerCase()); - const service = services[matchedServiceID]; + /** + * @swagger + * /service/{serviceId}: + * get: + * summary: Retrieve the declaration of a specific service through its ID. + * tags: [Services] + * produces: + * - application/json + * parameters: + * - in: path + * name: serviceId + * description: The ID of the service. + * schema: + * type: string + * required: true + * examples: + * service-1: + * value: service-1 + * summary: Simple service ID + * service-2: + * value: Service 2! + * summary: Service ID with spaces and special characters + * responses: + * 200: + * description: The full JSON declaration of the service with the given ID. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Service' + * 404: + * description: No service matching the provided ID is found. + */ + router.get('/service/:serviceId', (req, res) => { + const matchedServiceID = Object.keys(services).find(key => key.toLowerCase() === req.params.serviceId?.toLowerCase()); + const service = services[matchedServiceID]; - if (!service) { - res.status(404).send('Service not found'); + if (!service) { + res.status(404).send('Service not found'); - return; - } + return; + } - res.status(200).json({ - id: service.id, - name: service.name, - terms: service.getTerms().map(terms => ({ - type: terms.type, - sourceDocuments: terms.sourceDocuments.map(({ location, contentSelectors, insignificantContentSelectors, filters, executeClientScripts }) => ({ - location, - contentSelectors, - insignificantContentSelectors, - executeClientScripts, - filters: filters?.map(filter => filter.name), + res.status(200).json({ + id: service.id, + name: service.name, + terms: service.getTerms().map(terms => ({ + type: terms.type, + sourceDocuments: terms.sourceDocuments.map(({ location, contentSelectors, insignificantContentSelectors, filters, executeClientScripts }) => ({ + location, + contentSelectors, + insignificantContentSelectors, + executeClientScripts, + filters: filters?.map(filter => filter.name), + })), })), - })), + }); }); -}); -export default router; + return router; +} diff --git a/src/collection-api/server.js b/src/collection-api/server.js index 4d8996a9d..d76c8ab6f 100644 --- a/src/collection-api/server.js +++ b/src/collection-api/server.js @@ -15,7 +15,7 @@ if (process.env.NODE_ENV !== 'test') { const BASE_PATH = `/${config.get('@opentermsarchive/engine.collection-api.basePath')}/v1`.replace(/\/\/+/g, '/'); // ensure there are no double slashes -app.use(BASE_PATH, apiRouter(BASE_PATH)); +app.use(BASE_PATH, await apiRouter(BASE_PATH)); app.use(errorsMiddleware); const port = config.get('@opentermsarchive/engine.collection-api.port'); From 48dd9ff38a77580cd801489a921d12ec37ab8b7a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 2 Dec 2024 15:17:36 +0100 Subject: [PATCH 2/9] Make `servicesIds` parameter optional --- src/archivist/services/service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archivist/services/service.js b/src/archivist/services/service.js index 950574a3e..cc0d97bf0 100644 --- a/src/archivist/services/service.js +++ b/src/archivist/services/service.js @@ -70,6 +70,6 @@ export default class Service { } static getNumberOfTerms(services, servicesIds, termsTypes) { - return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0); + return (servicesIds || Object.keys(services)).reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0); } } From 6190a3cdbba5630c6981d962ae7fddd3079db654 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 2 Dec 2024 17:15:51 +0100 Subject: [PATCH 3/9] Define config for root path of collection Replace `services.declarationsPath` --- config/default.json | 4 +-- config/test.json | 4 +-- scripts/declarations/lint/index.mocha.js | 4 +-- scripts/declarations/validate/index.mocha.js | 4 +-- src/archivist/services/index.js | 6 ++-- test/test-declarations/README.yml | 7 ++++ .../declarations}/Service B!.json | 0 ...ervice_with_declaration_history.filters.js | 0 ...vice_with_declaration_history.history.json | 0 .../service_with_declaration_history.json | 0 ...ce_with_filters_history.filters.history.js | 0 .../service_with_filters_history.filters.js | 0 .../service_with_filters_history.json | 0 .../service_with_history.filters.history.js | 0 .../service_with_history.filters.js | 0 .../service_with_history.history.json | 0 .../declarations}/service_with_history.json | 0 ...multiple_source_documents_terms.filters.js | 0 ...ltiple_source_documents_terms.history.json | 0 ..._with_multiple_source_documents_terms.json | 0 .../service_without_history.filters.js | 0 .../service_without_history.json | 0 .../declarations/service\302\267A.json" | 0 test/test-declarations/metadata.yml | 33 +++++++++++++++++++ 24 files changed, 49 insertions(+), 13 deletions(-) create mode 100644 test/test-declarations/README.yml rename test/{services => test-declarations/declarations}/Service B!.json (100%) rename test/{services => test-declarations/declarations}/service_with_declaration_history.filters.js (100%) rename test/{services => test-declarations/declarations}/service_with_declaration_history.history.json (100%) rename test/{services => test-declarations/declarations}/service_with_declaration_history.json (100%) rename test/{services => test-declarations/declarations}/service_with_filters_history.filters.history.js (100%) rename test/{services => test-declarations/declarations}/service_with_filters_history.filters.js (100%) rename test/{services => test-declarations/declarations}/service_with_filters_history.json (100%) rename test/{services => test-declarations/declarations}/service_with_history.filters.history.js (100%) rename test/{services => test-declarations/declarations}/service_with_history.filters.js (100%) rename test/{services => test-declarations/declarations}/service_with_history.history.json (100%) rename test/{services => test-declarations/declarations}/service_with_history.json (100%) rename test/{services => test-declarations/declarations}/service_with_multiple_source_documents_terms.filters.js (100%) rename test/{services => test-declarations/declarations}/service_with_multiple_source_documents_terms.history.json (100%) rename test/{services => test-declarations/declarations}/service_with_multiple_source_documents_terms.json (100%) rename test/{services => test-declarations/declarations}/service_without_history.filters.js (100%) rename test/{services => test-declarations/declarations}/service_without_history.json (100%) rename "test/services/service\302\267A.json" => "test/test-declarations/declarations/service\302\267A.json" (100%) create mode 100644 test/test-declarations/metadata.yml diff --git a/config/default.json b/config/default.json index 2d18aeac3..3026217f3 100644 --- a/config/default.json +++ b/config/default.json @@ -1,9 +1,7 @@ { "@opentermsarchive/engine": { "trackingSchedule": "30 */12 * * *", - "services": { - "declarationsPath": "./declarations" - }, + "collectionPath": "./", "recorder": { "versions": { "storage": { diff --git a/config/test.json b/config/test.json index 49142b92f..cf14b8be3 100644 --- a/config/test.json +++ b/config/test.json @@ -1,8 +1,6 @@ { "@opentermsarchive/engine": { - "services": { - "declarationsPath": "./test/services" - }, + "collectionPath": "./test/test-declarations", "recorder": { "versions": { "storage": { diff --git a/scripts/declarations/lint/index.mocha.js b/scripts/declarations/lint/index.mocha.js index 0ac545bfb..a59c20ff7 100644 --- a/scripts/declarations/lint/index.mocha.js +++ b/scripts/declarations/lint/index.mocha.js @@ -16,8 +16,8 @@ const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml'); const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false }); const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true }); -const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath')); -const instancePath = path.resolve(declarationsPath, '../'); +const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath')); +const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH); export default async options => { let servicesToValidate = options.services || []; diff --git a/scripts/declarations/validate/index.mocha.js b/scripts/declarations/validate/index.mocha.js index f5564b98f..b72c73f4f 100644 --- a/scripts/declarations/validate/index.mocha.js +++ b/scripts/declarations/validate/index.mocha.js @@ -19,8 +19,8 @@ const fs = fsApi.promises; const MIN_DOC_LENGTH = 100; const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow -const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath')); -const instancePath = path.resolve(declarationsPath, '../'); +const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath')); +const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH); export default async options => { const schemaOnly = options.schemaOnly || false; diff --git a/src/archivist/services/index.js b/src/archivist/services/index.js index 45b21b61d..7d23a0972 100644 --- a/src/archivist/services/index.js +++ b/src/archivist/services/index.js @@ -1,4 +1,4 @@ -import fsApi from 'fs'; +import fs from 'fs/promises'; import path from 'path'; import { pathToFileURL } from 'url'; @@ -8,8 +8,8 @@ import Service from './service.js'; import SourceDocument from './sourceDocument.js'; import Terms from './terms.js'; -const fs = fsApi.promises; -const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath')); +export const DECLARATIONS_PATH = './declarations'; +const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'), DECLARATIONS_PATH); export async function load(servicesIdsToLoad = []) { let servicesIds = await getDeclaredServicesIds(); diff --git a/test/test-declarations/README.yml b/test/test-declarations/README.yml new file mode 100644 index 000000000..46f159f89 --- /dev/null +++ b/test/test-declarations/README.yml @@ -0,0 +1,7 @@ +# Test declarations folder + +This folder simulates a collection declarations folder and is used only for testing purposes. + +The structure mirrors a real collection declarations folder but is simplified and contains minimal test data. + + diff --git a/test/services/Service B!.json b/test/test-declarations/declarations/Service B!.json similarity index 100% rename from test/services/Service B!.json rename to test/test-declarations/declarations/Service B!.json diff --git a/test/services/service_with_declaration_history.filters.js b/test/test-declarations/declarations/service_with_declaration_history.filters.js similarity index 100% rename from test/services/service_with_declaration_history.filters.js rename to test/test-declarations/declarations/service_with_declaration_history.filters.js diff --git a/test/services/service_with_declaration_history.history.json b/test/test-declarations/declarations/service_with_declaration_history.history.json similarity index 100% rename from test/services/service_with_declaration_history.history.json rename to test/test-declarations/declarations/service_with_declaration_history.history.json diff --git a/test/services/service_with_declaration_history.json b/test/test-declarations/declarations/service_with_declaration_history.json similarity index 100% rename from test/services/service_with_declaration_history.json rename to test/test-declarations/declarations/service_with_declaration_history.json diff --git a/test/services/service_with_filters_history.filters.history.js b/test/test-declarations/declarations/service_with_filters_history.filters.history.js similarity index 100% rename from test/services/service_with_filters_history.filters.history.js rename to test/test-declarations/declarations/service_with_filters_history.filters.history.js diff --git a/test/services/service_with_filters_history.filters.js b/test/test-declarations/declarations/service_with_filters_history.filters.js similarity index 100% rename from test/services/service_with_filters_history.filters.js rename to test/test-declarations/declarations/service_with_filters_history.filters.js diff --git a/test/services/service_with_filters_history.json b/test/test-declarations/declarations/service_with_filters_history.json similarity index 100% rename from test/services/service_with_filters_history.json rename to test/test-declarations/declarations/service_with_filters_history.json diff --git a/test/services/service_with_history.filters.history.js b/test/test-declarations/declarations/service_with_history.filters.history.js similarity index 100% rename from test/services/service_with_history.filters.history.js rename to test/test-declarations/declarations/service_with_history.filters.history.js diff --git a/test/services/service_with_history.filters.js b/test/test-declarations/declarations/service_with_history.filters.js similarity index 100% rename from test/services/service_with_history.filters.js rename to test/test-declarations/declarations/service_with_history.filters.js diff --git a/test/services/service_with_history.history.json b/test/test-declarations/declarations/service_with_history.history.json similarity index 100% rename from test/services/service_with_history.history.json rename to test/test-declarations/declarations/service_with_history.history.json diff --git a/test/services/service_with_history.json b/test/test-declarations/declarations/service_with_history.json similarity index 100% rename from test/services/service_with_history.json rename to test/test-declarations/declarations/service_with_history.json diff --git a/test/services/service_with_multiple_source_documents_terms.filters.js b/test/test-declarations/declarations/service_with_multiple_source_documents_terms.filters.js similarity index 100% rename from test/services/service_with_multiple_source_documents_terms.filters.js rename to test/test-declarations/declarations/service_with_multiple_source_documents_terms.filters.js diff --git a/test/services/service_with_multiple_source_documents_terms.history.json b/test/test-declarations/declarations/service_with_multiple_source_documents_terms.history.json similarity index 100% rename from test/services/service_with_multiple_source_documents_terms.history.json rename to test/test-declarations/declarations/service_with_multiple_source_documents_terms.history.json diff --git a/test/services/service_with_multiple_source_documents_terms.json b/test/test-declarations/declarations/service_with_multiple_source_documents_terms.json similarity index 100% rename from test/services/service_with_multiple_source_documents_terms.json rename to test/test-declarations/declarations/service_with_multiple_source_documents_terms.json diff --git a/test/services/service_without_history.filters.js b/test/test-declarations/declarations/service_without_history.filters.js similarity index 100% rename from test/services/service_without_history.filters.js rename to test/test-declarations/declarations/service_without_history.filters.js diff --git a/test/services/service_without_history.json b/test/test-declarations/declarations/service_without_history.json similarity index 100% rename from test/services/service_without_history.json rename to test/test-declarations/declarations/service_without_history.json diff --git "a/test/services/service\302\267A.json" "b/test/test-declarations/declarations/service\302\267A.json" similarity index 100% rename from "test/services/service\302\267A.json" rename to "test/test-declarations/declarations/service\302\267A.json" diff --git a/test/test-declarations/metadata.yml b/test/test-declarations/metadata.yml new file mode 100644 index 000000000..f7888a58b --- /dev/null +++ b/test/test-declarations/metadata.yml @@ -0,0 +1,33 @@ +id: test +name: test +tagline: Test collection +description: This is a test collection used for testing purposes. +dataset: https://github.com/OpenTermsArchive/test-versions/releases +declarations: https://github.com/OpenTermsArchive/test-declarations +versions: https://github.com/OpenTermsArchive/test-versions +snapshots: https://github.com/OpenTermsArchive/test-snapshots +donations: null +logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png +languages: + - en +jurisdictions: + - EU +governance: + hosts: + - name: Computer test + administrators: + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png + curators: + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png + maintainers: + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png + sponsors: + - name: Open Terms Archive + url: https://opentermsarchive.org/ + logo: https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png From 641ba8d19ae41b67cd9919dd759438de6274f61c Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Mon, 2 Dec 2024 17:17:20 +0100 Subject: [PATCH 4/9] Expose collection metadata in collection-api --- package-lock.json | 1 + package.json | 1 + src/collection-api/routes/index.js | 1 + src/collection-api/routes/metadata.js | 163 +++++++++++++++++++++ src/collection-api/routes/metadata.test.js | 89 +++++++++++ test/test-declarations/metadata.yml | 2 +- 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 src/collection-api/routes/metadata.js create mode 100644 src/collection-api/routes/metadata.test.js diff --git a/package-lock.json b/package-lock.json index fbf98fc6e..f8fa1fe80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ "https-proxy-agent": "^5.0.0", "iconv-lite": "^0.6.3", "joplin-turndown-plugin-gfm": "^1.0.12", + "js-yaml": "^4.1.0", "jsdom": "^18.1.0", "json-source-map": "^0.6.1", "lodash": "^4.17.21", diff --git a/package.json b/package.json index c294d4e87..f2bbd5daa 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "https-proxy-agent": "^5.0.0", "iconv-lite": "^0.6.3", "joplin-turndown-plugin-gfm": "^1.0.12", + "js-yaml": "^4.1.0", "jsdom": "^18.1.0", "json-source-map": "^0.6.1", "lodash": "^4.17.21", diff --git a/src/collection-api/routes/index.js b/src/collection-api/routes/index.js index 1e675127b..630291d1d 100644 --- a/src/collection-api/routes/index.js +++ b/src/collection-api/routes/index.js @@ -32,6 +32,7 @@ export default async function apiRouter(basePath) { const services = await Services.load(); + router.use(await metadataRouter(services)); router.use(servicesRouter(services)); router.use(versionsRouter); diff --git a/src/collection-api/routes/metadata.js b/src/collection-api/routes/metadata.js new file mode 100644 index 000000000..2d51e1ad8 --- /dev/null +++ b/src/collection-api/routes/metadata.js @@ -0,0 +1,163 @@ +import fs from 'fs/promises'; +import path from 'path'; + +import config from 'config'; +import express from 'express'; +import yaml from 'js-yaml'; + +import Service from '../../archivist/services/service.js'; + +/** + * @swagger + * tags: + * name: Metadata + * description: Collection metadata API + * components: + * schemas: + * Metadata: + * type: object + * description: Collection metadata + * properties: + * id: + * type: string + * description: Unique identifier of the collection + * name: + * type: string + * description: Display name of the collection + * tagline: + * type: string + * description: Short description of the collection + * description: + * type: string + * nullable: true + * description: Detailed description of the collection + * totalTerms: + * type: integer + * description: Total number of terms tracked in the collection + * totalServices: + * type: integer + * description: Total number of services tracked in the collection + * engineVersion: + * type: string + * description: Version of the Open Terms Archive engine in SemVer format (MAJOR.MINOR.PATCH) + * dataset: + * type: string + * format: uri + * description: URL to the dataset releases + * declarations: + * type: string + * format: uri + * description: URL to the declarations repository + * versions: + * type: string + * format: uri + * description: URL to the versions repository + * snapshots: + * type: string + * format: uri + * description: URL to the snapshots repository + * logo: + * type: string + * format: uri + * nullable: true + * description: URL to the collection logo + * languages: + * type: array + * items: + * type: string + * description: List of ISO 639 language codes representing languages allowed by the collection + * jurisdictions: + * type: array + * items: + * type: string + * description: List of ISO 3166-2 country codes representing jurisdictions covered by the collection + * trackingPeriods: + * type: array + * items: + * type: object + * properties: + * startDate: + * type: string + * format: date + * description: The date when tracking started for this period + * schedule: + * type: string + * description: A cron expression defining when terms are tracked (e.g. "0 0 * * *" for daily at midnight) + * serverLocation: + * type: string + * description: The geographic location of the server used for tracking + * endDate: + * type: string + * format: date + * description: The date when tracking ended for this period + * governance: + * type: object + * properties: + * hosts: + * type: array + * items: + * $ref: '#/components/schemas/Organization' + * administrators: + * type: array + * items: + * $ref: '#/components/schemas/Organization' + * curators: + * type: array + * items: + * $ref: '#/components/schemas/Organization' + * maintainers: + * type: array + * items: + * $ref: '#/components/schemas/Organization' + * sponsors: + * type: array + * items: + * $ref: '#/components/schemas/Organization' + * Organization: + * type: object + * properties: + * name: + * type: string + * description: Name of the organization + * url: + * type: string + * format: uri + * description: URL to the organization's website + * logo: + * type: string + * format: uri + * description: URL to the organization's logo + */ +export default async function metadataRouter(services) { + const router = express.Router(); + const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath')); + const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, '/metadata.yml'), 'utf8')); + const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url))); + + /** + * @swagger + * /metadata: + * get: + * summary: Get collection metadata + * tags: [Metadata] + * produces: + * - application/json + * responses: + * 200: + * description: Collection metadata + */ + router.get('/metadata', (req, res) => { + const dynamicMetadata = { + totalServices: Object.keys(services).length, + totalTerms: Service.getNumberOfTerms(services), + engineVersion, + }; + + res.json({ + ...dynamicMetadata, + ...STATIC_METADATA, + }); + }); + + return router; +} diff --git a/src/collection-api/routes/metadata.test.js b/src/collection-api/routes/metadata.test.js new file mode 100644 index 000000000..981641f0a --- /dev/null +++ b/src/collection-api/routes/metadata.test.js @@ -0,0 +1,89 @@ +import fs from 'fs/promises'; + +import { expect } from 'chai'; +import config from 'config'; +import request from 'supertest'; + +import app from '../server.js'; + +const basePath = config.get('@opentermsarchive/engine.collection-api.basePath'); +const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url))); + +const EXPECTED_RESPONSE = { + totalServices: 7, + totalTerms: 8, + id: 'test', + name: 'test', + tagline: 'Test collection', + description: 'This is a test collection used for testing purposes.', + dataset: 'https://github.com/OpenTermsArchive/test-versions/releases', + declarations: 'https://github.com/OpenTermsArchive/test-declarations', + versions: 'https://github.com/OpenTermsArchive/test-versions', + snapshots: 'https://github.com/OpenTermsArchive/test-snapshots', + donations: null, + logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png', + languages: [ + 'en', + ], + jurisdictions: [ + 'EU', + ], + governance: { + hosts: [ + { name: 'Localhost' }, + ], + administrators: [ + { + name: 'Open Terms Archive', + url: 'https://opentermsarchive.org/', + logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png', + }, + ], + curators: [ + { + name: 'Open Terms Archive', + url: 'https://opentermsarchive.org/', + logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png', + }, + ], + maintainers: [ + { + name: 'Open Terms Archive', + url: 'https://opentermsarchive.org/', + logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png', + }, + ], + sponsors: [ + { + name: 'Open Terms Archive', + url: 'https://opentermsarchive.org/', + logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png', + }, + ], + }, +}; + +describe('Metadata API', () => { + describe('GET /metadata', () => { + let response; + + before(async () => { + response = await request(app).get(`${basePath}/v1/metadata`); + }); + + it('responds with 200 status code', () => { + expect(response.status).to.equal(200); + }); + + it('responds with Content-Type application/json', () => { + expect(response.type).to.equal('application/json'); + }); + + it('returns expected metadata object', () => { + expect(response.body).to.deep.equal({ + ...EXPECTED_RESPONSE, + engineVersion, + }); + }); + }); +}); diff --git a/test/test-declarations/metadata.yml b/test/test-declarations/metadata.yml index f7888a58b..05a06aca0 100644 --- a/test/test-declarations/metadata.yml +++ b/test/test-declarations/metadata.yml @@ -14,7 +14,7 @@ jurisdictions: - EU governance: hosts: - - name: Computer test + - name: Localhost administrators: - name: Open Terms Archive url: https://opentermsarchive.org/ From 134fe47a96a5c593cb84f4ab517dc93454ce8a96 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 3 Dec 2024 11:10:38 +0100 Subject: [PATCH 5/9] Add changelog entry --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f6c99788..570d8db67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [major] + +> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs. + +### Added + +- Expose collection metadata through the collection API; requires a [metadata file](https://docs.opentermsarchive.org/collections/metadata/) at the root of your collection folder + +### Changed + +- **Breaking:** Replace `@opentermsarchive/engine.services.declarationsPath` with `@opentermsarchive/engine.collectionPath`; ensure your declarations are located in `./declarations` in your collection folder + ## 3.0.0 - 2024-12-03 _Full changeset and discussions: [#1122](https://github.com/OpenTermsArchive/engine/pull/1122)._ From 782755e4a425f2f8807728250fd30d8c1ac2008d Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 3 Dec 2024 15:11:32 +0100 Subject: [PATCH 6/9] Move dynamic metadata to the bottom of the result --- src/collection-api/routes/metadata.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collection-api/routes/metadata.js b/src/collection-api/routes/metadata.js index 2d51e1ad8..f42bf5628 100644 --- a/src/collection-api/routes/metadata.js +++ b/src/collection-api/routes/metadata.js @@ -154,8 +154,8 @@ export default async function metadataRouter(services) { }; res.json({ - ...dynamicMetadata, ...STATIC_METADATA, + ...dynamicMetadata, }); }); From c577381374889165b729f0c692022a7a758b5f3c Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 3 Dec 2024 15:25:25 +0100 Subject: [PATCH 7/9] Fix CI config --- config/ci.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/ci.json b/config/ci.json index be121e567..6a7b506c1 100644 --- a/config/ci.json +++ b/config/ci.json @@ -1,7 +1,5 @@ { "@opentermsarchive/engine": { - "services": { - "declarationsPath": "./demo-declarations/declarations" - } + "collectionPath": "./demo-declarations/" } } From 97ca88bf0c48aa07fbe6c1048cbd1d19013ae8a0 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 3 Dec 2024 15:42:07 +0100 Subject: [PATCH 8/9] Inject configuration dependency --- src/collection-api/routes/index.js | 6 +++++- src/collection-api/routes/metadata.js | 5 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/collection-api/routes/index.js b/src/collection-api/routes/index.js index 630291d1d..4585c7e2f 100644 --- a/src/collection-api/routes/index.js +++ b/src/collection-api/routes/index.js @@ -1,3 +1,6 @@ +import path from 'path'; + +import config from 'config'; import express from 'express'; import helmet from 'helmet'; @@ -30,9 +33,10 @@ export default async function apiRouter(basePath) { res.json({ message: 'Welcome to an instance of the Open Terms Archive API. Documentation is available at /docs. Learn more on Open Terms Archive on https://opentermsarchive.org.' }); }); + const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath')); const services = await Services.load(); - router.use(await metadataRouter(services)); + router.use(await metadataRouter(collectionPath, services)); router.use(servicesRouter(services)); router.use(versionsRouter); diff --git a/src/collection-api/routes/metadata.js b/src/collection-api/routes/metadata.js index f42bf5628..27c282216 100644 --- a/src/collection-api/routes/metadata.js +++ b/src/collection-api/routes/metadata.js @@ -1,7 +1,6 @@ import fs from 'fs/promises'; import path from 'path'; -import config from 'config'; import express from 'express'; import yaml from 'js-yaml'; @@ -128,9 +127,9 @@ import Service from '../../archivist/services/service.js'; * format: uri * description: URL to the organization's logo */ -export default async function metadataRouter(services) { +export default async function metadataRouter(collectionPath, services) { const router = express.Router(); - const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath')); + const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, '/metadata.yml'), 'utf8')); const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url))); From 21348ebf26914c15bf57e176825ff73e4c216ddd Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 3 Dec 2024 15:42:16 +0100 Subject: [PATCH 9/9] Remove magic strings --- src/collection-api/routes/metadata.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/collection-api/routes/metadata.js b/src/collection-api/routes/metadata.js index 27c282216..0a1ca1fa4 100644 --- a/src/collection-api/routes/metadata.js +++ b/src/collection-api/routes/metadata.js @@ -6,6 +6,9 @@ import yaml from 'js-yaml'; import Service from '../../archivist/services/service.js'; +const METADATA_FILENAME = 'metadata.yml'; +const PACKAGE_JSON_PATH = '../../../package.json'; + /** * @swagger * tags: @@ -130,8 +133,8 @@ import Service from '../../archivist/services/service.js'; export default async function metadataRouter(collectionPath, services) { const router = express.Router(); - const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, '/metadata.yml'), 'utf8')); - const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url))); + const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, METADATA_FILENAME), 'utf8')); + const { version: engineVersion } = JSON.parse(await fs.readFile(new URL(PACKAGE_JSON_PATH, import.meta.url))); /** * @swagger