diff --git a/packages/api/pw-test.config.cjs b/packages/api/pw-test.config.cjs index 022ad0f970..a3f087f0d0 100644 --- a/packages/api/pw-test.config.cjs +++ b/packages/api/pw-test.config.cjs @@ -6,9 +6,7 @@ const { once } = require('events') /** @typedef {{ proc: execa.ExecaChildProcess }} ProcessObject */ -dotenv.config({ - path: path.join(__dirname, '../../.env'), -}) +dotenv.config({ path: path.join(__dirname, '../../.env') }) const cli = path.join(__dirname, 'scripts/cli.js') @@ -31,21 +29,10 @@ module.exports = { buildConfig: { inject: [path.join(__dirname, './scripts/node-globals.js')], plugins: [nodeBuiltinsPlugin], - define: { - DATABASE_URL: JSON.stringify(process.env.DATABASE_URL), - DATABASE_TOKEN: JSON.stringify(process.env.DATABASE_TOKEN), - }, }, buildSWConfig: { - inject: [ - path.join(__dirname, './scripts/node-globals.js'), - path.join(__dirname, './test/scripts/worker-globals.js'), - ], + inject: [path.join(__dirname, './scripts/node-globals.js')], plugins: [nodeBuiltinsPlugin], - define: { - DATABASE_URL: JSON.stringify(process.env.DATABASE_URL), - DATABASE_TOKEN: JSON.stringify(process.env.DATABASE_TOKEN), - }, }, beforeTests: async () => { const mock = await startMockServer('AWS S3', 9095, 'test/mocks/aws-s3') diff --git a/packages/api/scripts/cli.js b/packages/api/scripts/cli.js index e8e1a4272c..753c8e26b9 100755 --- a/packages/api/scripts/cli.js +++ b/packages/api/scripts/cli.js @@ -58,9 +58,9 @@ prog inject: [path.join(__dirname, 'node-globals.js')], plugins: [PluginAlias], define: { - VERSION: JSON.stringify(version), - COMMITHASH: JSON.stringify(git.long(__dirname)), - BRANCH: JSON.stringify(git.branch(__dirname)), + NFT_STORAGE_VERSION: JSON.stringify(version), + NFT_STORAGE_COMMITHASH: JSON.stringify(git.long(__dirname)), + NFT_STORAGE_BRANCH: JSON.stringify(git.branch(__dirname)), global: 'globalThis', }, minify: opts.env === 'dev' ? false : true, diff --git a/packages/api/src/bindings.d.ts b/packages/api/src/bindings.d.ts index bdcdb71c5f..7fce615b90 100644 --- a/packages/api/src/bindings.d.ts +++ b/packages/api/src/bindings.d.ts @@ -7,30 +7,74 @@ import { UserOutput, UserOutputKey } from './utils/db-client-types.js' import { DBClient } from './utils/db-client.js' import { Logging } from './utils/logs.js' -declare global { - const SALT: string - const DEBUG: string - const CLUSTER_SERVICE: 'IpfsCluster' | 'IpfsCluster2' | 'IpfsCluster3' - const CLUSTER_API_URL: string - const CLUSTER_BASIC_AUTH_TOKEN: string - const MAGIC_SECRET_KEY: string - const DATABASE_URL: string - const DATABASE_TOKEN: string - const MAILCHIMP_API_KEY: string - const LOGTAIL_TOKEN: string - const ENV: 'dev' | 'staging' | 'production' - const SENTRY_DSN: string - const BRANCH: string - const VERSION: string - const COMMITHASH: string - const MAINTENANCE_MODE: Mode - const METAPLEX_AUTH_TOKEN: string - const S3_ENDPOINT: string - const S3_REGION: string - const S3_ACCESS_KEY_ID: string - const S3_SECRET_ACCESS_KEY: string - const S3_BUCKET_NAME: string - const PRIVATE_KEY: string +export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production' + +export interface ServiceConfiguration { + /** Is this a debug build? */ + DEBUG: boolean + + /** Target runtime environment */ + ENV: RuntimeEnvironmentName + + /** Semantic version for current build */ + NFT_STORAGE_VERSION: string + + /** Git branch name of current build */ + NFT_STORAGE_BRANCH: string + + /** Git commit hash of current build */ + NFT_STORAGE_COMMITHASH: string + + /** Current maintenance mode */ + MAINTENANCE_MODE: Mode + + /** Salt for API key generation */ + SALT: string + + /** API key for special metaplex upload account */ + METAPLEX_AUTH_TOKEN: string + + /** UCAN private signing key */ + PRIVATE_KEY: string + + /** API url for active IPFS cluster endpoint */ + CLUSTER_API_URL: string + + /** Auth token for IPFS culster */ + CLUSTER_BASIC_AUTH_TOKEN: string + + /** Postgrest endpoint URL */ + DATABASE_URL: string + + /** Postgrest auth token */ + DATABASE_TOKEN: string + + /** S3 endpoint URL */ + S3_ENDPOINT: string + + /** S3 region */ + S3_REGION: string + + /** S3 access key id */ + S3_ACCESS_KEY_ID: string + + /** S3 secret key */ + S3_SECRET_ACCESS_KEY: string + + /** S3 bucket name */ + S3_BUCKET_NAME: string + + /** Magic link secret key */ + MAGIC_SECRET_KEY: string + + /** Logtail auth token */ + LOGTAIL_TOKEN: string + + /** Sentry DSN */ + SENTRY_DSN: string + + /** Mailchimp api key */ + MAILCHIMP_API_KEY: string } export interface Ucan { diff --git a/packages/api/src/cluster.js b/packages/api/src/cluster.js index d6ebf00929..f9ac419313 100644 --- a/packages/api/src/cluster.js +++ b/packages/api/src/cluster.js @@ -1,9 +1,13 @@ import { Cluster } from '@nftstorage/ipfs-cluster' -import { cluster } from './constants.js' +import { getServiceConfig } from './config.js' import { HTTPError } from './errors.js' -const client = new Cluster(cluster.apiUrl, { - headers: { Authorization: `Basic ${cluster.basicAuthToken}` }, +const { CLUSTER_API_URL, CLUSTER_BASIC_AUTH_TOKEN } = getServiceConfig() + +const client = new Cluster(CLUSTER_API_URL, { + headers: { + Authorization: `Basic ${CLUSTER_BASIC_AUTH_TOKEN}`, + }, }) /** diff --git a/packages/api/src/config.js b/packages/api/src/config.js new file mode 100644 index 0000000000..5b5220d33d --- /dev/null +++ b/packages/api/src/config.js @@ -0,0 +1,248 @@ +import { + modes as MaintenanceModes, + DEFAULT_MODE, +} from './middleware/maintenance.js' + +/** + * @typedef {import('./bindings').ServiceConfiguration} ServiceConfiguration + * @typedef {import('./bindings').RuntimeEnvironmentName} RuntimeEnvironmentName + */ + +/** + * Default configuration values to be used in test and dev if no explicit definition is found. + * + * @type Record + */ +export const DEFAULT_CONFIG_VALUES = { + SALT: 'secret', + DEBUG: 'true', + DATABASE_URL: 'http://localhost:3000', + DATABASE_TOKEN: + 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI', + CLUSTER_API_URL: 'http://localhost:9094', + CLUSTER_BASIC_AUTH_TOKEN: 'dGVzdDp0ZXN0', // test:test + MAGIC_SECRET_KEY: 'test', + ENV: 'test', + SENTRY_DSN: 'https://test@test.ingest.sentry.io/0000000', + NFT_STORAGE_BRANCH: 'test', + NFT_STORAGE_VERSION: 'test', + NFT_STORAGE_COMMITHASH: 'test', + MAINTENANCE_MODE: 'rw', + METAPLEX_AUTH_TOKEN: 'metaplex-test-token', + MAILCHIMP_API_KEY: '', + LOGTAIL_TOKEN: '', + S3_ENDPOINT: 'http://localhost:9095', + S3_REGION: 'test', + S3_ACCESS_KEY_ID: 'test', + S3_SECRET_ACCESS_KEY: 'test', + S3_BUCKET_NAME: 'test', + PRIVATE_KEY: 'xmbtWjE9eYuAxae9G65lQSkw36HV6H+0LSFq2aKqVwY=', +} + +/** + * If the CLUSTER_SERVICE variable is set, the service URL will be resolved from here. + * + * @type Record */ +const CLUSTER_SERVICE_URLS = { + IpfsCluster: 'https://nft.storage.ipfscluster.io/api/', + IpfsCluster2: 'https://nft2.storage.ipfscluster.io/api/', + IpfsCluster3: 'https://nft3.storage.ipfscluster.io/api/', +} + +/** + * @param {RuntimeEnvironmentName} env + * @returns {boolean} true if the named runtime environment should fallback to values from DEFAULT_CONFIG_VALUES if no config var is present. + */ +const allowDefaultConfigValues = (env) => env === 'test' || env === 'dev' + +/** @type ServiceConfiguration|undefined */ +let _globalConfig + +/** + * Returns a {@link ServiceConfiguration} object containing the runtime config options for the API service. + * Includes anything injected by the environment (secrets, URLs for services we call out to, maintenance flag, etc). + * + * Loaded from global variables injected by CloudFlare Worker runtime. + * + * Lazily loaded and cached on first access. + */ +export const getServiceConfig = () => { + if (_globalConfig) { + return _globalConfig + } + _globalConfig = loadServiceConfig() + return _globalConfig +} + +/** + * Override the global service configuration for testing purposes. + * Note that some files call {@link getServiceConfig} at module scope, + * so they may not pick up the overriden config. + * + * @param {ServiceConfiguration} config + */ +export const overrideServiceConfigForTesting = (config) => { + _globalConfig = config +} + +/** + * Load a {@link ServiceConfiguration} from the global environment. + * + * Exported for testing. See {@link getServiceConfig} for main public accessor. + * + * Config values are resolved by looking for global variables with the names matching the keys of {@link DEFAULT_CONFIG_VALUES}. + * + * If no global value is found for a variable, an error will be thrown if the runtimeEnvironment (ENV variable) + * is set to a "production like" environment. + * + * If {@link allowDefaultConfigValues} returns true for the current environment, the value from {@link DEFAULT_CONFIG_VALUES} will be + * used if a variable is missing. + * + * @returns {ServiceConfiguration} + */ +export function loadServiceConfig() { + const vars = loadConfigVariables() + return serviceConfigFromVariables(vars) +} + +/** + * Parse a {@link ServiceConfiguration} out of the given `configVars` map. + * @param {Record} vars map of variable names to values. + * + * Exported for testing. See {@link getServiceConfig} for main public accessor. + * + * @returns {ServiceConfiguration} + */ +export function serviceConfigFromVariables(vars) { + let clusterUrl = vars.CLUSTER_API_URL + if (vars.CLUSTER_SERVICE) { + const serviceUrl = CLUSTER_SERVICE_URLS[vars.CLUSTER_SERVICE] + if (!serviceUrl) { + throw new Error(`unknown cluster service: ${vars.CLUSTER_SERVICE}`) + } + clusterUrl = serviceUrl + } + + return { + ENV: parseRuntimeEnv(vars.ENV), + DEBUG: boolValue(vars.DEBUG), + MAINTENANCE_MODE: maintenanceModeFromString(vars.MAINTENANCE_MODE), + + SALT: vars.SALT, + DATABASE_URL: vars.DATABASE_URL, + DATABASE_TOKEN: vars.DATABASE_TOKEN, + CLUSTER_API_URL: clusterUrl, + CLUSTER_BASIC_AUTH_TOKEN: vars.CLUSTER_BASIC_AUTH_TOKEN, + MAGIC_SECRET_KEY: vars.MAGIC_SECRET_KEY, + SENTRY_DSN: vars.SENTRY_DSN, + NFT_STORAGE_BRANCH: vars.NFT_STORAGE_BRANCH, + NFT_STORAGE_VERSION: vars.NFT_STORAGE_VERSION, + NFT_STORAGE_COMMITHASH: vars.NFT_STORAGE_COMMITHASH, + METAPLEX_AUTH_TOKEN: vars.METAPLEX_AUTH_TOKEN, + MAILCHIMP_API_KEY: vars.MAILCHIMP_API_KEY, + LOGTAIL_TOKEN: vars.LOGTAIL_TOKEN, + S3_ENDPOINT: vars.S3_ENDPOINT, + S3_REGION: vars.S3_REGION, + S3_ACCESS_KEY_ID: vars.S3_ACCESS_KEY_ID, + S3_SECRET_ACCESS_KEY: vars.S3_SECRET_ACCESS_KEY, + S3_BUCKET_NAME: vars.S3_BUCKET_NAME, + PRIVATE_KEY: vars.PRIVATE_KEY, + } +} + +/** + * Loads configuration variables from the global environment and returns a JS object + * keyed by variable names. + * + * Exported for testing. See {@link getServiceConfig} for main config accessor. + * + * @returns { Record} an object with `vars` containing all config variables and their values. guaranteed to have a value for each key defined in DEFAULT_CONFIG_VALUES + * @throws if a config variable is missing, unless ENV is 'test' or 'dev', in which case the default value will be used for missing vars. + */ +export function loadConfigVariables() { + /** @type Record */ + const vars = {} + + /** @type Record */ + const globals = globalThis + + const notFound = [] + for (const name of Object.keys(DEFAULT_CONFIG_VALUES)) { + const val = globals[name] + if (typeof val === 'string') { + vars[name] = val + } else { + notFound.push(name) + } + } + + if (notFound.length !== 0) { + const env = parseRuntimeEnv(vars.ENV) + if (!allowDefaultConfigValues(env)) { + throw new Error( + 'Missing required config variables: ' + notFound.join(', ') + ) + } + console.warn( + 'Using default values for config variables: ', + notFound.join(', ') + ) + for (const name of notFound) { + vars[name] = DEFAULT_CONFIG_VALUES[name] + } + } + + return vars +} + +/** + * Validates that `s` is a defined runtime environment name and returns it. + * + * @param {unknown} s + * @returns {RuntimeEnvironmentName} + */ +function parseRuntimeEnv(s) { + if (!s) { + return 'test' + } + + switch (s) { + case 'test': + case 'dev': + case 'staging': + case 'production': + return s + default: + throw new Error('invalid runtime environment name: ' + s) + } +} + +/** + * If `s` is undefined, return the default maintenance mode. Otherwise, make sure it's a valid mode and return. + * + * @param {string|undefined} s + * @returns {import('./middleware/maintenance').Mode} + */ +function maintenanceModeFromString(s) { + if (s === undefined) { + return DEFAULT_MODE + } + for (const m of MaintenanceModes) { + if (s === m) { + return m + } + } + throw new Error( + `invalid maintenance mode value "${s}". valid choices: ${MaintenanceModes}` + ) +} + +/** + * Returns `true` if the string `s` is equal to `"true"` (case-insensitive) or `"1", and false for `"false"`, `"0"` or an empty value. + * + * @param {string} s + * @returns {boolean} + */ +function boolValue(s) { + return Boolean(s && JSON.parse(String(s).toLowerCase())) +} diff --git a/packages/api/src/constants.js b/packages/api/src/constants.js deleted file mode 100644 index cc7fbc3960..0000000000 --- a/packages/api/src/constants.js +++ /dev/null @@ -1,54 +0,0 @@ -// let MAGIC_SECRET_KEY, SALT, PINATA_JWT, SENTRY_DSN, DATABASE_TOKEN, CLUSTER_SERVICE, LOGTAIL_TOKEN, MAILCHIMP_API_KEY, METAPLEX_AUTH_TOKEN - -export const secrets = { - privateKey: PRIVATE_KEY, - salt: SALT, - magic: MAGIC_SECRET_KEY, - sentry: SENTRY_DSN, - database: DATABASE_TOKEN, - mailchimp: MAILCHIMP_API_KEY, - logtail: LOGTAIL_TOKEN, - metaplexAuth: - typeof METAPLEX_AUTH_TOKEN !== 'undefined' - ? METAPLEX_AUTH_TOKEN - : undefined, -} - -const CLUSTER1 = 'https://nft.storage.ipfscluster.io/api/' -const CLUSTER2 = 'https://nft2.storage.ipfscluster.io/api/' -const CLUSTER3 = 'https://nft3.storage.ipfscluster.io/api/' - -let clusterUrl -if (typeof CLUSTER_SERVICE !== 'undefined' && CLUSTER_SERVICE) { - if (CLUSTER_SERVICE === 'IpfsCluster') { - clusterUrl = CLUSTER1 - } else if (CLUSTER_SERVICE === 'IpfsCluster2') { - clusterUrl = CLUSTER2 - } else if (CLUSTER_SERVICE === 'IpfsCluster3') { - clusterUrl = CLUSTER3 - } else { - throw new Error(`unknown cluster service: ${CLUSTER_SERVICE}`) - } -} else { - clusterUrl = CLUSTER_API_URL -} - -export const cluster = { - apiUrl: clusterUrl, - basicAuthToken: CLUSTER_BASIC_AUTH_TOKEN, -} - -export const database = { - url: DATABASE_URL, -} - -export const isDebug = DEBUG === 'true' - -export const s3 = { - endpoint: typeof S3_ENDPOINT !== 'undefined' ? S3_ENDPOINT : '', - region: typeof S3_REGION !== 'undefined' ? S3_REGION : '', - accessKeyId: typeof S3_ACCESS_KEY_ID !== 'undefined' ? S3_ACCESS_KEY_ID : '', - secretAccessKey: - typeof S3_SECRET_ACCESS_KEY !== 'undefined' ? S3_SECRET_ACCESS_KEY : '', - bucketName: typeof S3_BUCKET_NAME !== 'undefined' ? S3_BUCKET_NAME : '', -} diff --git a/packages/api/src/index.js b/packages/api/src/index.js index ce6880f329..9b4c5efcc4 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -25,21 +25,16 @@ import { userDIDRegister } from './routes/user-did-register.js' import { userTags } from './routes/user-tags.js' import { ucanToken } from './routes/ucan-token.js' import { did } from './routes/did.js' +import { getServiceConfig } from './config.js' import { withMode, READ_ONLY as RO, READ_WRITE as RW, - DEFAULT_MODE, - setMaintenanceModeGetter, } from './middleware/maintenance.js' import { getContext } from './utils/context.js' import { withAuth } from './middleware/auth.js' -const getMaintenanceMode = () => - typeof MAINTENANCE_MODE !== 'undefined' ? MAINTENANCE_MODE : DEFAULT_MODE -setMaintenanceModeGetter(getMaintenanceMode) - const r = new Router(getContext, { onError(req, err, ctx) { return HTTPError.respond(err, ctx) @@ -62,11 +57,17 @@ r.add( 'get', '/version', (event) => { + const { + NFT_STORAGE_VERSION, + NFT_STORAGE_COMMITHASH, + NFT_STORAGE_BRANCH, + MAINTENANCE_MODE, + } = getServiceConfig() return new JSONResponse({ - version: VERSION, - commit: COMMITHASH, - branch: BRANCH, - mode: getMaintenanceMode(), + version: NFT_STORAGE_VERSION, + commit: NFT_STORAGE_COMMITHASH, + branch: NFT_STORAGE_BRANCH, + mode: MAINTENANCE_MODE, }) }, [postCors] diff --git a/packages/api/src/middleware/maintenance.js b/packages/api/src/middleware/maintenance.js index 7f9ed5aac8..ab0e4f2daa 100644 --- a/packages/api/src/middleware/maintenance.js +++ b/packages/api/src/middleware/maintenance.js @@ -1,4 +1,5 @@ import { ErrorMaintenance, HTTPError } from '../errors.js' +import { getServiceConfig } from '../config.js' /** * @typedef {'rw' | 'r-' | '--'} Mode @@ -29,26 +30,7 @@ export const modes = Object.freeze([NO_READ_OR_WRITE, READ_ONLY, READ_WRITE]) export const DEFAULT_MODE = READ_WRITE /** @type {() => Mode} */ -let getMaintenanceMode = () => DEFAULT_MODE - -/** - * Sets the function that returns the current maintenance mode. This allows us - * to pass a function that accesses a global variable. Cloudflare secrets are - * exposed as global variables so this is the way to dynamically access them. - * - * The return value from the getter function dictates which request handlers - * (that have been wrapped with `withMode`) are enabled. The returned value is - * one of: - * - * -- = no reading or writing - * r- = read only mode - * rw = read and write (normal operation) - * - * @param {() => Mode} getter - */ -export function setMaintenanceModeGetter(getter) { - getMaintenanceMode = getter -} +let getMaintenanceMode = () => getServiceConfig().MAINTENANCE_MODE /** * Specify the mode (permissions) a request hander requires to operate e.g. diff --git a/packages/api/src/routes/blog-subscribe.js b/packages/api/src/routes/blog-subscribe.js index d6e357d4fc..c8d17e0bc5 100644 --- a/packages/api/src/routes/blog-subscribe.js +++ b/packages/api/src/routes/blog-subscribe.js @@ -1,9 +1,9 @@ import { JSONResponse } from '../utils/json-response.js' -import { secrets } from '../constants.js' +import { getServiceConfig } from '../config.js' const SERVER_PREFIX = 'us5' const LIST_ID = '64f6e3fd11' -const API_KEY = secrets.mailchimp +const API_KEY = getServiceConfig().MAILCHIMP_API_KEY const TOKEN = btoa(`:${API_KEY}`) const headers = { diff --git a/packages/api/src/routes/metaplex-upload.js b/packages/api/src/routes/metaplex-upload.js index e59812c0fd..6e65b7a93a 100644 --- a/packages/api/src/routes/metaplex-upload.js +++ b/packages/api/src/routes/metaplex-upload.js @@ -14,7 +14,7 @@ import { ErrorMetaplexTokenNotFound, ErrorInvalidMetaplexToken, } from '../errors.js' -import { secrets } from '../constants.js' +import { getServiceConfig } from '../config.js' import { uploadCarWithStat, carStat } from './nfts-upload.js' /** @@ -72,7 +72,8 @@ export async function metaplexUpload(event, ctx) { * @param {import('../utils/db-client').DBClient} db */ async function validate(db) { - if (!secrets.metaplexAuth) { + const { METAPLEX_AUTH_TOKEN } = getServiceConfig() + if (!METAPLEX_AUTH_TOKEN) { throw new Error('missing metaplex auth key') } // note: we need to specify the foreign key to use in the select statement below @@ -81,7 +82,7 @@ async function validate(db) { const { error, data } = await db.client .from('auth_key') .select('id,user:auth_key_user_id_fkey(id)') - .eq('secret', secrets.metaplexAuth) + .eq('secret', METAPLEX_AUTH_TOKEN) .single() if (error) { diff --git a/packages/api/src/routes/nfts-check.js b/packages/api/src/routes/nfts-check.js index 9b64a599e0..ae9216b28b 100644 --- a/packages/api/src/routes/nfts-check.js +++ b/packages/api/src/routes/nfts-check.js @@ -1,11 +1,12 @@ import { JSONResponse } from '../utils/json-response.js' import { HTTPError } from '../errors.js' -import { secrets, database } from '../constants.js' +import { getServiceConfig } from '../config.js' import { DBClient } from '../utils/db-client' import { parseCid } from '../utils/utils.js' import { toCheckNftResponse } from '../utils/db-transforms.js' -const db = new DBClient(database.url, secrets.database) +const { DATABASE_URL, DATABASE_TOKEN } = getServiceConfig() +const db = new DBClient(DATABASE_URL, DATABASE_TOKEN) /** @type {import('../bindings').Handler} */ export const nftCheck = async (event, { params }) => { diff --git a/packages/api/src/routes/tokens-create.js b/packages/api/src/routes/tokens-create.js index 61b1da2e98..9a3105af76 100644 --- a/packages/api/src/routes/tokens-create.js +++ b/packages/api/src/routes/tokens-create.js @@ -1,7 +1,7 @@ import { checkAuth } from '../utils/auth.js' import { JSONResponse } from '../utils/json-response.js' import { signJWT } from '../utils/jwt.js' -import { secrets } from '../constants.js' +import { getServiceConfig } from '../config.js' /** @type {import('../bindings').Handler} */ export const tokensCreate = async (event, ctx) => { @@ -18,7 +18,7 @@ export const tokensCreate = async (event, ctx) => { iat: created.valueOf(), name: body.name, }, - secrets.salt + getServiceConfig().SALT ) const key = await db.createKey({ diff --git a/packages/api/src/utils/auth.js b/packages/api/src/utils/auth.js index 947aa64bc2..8eb5588b01 100644 --- a/packages/api/src/utils/auth.js +++ b/packages/api/src/utils/auth.js @@ -1,5 +1,5 @@ import { Magic } from '@magic-sdk/admin' -import { secrets } from '../constants.js' +import { getServiceConfig } from '../config.js' import { HTTPError, ErrorUserNotFound, @@ -8,9 +8,11 @@ import { ErrorTokenBlocked, } from '../errors.js' import { parseJWT, verifyJWT } from './jwt.js' -export const magic = new Magic(secrets.magic) import * as Ucan from 'ucan-storage/ucan-storage' +const { MAGIC_SECRET_KEY, SALT } = getServiceConfig() +export const magic = new Magic(MAGIC_SECRET_KEY) + /** * * @param {import('./db-client-types.js').UserOutput} user @@ -51,7 +53,7 @@ export async function validate(event, { log, db, ucanService }, options) { } // validate access tokens - if (await verifyJWT(token, secrets.salt)) { + if (await verifyJWT(token, SALT)) { const decoded = parseJWT(token) const user = await db.getUser(decoded.sub) diff --git a/packages/api/src/utils/context.js b/packages/api/src/utils/context.js index a6ca227f20..67ddc256b3 100644 --- a/packages/api/src/utils/context.js +++ b/packages/api/src/utils/context.js @@ -1,20 +1,21 @@ import Toucan from 'toucan-js' import { DBClient } from './db-client.js' import { S3BackupClient } from './s3-backup-client.js' -import { secrets, database, isDebug, s3 as s3Config } from '../constants.js' +import { getServiceConfig } from '../config.js' import { Logging } from './logs.js' import pkg from '../../package.json' import { Service } from 'ucan-storage/service' -const db = new DBClient(database.url, secrets.database) +const config = getServiceConfig() +const db = new DBClient(config.DATABASE_URL, config.DATABASE_TOKEN) -const backup = s3Config.accessKeyId +const backup = config.S3_ACCESS_KEY_ID ? new S3BackupClient( - s3Config.region, - s3Config.accessKeyId, - s3Config.secretAccessKey, - s3Config.bucketName, - { endpoint: s3Config.endpoint, appName: 'nft' } + config.S3_REGION, + config.S3_ACCESS_KEY_ID, + config.S3_SECRET_ACCESS_KEY, + config.S3_BUCKET_NAME, + { endpoint: config.S3_ENDPOINT, appName: 'nft' } ) : undefined @@ -23,15 +24,15 @@ if (!backup) { } const sentryOptions = { - dsn: secrets.sentry, + dsn: config.SENTRY_DSN, allowedHeaders: ['user-agent', 'x-client'], allowedSearchParams: /(.*)/, debug: false, - environment: ENV, + environment: config.ENV, rewriteFrames: { root: '/', }, - release: VERSION, + release: config.NFT_STORAGE_VERSION, pkg, } @@ -48,11 +49,11 @@ export async function getContext(event, params) { ...sentryOptions, }) const log = new Logging(event, { - token: secrets.logtail, - debug: isDebug, + token: config.LOGTAIL_TOKEN, + debug: config.DEBUG, sentry, }) - const ucanService = await Service.fromPrivateKey(secrets.privateKey) + const ucanService = await Service.fromPrivateKey(config.PRIVATE_KEY) return { params, db, backup, log, ucanService } } diff --git a/packages/api/src/utils/logs.js b/packages/api/src/utils/logs.js index ce88af242a..6d530d1cfe 100644 --- a/packages/api/src/utils/logs.js +++ b/packages/api/src/utils/logs.js @@ -1,5 +1,8 @@ import { nanoid } from 'nanoid/non-secure' +import { getServiceConfig } from '../config' +const { NFT_STORAGE_VERSION, NFT_STORAGE_COMMITHASH, NFT_STORAGE_BRANCH } = + getServiceConfig() const logtailApiURL = 'https://in.logtail.com/' const buildMetadataFromHeaders = (/** @type {Headers} */ headers) => { @@ -53,9 +56,9 @@ export class Logging { cf: rCf, }, cloudflare_worker: { - version: VERSION, - commit: COMMITHASH, - branch: BRANCH, + version: NFT_STORAGE_VERSION, + commit: NFT_STORAGE_COMMITHASH, + branch: NFT_STORAGE_BRANCH, worker_id: nanoid(10), worker_started: this.startTs, }, diff --git a/packages/api/src/utils/router.js b/packages/api/src/utils/router.js index cef348728a..500f05e3bb 100644 --- a/packages/api/src/utils/router.js +++ b/packages/api/src/utils/router.js @@ -1,5 +1,5 @@ import { parse } from 'regexparam' -import { database, cluster } from '../constants.js' +import { getServiceConfig } from '../config.js' /** * @typedef {{ params: Record }} BasicRouteContext @@ -182,7 +182,8 @@ class Router { listen(event) { const url = new URL(event.request.url) // Add more if needed for other backends - const passThrough = [database.url, cluster.apiUrl] + const { DATABASE_URL, CLUSTER_API_URL } = getServiceConfig() + const passThrough = [DATABASE_URL, CLUSTER_API_URL] // Ignore http requests from the passthrough list above if (!passThrough.includes(`${url.protocol}//${url.host}`)) { diff --git a/packages/api/test/config.spec.js b/packages/api/test/config.spec.js new file mode 100644 index 0000000000..70e8d677c6 --- /dev/null +++ b/packages/api/test/config.spec.js @@ -0,0 +1,155 @@ +import * as assert from 'assert' +import { + serviceConfigFromVariables, + DEFAULT_CONFIG_VALUES, + loadConfigVariables, + loadServiceConfig, +} from '../src/config.js' + +/** @type Record */ +const globals = globalThis + +/** + * Helper to remove all the config variables we care about from the global context. + */ +const scrubGlobals = () => { + for (const name of Object.keys(DEFAULT_CONFIG_VALUES)) { + delete globals[name] + } +} + +/** + * @param {Record} vars + */ +const defineGlobals = (vars) => { + for (const [name, value] of Object.entries(vars)) { + globals[name] = value + } +} + +describe('loadServiceConfig', () => { + beforeEach(scrubGlobals) + afterEach(scrubGlobals) + + it('fails if config vars are missing when ENV == "staging" or "production"', () => { + const strictEnvs = ['staging', 'production'] + for (const env of strictEnvs) { + defineGlobals({ ENV: env }) + assert.throws(() => loadServiceConfig()) + } + }) + + it('uses default config values for missing vars when ENV == "test" or "dev"', () => { + const lenientEnvs = ['test', 'dev'] + for (const env of lenientEnvs) { + defineGlobals({ ENV: env }) + const cfg = loadServiceConfig() + assert.equal(cfg.ENV, env) + assert.ok(cfg.MAINTENANCE_MODE) + } + }) +}) + +describe('loadConfigVariables', () => { + beforeEach(scrubGlobals) + afterEach(scrubGlobals) + + it('looks up values on the globalThis object', () => { + globals['SALT'] = 'extra-salty' + const vars = loadConfigVariables() + assert.equal(vars.SALT, 'extra-salty') + }) + + it('only includes variables with keys in DEFAULT_CONFIG_VALUES', () => { + globals['FOO'] = 'ignored' + globals['SALT'] = 'extra-salty' + const vars = loadConfigVariables() + assert.equal(vars.SALT, 'extra-salty') + assert.equal(vars['FOO'], undefined) + }) +}) + +describe('serviceConfigFromVariables', () => { + it('sets DEBUG to true if DEBUG is equal to "true" or "1"', () => { + const truthyValues = ['true', 'TRUE', '1'] + for (const t of truthyValues) { + const cfg = serviceConfigFromVariables({ DEBUG: t }) + assert.equal(cfg.DEBUG, true) + } + }) + + it('sets isDebugBuild to false if DEBUG is missing or has a falsy string value ("false", "0", "")', () => { + assert.equal(serviceConfigFromVariables({}).DEBUG, false) + + const falsyValues = ['false', 'FALSE', '0', ''] + + for (const f of falsyValues) { + assert.equal(serviceConfigFromVariables({ DEBUG: f }).DEBUG, false) + } + }) + + it('defaults ENV to "test" if ENV is not set', () => { + assert.equal(serviceConfigFromVariables({}).ENV, 'test') + }) + + it('fails if ENV is set to an unknown environment name', () => { + assert.throws( + () => serviceConfigFromVariables({ ENV: 'not-a-real-env' }), + /invalid/ + ) + }) + + it('sets ENV if it contains a valid environment name', () => { + const envs = ['test', 'dev', 'staging', 'production'] + for (const e of envs) { + assert.equal(serviceConfigFromVariables({ ENV: e }).ENV, e) + } + }) + + it('fails if MAINTENANCE_MODE is set to an invalid mode string', () => { + assert.throws( + () => serviceConfigFromVariables({ MAINTENANCE_MODE: 'not-a-real-mode' }), + /invalid/ + ) + }) + + it('sets MAINTENANCE_MODE if it contains a valid mode string', () => { + const modes = ['--', 'r-', 'rw'] + for (const m of modes) { + assert.equal( + serviceConfigFromVariables({ MAINTENANCE_MODE: m }).MAINTENANCE_MODE, + m + ) + } + }) + + describe('uses unaltered values for string config variables', () => { + const stringValuedVars = [ + 'NFT_STORAGE_VERSION', + 'NFT_STORAGE_BRANCH', + 'NFT_STORAGE_COMMITHASH', + 'SALT', + 'METAPLEX_AUTH_TOKEN', + 'PRIVATE_KEY', + 'CLUSTER_API_URL', + 'CLUSTER_BASIC_AUTH_TOKEN', + 'DATABASE_URL', + 'DATABASE_TOKEN', + 'S3_ENDPOINT', + 'S3_REGION', + 'S3_ACCESS_KEY_ID', + 'S3_BUCKET_NAME', + 'MAGIC_SECRET_KEY', + 'LOGTAIL_TOKEN', + 'SENTRY_DSN', + 'MAILCHIMP_API_KEY', + ] + + for (const key of stringValuedVars) { + const val = `value for ${key}` + const cfg = serviceConfigFromVariables({ [key]: val }) + // @ts-expect-error TS doesn't like us indexing the config object with arbitrary strings + assert.equal(cfg[key], val) + } + }) +}) diff --git a/packages/api/test/db-client.spec.js b/packages/api/test/db-client.spec.js index ee42db2603..a90faba3a3 100644 --- a/packages/api/test/db-client.spec.js +++ b/packages/api/test/db-client.spec.js @@ -1,7 +1,7 @@ import assert from 'assert' import { signJWT } from '../src/utils/jwt.js' import { createClientWithUser, DBTestClient } from './scripts/helpers.js' -import { SALT } from './scripts/worker-globals.js' +import { getServiceConfig } from '../src/config.js' describe('DB Client', () => { /** @type{DBTestClient} */ @@ -11,7 +11,8 @@ describe('DB Client', () => { client = await createClientWithUser() }) - it('getUser should list all keys', async () => { + it('getUser should list only active keys', async () => { + const config = getServiceConfig() const issuer1 = `did:eth:0x73573${Date.now()}` const token1 = await signJWT( { @@ -20,7 +21,7 @@ describe('DB Client', () => { iat: Date.now(), name: 'key1', }, - SALT + config.SALT ) await client.client.createKey({ name: 'key1', @@ -35,7 +36,7 @@ describe('DB Client', () => { iat: Date.now(), name: 'key2', }, - SALT + config.SALT ) const key2 = await client.client.createKey({ name: 'key2', diff --git a/packages/api/test/maintenance.spec.js b/packages/api/test/maintenance.spec.js index 7cb59b4231..9c2f69826c 100644 --- a/packages/api/test/maintenance.spec.js +++ b/packages/api/test/maintenance.spec.js @@ -1,14 +1,31 @@ import assert from 'assert' import { - modes, withMode, READ_ONLY, READ_WRITE, NO_READ_OR_WRITE, - setMaintenanceModeGetter, } from '../src/middleware/maintenance.js' +import { + getServiceConfig, + overrideServiceConfigForTesting, +} from '../src/config.js' + +const baseConfig = getServiceConfig() + +/** @param {import('../src/middleware/maintenance.js').Mode} mode */ +const setMode = (mode) => { + overrideServiceConfigForTesting({ + ...baseConfig, + MAINTENANCE_MODE: mode, + }) +} + describe('maintenance middleware', () => { + afterEach(() => { + overrideServiceConfigForTesting(baseConfig) + }) + it('should throw error when in maintenance', () => { /** @type {import('../src/bindings').Handler} */ let handler @@ -16,21 +33,20 @@ describe('maintenance middleware', () => { // @ts-expect-error not passing params to our test handler handler() } - handler = withMode(() => new Response(), READ_ONLY) - setMaintenanceModeGetter(() => READ_WRITE) + setMode(READ_WRITE) assert.doesNotThrow(block) - setMaintenanceModeGetter(() => READ_ONLY) + setMode(READ_ONLY) assert.doesNotThrow(block) - setMaintenanceModeGetter(() => NO_READ_OR_WRITE) + setMode(NO_READ_OR_WRITE) assert.throws(block, /API undergoing maintenance/) handler = withMode(() => new Response(), READ_WRITE) - setMaintenanceModeGetter(() => READ_WRITE) + setMode(READ_WRITE) assert.doesNotThrow(block) - setMaintenanceModeGetter(() => READ_ONLY) + setMode(READ_ONLY) assert.throws(block, /API undergoing maintenance/) - setMaintenanceModeGetter(() => NO_READ_OR_WRITE) + setMode(NO_READ_OR_WRITE) assert.throws(block, /API undergoing maintenance/) }) @@ -45,7 +61,7 @@ describe('maintenance middleware', () => { const invalidModes = ['', null, undefined, ['r', '-'], 'rwx'] invalidModes.forEach((m) => { // @ts-expect-error purposely passing invalid mode - setMaintenanceModeGetter(() => m) + setMode(m) assert.throws(block, /invalid maintenance mode/) }) }) diff --git a/packages/api/test/nfts-upload.spec.js b/packages/api/test/nfts-upload.spec.js index 90c6f5db3b..86c1bbd1c1 100644 --- a/packages/api/test/nfts-upload.spec.js +++ b/packages/api/test/nfts-upload.spec.js @@ -12,9 +12,9 @@ import { rawClient, } from './scripts/helpers.js' import { createCar } from './scripts/car.js' -import { S3_ENDPOINT, S3_BUCKET_NAME } from './scripts/worker-globals.js' import { build } from 'ucan-storage/ucan-storage' import { KeyPair } from 'ucan-storage/keypair' +import { getServiceConfig } from '../src/config.js' describe('NFT Upload ', () => { /** @type{DBTestClient} */ @@ -591,5 +591,6 @@ function getRandomBytes(n) { * @returns */ function expectedBackupUrl(root, userId, carHash) { + const { S3_ENDPOINT, S3_BUCKET_NAME } = getServiceConfig() return `${S3_ENDPOINT}/${S3_BUCKET_NAME}/raw/${root}/nft-${userId}/${carHash}.car` } diff --git a/packages/api/test/scripts/helpers.js b/packages/api/test/scripts/helpers.js index 3b4337aca0..2ae31b1655 100644 --- a/packages/api/test/scripts/helpers.js +++ b/packages/api/test/scripts/helpers.js @@ -1,24 +1,22 @@ import { Cluster } from '@nftstorage/ipfs-cluster' import { signJWT } from '../../src/utils/jwt.js' -import { - SALT, - CLUSTER_API_URL, - CLUSTER_BASIC_AUTH_TOKEN, -} from './worker-globals.js' import { PostgrestClient } from '@supabase/postgrest-js' import { DBClient } from '../../src/utils/db-client.js' +import { getServiceConfig } from '../../src/config.js' -export const cluster = new Cluster(CLUSTER_API_URL, { - headers: { Authorization: `Basic ${CLUSTER_BASIC_AUTH_TOKEN}` }, +const config = getServiceConfig() + +export const cluster = new Cluster(config.CLUSTER_API_URL, { + headers: { Authorization: `Basic ${config.CLUSTER_BASIC_AUTH_TOKEN}` }, }) -export const rawClient = new PostgrestClient(DATABASE_URL, { +export const rawClient = new PostgrestClient(config.DATABASE_URL, { headers: { - Authorization: `Bearer ${DATABASE_TOKEN}`, + Authorization: `Bearer ${config.DATABASE_TOKEN}`, }, }) -export const client = new DBClient(DATABASE_URL, DATABASE_TOKEN) +export const client = new DBClient(config.DATABASE_URL, config.DATABASE_TOKEN) /** * @param {{publicAddress?: string, issuer?: string, name?: string}} userInfo @@ -35,7 +33,7 @@ export async function createTestUser({ iat: Date.now(), name: 'test', }, - SALT + config.SALT ) return createTestUserWithFixedToken({ token, publicAddress, issuer, name }) @@ -89,8 +87,12 @@ export async function createTestUserWithFixedToken({ }) .single() - if (error || !user) { - throw new Error('error creating user') + if (error) { + throw error + } + + if (!user) { + throw new Error('error creating user: no error returned, but user is null') } await client.createKey({ diff --git a/packages/api/test/scripts/worker-globals.js b/packages/api/test/scripts/worker-globals.js deleted file mode 100644 index a1d43fa756..0000000000 --- a/packages/api/test/scripts/worker-globals.js +++ /dev/null @@ -1,22 +0,0 @@ -export const SALT = 'secret' -export const DEBUG = 'true' -export const CLUSTER_API_URL = 'http://localhost:9094' -// will be used with we can active auth in cluster base64 of test:test -export const CLUSTER_BASIC_AUTH_TOKEN = 'dGVzdDp0ZXN0' -export const CLUSTER_SERVICE = '' -export const MAGIC_SECRET_KEY = 'test' -export const ENV = 'test' -export const SENTRY_DSN = 'https://test@test.ingest.sentry.io/0000000' -export const BRANCH = 'test' -export const VERSION = 'test' -export const COMMITHASH = 'test' -export const MAINTENANCE_MODE = 'rw' -export const METAPLEX_AUTH_TOKEN = 'metaplex-test-token' -export const MAILCHIMP_API_KEY = '' -export const LOGTAIL_TOKEN = '' -export const S3_ENDPOINT = 'http://localhost:9095' -export const S3_REGION = 'test' -export const S3_ACCESS_KEY_ID = 'test' -export const S3_SECRET_ACCESS_KEY = 'test' -export const S3_BUCKET_NAME = 'test' -export const PRIVATE_KEY = 'xmbtWjE9eYuAxae9G65lQSkw36HV6H+0LSFq2aKqVwY=' diff --git a/packages/api/test/version.spec.js b/packages/api/test/version.spec.js index 27b34aa0d3..9b5a1d9def 100644 --- a/packages/api/test/version.spec.js +++ b/packages/api/test/version.spec.js @@ -1,20 +1,16 @@ import assert from 'assert' -import { - VERSION, - COMMITHASH, - BRANCH, - MAINTENANCE_MODE, -} from './scripts/worker-globals.js' +import { getServiceConfig } from '../src/config.js' describe('/version', () => { it('should get version information', async () => { + const cfg = getServiceConfig() const res = await fetch('/version') assert(res) assert(res.ok) const { version, commit, branch, mode } = await res.json() - assert.strictEqual(version, VERSION) - assert.strictEqual(commit, COMMITHASH) - assert.strictEqual(branch, BRANCH) - assert.strictEqual(mode, MAINTENANCE_MODE) + assert.strictEqual(version, cfg.NFT_STORAGE_VERSION) + assert.strictEqual(commit, cfg.NFT_STORAGE_COMMITHASH) + assert.strictEqual(branch, cfg.NFT_STORAGE_BRANCH) + assert.strictEqual(mode, cfg.MAINTENANCE_MODE) }) })