chore: refactor service configuration loading (#1914)

* chore: refactor service configuration loading

Refactors the way we load configuration variables. Key changes:

- constants.js is replaced with config.js
- config.js exports `getServiceConfiguration`, which lazily loads
the config into a JS `ServiceConfiguration` object
- default values for unit tests are defined in code, in config.js
- runtime values are dynamically looked up on `globalThis`, to avoid
undefined reference errors if a var is missing.
- if a config variable doesn't exist at runtime, we log a warning and
use the default value

All existing tests pass, but the config code itself isn't tested yet.

* embrace truthiness

* add test for serviceConfigFromVariables

* fancier bool parsing

* add tests for loading config vars & throw on missing vars in prod/staging

* add hack to workaround global VERSION issue

* include all missing vars in error / warning message

* flatten out config object & simplify default value fallback

* fixup for prev commit: get maintenance mode from config

* chore: add namespace to VERSION, etc

* fix maintenence mode tests

* rm dev cruft

* fix rebase cruft

packages/api/pw-test.config.cjs

@@ -6,9 +6,7 @@ const { once } = require('events')
 
 /** @typedef {{ proc: execa.ExecaChildProcess<string> }} ProcessObject */
 
-dotenv.config({
-  path: path.join(__dirname, '../../.env'),
-})
+dotenv.config({ path: path.join(__dirname, '../../.env') })
 
 const cli = path.join(__dirname, 'scripts/cli.js')
 
@@ -31,21 +29,10 @@ module.exports = {
   buildConfig: {
     inject: [path.join(__dirname, './scripts/node-globals.js')],
     plugins: [nodeBuiltinsPlugin],
-    define: {
-      DATABASE_URL: JSON.stringify(process.env.DATABASE_URL),
-      DATABASE_TOKEN: JSON.stringify(process.env.DATABASE_TOKEN),
-    },
   },
   buildSWConfig: {
-    inject: [
-      path.join(__dirname, './scripts/node-globals.js'),
-      path.join(__dirname, './test/scripts/worker-globals.js'),
-    ],
+    inject: [path.join(__dirname, './scripts/node-globals.js')],
     plugins: [nodeBuiltinsPlugin],
-    define: {
-      DATABASE_URL: JSON.stringify(process.env.DATABASE_URL),
-      DATABASE_TOKEN: JSON.stringify(process.env.DATABASE_TOKEN),
-    },
   },
   beforeTests: async () => {
     const mock = await startMockServer('AWS S3', 9095, 'test/mocks/aws-s3')

packages/api/scripts/cli.js

@@ -58,9 +58,9 @@ prog
         inject: [path.join(__dirname, 'node-globals.js')],
         plugins: [PluginAlias],
         define: {
-          VERSION: JSON.stringify(version),
-          COMMITHASH: JSON.stringify(git.long(__dirname)),
-          BRANCH: JSON.stringify(git.branch(__dirname)),
+          NFT_STORAGE_VERSION: JSON.stringify(version),
+          NFT_STORAGE_COMMITHASH: JSON.stringify(git.long(__dirname)),
+          NFT_STORAGE_BRANCH: JSON.stringify(git.branch(__dirname)),
           global: 'globalThis',
         },
         minify: opts.env === 'dev' ? false : true,

packages/api/src/bindings.d.ts

@@ -7,30 +7,74 @@ import { UserOutput, UserOutputKey } from './utils/db-client-types.js'
 import { DBClient } from './utils/db-client.js'
 import { Logging } from './utils/logs.js'
 
-declare global {
-  const SALT: string
-  const DEBUG: string
-  const CLUSTER_SERVICE: 'IpfsCluster' | 'IpfsCluster2' | 'IpfsCluster3'
-  const CLUSTER_API_URL: string
-  const CLUSTER_BASIC_AUTH_TOKEN: string
-  const MAGIC_SECRET_KEY: string
-  const DATABASE_URL: string
-  const DATABASE_TOKEN: string
-  const MAILCHIMP_API_KEY: string
-  const LOGTAIL_TOKEN: string
-  const ENV: 'dev' | 'staging' | 'production'
-  const SENTRY_DSN: string
-  const BRANCH: string
-  const VERSION: string
-  const COMMITHASH: string
-  const MAINTENANCE_MODE: Mode
-  const METAPLEX_AUTH_TOKEN: string
-  const S3_ENDPOINT: string
-  const S3_REGION: string
-  const S3_ACCESS_KEY_ID: string
-  const S3_SECRET_ACCESS_KEY: string
-  const S3_BUCKET_NAME: string
-  const PRIVATE_KEY: string
+export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
+
+export interface ServiceConfiguration {
+  /** Is this a debug build? */
+  DEBUG: boolean
+
+  /** Target runtime environment */
+  ENV: RuntimeEnvironmentName
+
+  /** Semantic version for current build */
+  NFT_STORAGE_VERSION: string
+
+  /** Git branch name of current build */
+  NFT_STORAGE_BRANCH: string
+
+  /** Git commit hash of current build */
+  NFT_STORAGE_COMMITHASH: string
+
+  /** Current maintenance mode */
+  MAINTENANCE_MODE: Mode
+
+  /** Salt for API key generation */
+  SALT: string
+
+  /** API key for special metaplex upload account */
+  METAPLEX_AUTH_TOKEN: string
+
+  /** UCAN private signing key */
+  PRIVATE_KEY: string
+
+  /** API url for active IPFS cluster endpoint */
+  CLUSTER_API_URL: string
+
+  /** Auth token for IPFS culster */
+  CLUSTER_BASIC_AUTH_TOKEN: string
+
+  /** Postgrest endpoint URL */
+  DATABASE_URL: string
+
+  /** Postgrest auth token */
+  DATABASE_TOKEN: string
+
+  /** S3 endpoint URL */
+  S3_ENDPOINT: string
+
+  /** S3 region */
+  S3_REGION: string
+
+  /** S3 access key id */
+  S3_ACCESS_KEY_ID: string
+
+  /** S3 secret key */
+  S3_SECRET_ACCESS_KEY: string
+
+  /** S3 bucket name */
+  S3_BUCKET_NAME: string
+
+  /** Magic link secret key */
+  MAGIC_SECRET_KEY: string
+
+  /** Logtail auth token */
+  LOGTAIL_TOKEN: string
+
+  /** Sentry DSN */
+  SENTRY_DSN: string
+
+  /** Mailchimp api key */
+  MAILCHIMP_API_KEY: string
 }
 
 export interface Ucan {

packages/api/src/cluster.js

@@ -1,9 +1,13 @@
 import { Cluster } from '@nftstorage/ipfs-cluster'
-import { cluster } from './constants.js'
+import { getServiceConfig } from './config.js'
 import { HTTPError } from './errors.js'
 
-const client = new Cluster(cluster.apiUrl, {
-  headers: { Authorization: `Basic ${cluster.basicAuthToken}` },
+const { CLUSTER_API_URL, CLUSTER_BASIC_AUTH_TOKEN } = getServiceConfig()
+
+const client = new Cluster(CLUSTER_API_URL, {
+  headers: {
+    Authorization: `Basic ${CLUSTER_BASIC_AUTH_TOKEN}`,
+  },
 })
 
 /**