add split build

package.json

@@ -11,7 +11,7 @@
     "patch-package": "^8.0.0",
     "postinstall-postinstall": "^2.1.0",
     "syncpack": "^9.8.4",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   },
   "lint-staged": {
     "*.{ts,tsx}": [

packages/cli-common/package.json

@@ -18,8 +18,8 @@
     "@jest/globals": "29.7.0",
     "@types/lodash": "^4.14.192",
     "@types/node": "18",
-    "@typescript-eslint/eslint-plugin": "6.7.4",
-    "@typescript-eslint/parser": "6.7.4",
+    "@typescript-eslint/eslint-plugin": "6.10.0",
+    "@typescript-eslint/parser": "6.10.0",
     "eslint": "^8.36.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-plugin-import": "^2.27.5",
@@ -29,7 +29,7 @@
     "jest": "29.7.0",
     "shx": "^0.3.3",
     "tslib": "^2.5.0",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   },
   "scripts": {
     "lint": "eslint . --ext .ts,.tsx --cache",

packages/cli/bin/dev

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+require('disposablestack/auto')
 const oclif = require('@oclif/core')
 
 const path = require('path')

packages/cli/bin/run

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 require('source-map-support').install()
+require('disposablestack/auto')
 
 const oclif = require('@oclif/core')
 

packages/cli/package.json

@@ -30,6 +30,7 @@
     "@preevy/driver-kube-pod": "0.0.56",
     "@preevy/driver-lightsail": "0.0.56",
     "@preevy/plugin-github-pr-link": "0.0.56",
+    "disposablestack": "^1.1.2",
     "inquirer": "^8.0.0",
     "iter-tools-es": "^7.5.3",
     "lodash": "^4.17.21",
@@ -42,8 +43,8 @@
     "@types/lodash": "^4.14.192",
     "@types/node": "18",
     "@types/shell-escape": "^0.2.1",
-    "@typescript-eslint/eslint-plugin": "6.7.4",
-    "@typescript-eslint/parser": "6.7.4",
+    "@typescript-eslint/eslint-plugin": "6.10.0",
+    "@typescript-eslint/parser": "6.10.0",
     "eslint": "^8.36.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-oclif": "^4",
@@ -59,7 +60,7 @@
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^4.2.0",
     "tslib": "^2.5.0",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   },
   "oclif": {
     "bin": "preevy",

packages/cli/src/commands/up.ts

@@ -22,6 +22,14 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
   static flags = {
     ...envIdFlags,
     ...tunnelServerFlags,
+    'split-build': Flags.custom<commands.SplitBuildSpec>({
+      description: 'Build locally and deploy remotely using an image registry',
+      required: false,
+      parse: async input => {
+        const pairs = input.split(',')
+        return commands.splitBuildSpecSchema.parse(Object.fromEntries(pairs.map(pair => pair.split('='))))
+      },
+    })(),
     'skip-unchanged-files': Flags.boolean({
       description: 'Detect and skip unchanged files when copying (default: true)',
       default: true,
@@ -144,6 +152,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       cwd: process.cwd(),
       skipUnchangedFiles: flags['skip-unchanged-files'],
       version: this.config.version,
+      splitBuildSpec: flags['split-build'],
     })
 
     this.log(`Preview environment ${envId} provisioned at: ${machine.locationDescription}`)

packages/common/package.json

@@ -24,16 +24,16 @@
     "@types/node": "18",
     "@types/shell-escape": "^0.2.1",
     "@types/ssh2": "^1.11.8",
-    "@typescript-eslint/eslint-plugin": "6.7.4",
-    "@typescript-eslint/parser": "6.7.4",
+    "@typescript-eslint/eslint-plugin": "6.10.0",
+    "@typescript-eslint/parser": "6.10.0",
     "esbuild": "^0.17.14",
     "eslint": "^8.36.0",
     "husky": "^8.0.0",
     "jest": "29.7.0",
     "lint-staged": "^14.0.1",
     "ts-jest": "29.1.1",
     "tsx": "^3.12.3",
-    "typescript": "^5.0.4",
+    "typescript": "^5.2.2",
     "yaml": "^2.3.2"
   },
   "scripts": {

packages/compose-tunnel-agent/package.json

@@ -38,8 +38,8 @@
     "@types/node-fetch": "^2.6.3",
     "@types/shell-escape": "^0.2.1",
     "@types/ssh2": "^1.11.8",
-    "@typescript-eslint/eslint-plugin": "6.7.4",
-    "@typescript-eslint/parser": "6.7.4",
+    "@typescript-eslint/eslint-plugin": "6.10.0",
+    "@typescript-eslint/parser": "6.10.0",
     "esbuild": "^0.17.14",
     "eslint": "^8.36.0",
     "husky": "^8.0.0",
@@ -49,7 +49,7 @@
     "shx": "^0.3.3",
     "strip-ansi": "6.0.0",
     "tsx": "^3.12.3",
-    "typescript": "^5.0.4",
+    "typescript": "^5.2.2",
     "wait-for-expect": "^3.0.2"
   },
   "scripts": {

packages/core/package.json

@@ -52,8 +52,8 @@
     "@types/sshpk": "^1.17.1",
     "@types/tar": "^6.1.4",
     "@types/tar-stream": "^2.2.2",
-    "@typescript-eslint/eslint-plugin": "6.7.4",
-    "@typescript-eslint/parser": "6.7.4",
+    "@typescript-eslint/eslint-plugin": "6.10.0",
+    "@typescript-eslint/parser": "6.10.0",
     "eslint": "^8.36.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-plugin-import": "^2.27.5",
@@ -65,7 +65,7 @@
     "ts-jest": "29.1.1",
     "ts-node": "^10.9.1",
     "tslib": "^2.5.0",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   },
   "scripts": {
     "test": "yarn jest",

packages/core/src/commands/index.ts

@@ -1,7 +1,6 @@
-import up from './up'
-import ls from './ls'
-import shell from './shell'
+export { SplitBuildSpec, splitBuildSpecSchema, default as up } from './up'
+export { default as ls } from './ls'
+export { default as shell } from './shell'
 
-export { up, ls, shell }
 export { urls } from './urls'
 export * as proxy from './proxy'

packages/core/src/commands/up/index.ts

@@ -3,8 +3,11 @@ import fs from 'fs'
 import path from 'path'
 import { rimraf } from 'rimraf'
 import yaml from 'yaml'
+import { spawn } from 'child_process'
+import { mapValues, pickBy } from 'lodash'
+import { childProcessPromise } from '../../child-process'
 import { TunnelOpts } from '../../ssh'
-import { composeModelFilename, fixModelForRemote, localComposeClient, addScriptInjectionsToModel } from '../../compose'
+import { composeModelFilename, fixModelForRemote, localComposeClient, addScriptInjectionsToModel, ComposeBuild, ComposeService } from '../../compose'
 import { ensureCustomizedMachine } from './machine'
 import { wrapWithDockerSocket } from '../../docker'
 import { addComposeTunnelAgentService } from '../../compose-tunnel-agent-client'
@@ -14,6 +17,11 @@ import { Logger } from '../../log'
 import { FileToCopy, uploadWithSpinner } from '../../upload-files'
 import { envMetadata } from '../../env-metadata'
 import { EnvId } from '../../env-id'
+import { SplitBuildSpec } from './split-build'
+import { gitContext } from '../../git'
+import { randomString } from '../../strings'
+
+export { splitBuildSpecSchema, SplitBuildSpec } from './split-build'
 
 const createCopiedFileInDataDir = (
   { projectLocalDataDir, filesToCopy } : {
@@ -35,10 +43,11 @@ const createCopiedFileInDataDir = (
   return result
 }
 
-const calcComposeUpArgs = ({ userSpecifiedServices, debug, cwd } : {
+const calcComposeUpArgs = ({ userSpecifiedServices, debug, cwd, build } : {
   userSpecifiedServices: string[]
   debug: boolean
   cwd: string
+  build: boolean
 }) => {
   const upServices = userSpecifiedServices.length
     ? userSpecifiedServices.concat(COMPOSE_TUNNEL_AGENT_SERVICE_NAME)
@@ -47,7 +56,8 @@ const calcComposeUpArgs = ({ userSpecifiedServices, debug, cwd } : {
   return [
     ...debug ? ['--verbose'] : [],
     '--project-directory', cwd,
-    'up', '-d', '--remove-orphans', '--build',
+    'up', '-d', '--remove-orphans',
+    ...build ? ['--build'] : [],
     ...upServices,
   ]
 }
@@ -59,6 +69,20 @@ const serviceLinkEnvVars = (
     .map(({ name, port, url }) => [`PREEVY_BASE_URI_${name.replace(/[^a-zA-Z0-9_]/g, '_')}_${port}`.toUpperCase(), url])
 )
 
+const isEcr = (registry: string) => Boolean(registry.match(/^[0-9]+\.dkr\.ecr\..*\.amazonaws\.com\/.+/))
+
+const registryImageName = (
+  { registry, image, tag }: {
+    registry: string
+    image: string
+    tag: string
+  },
+  { ecrFormat }: { ecrFormat?: boolean } = {},
+) => {
+  const formatForEcr = ecrFormat === undefined ? isEcr(registry) : ecrFormat
+  return formatForEcr ? `${registry}:${image}-${tag}` : `${registry}/${image}:${tag}`
+}
+
 const up = async ({
   debug,
   machineDriver,
@@ -79,6 +103,7 @@ const up = async ({
   envId,
   expectedServiceUrls,
   projectName,
+  splitBuildSpec,
 }: {
   debug: boolean
   machineDriver: MachineDriver
@@ -99,6 +124,7 @@ const up = async ({
   envId: EnvId
   expectedServiceUrls: { name: string; port: number; url: string }[]
   projectName: string
+  splitBuildSpec?: SplitBuildSpec
 }): Promise<{ machine: MachineBase }> => {
   const remoteDir = remoteProjectDir(projectName)
 
@@ -110,6 +136,14 @@ const up = async ({
     projectName: userSpecifiedProjectName,
   })
 
+  await using cleanup = new AsyncDisposableStack()
+
+  const { machine, connection, userAndGroup, dockerPlatform } = await ensureCustomizedMachine({
+    machineDriver, machineCreationDriver, machineDriverName, envId, log, debug,
+  })
+
+  cleanup.defer(() => connection.close())
+
   const { model: fixedModel, filesToCopy } = await fixModelForRemote(
     { cwd, remoteBaseDir: remoteDir },
     await composeClientWithInjectedArgs.getModel()
@@ -124,10 +158,6 @@ const up = async ({
     createCopiedFile('tunnel_server_public_key', formatPublicKey(hostKey)),
   ])
 
-  const { machine, connection, userAndGroup } = await ensureCustomizedMachine({
-    machineDriver, machineCreationDriver, machineDriverName, envId, log, debug,
-  })
-
   let remoteModel = addComposeTunnelAgentService({
     envId,
     debug,
@@ -150,32 +180,70 @@ const up = async ({
     )
   }
 
-  try {
-    const { exec } = connection
+  if (splitBuildSpec) {
+    const tagSuffix = await gitContext(cwd)?.commit({ short: true }) ?? randomString.lowercaseNumeric(8)
+    const serviceImage = (service: string, tag: string) => registryImageName(
+      {
+        registry: splitBuildSpec.registry,
+        image: `preevy-${envId}-${service}`,
+        tag,
+      },
+      { ecrFormat: splitBuildSpec.ecrFormat },
+    )
 
-    const modelStr = yaml.stringify(remoteModel)
-    log.debug('model', modelStr)
-    const composeFilePath = await createCopiedFile(composeModelFilename, modelStr)
+    const buildServices = mapValues(
+      pickBy(remoteModel.services, ({ build }) => build),
+      ({ build }: { build: ComposeBuild}, service) => {
+        const latestImage = serviceImage(service, 'latest')
+        const thisImage = serviceImage(service, tagSuffix)
+        return ({
+          build: Object.assign(build, {
+            tags: (build.tags ?? []).concat(thisImage, latestImage),
+            cache_from: (build.cache_from ?? []).concat(latestImage),
+          }),
+        })
+      },
+    )
 
-    await exec(`mkdir -p "${remoteDir}"`)
+    const buildFilename = path.join(projectLocalDataDir, 'build.yaml')
+    await fs.promises.writeFile(buildFilename, yaml.stringify({ services: buildServices }))
+
+    await childProcessPromise(spawn('docker', [
+      'buildx', 'bake',
+      '-f', buildFilename,
+      `--set=*.platform=${splitBuildSpec.platform ?? dockerPlatform}`,
+      '--push',
+    ], {
+      stdio: 'inherit',
+    }))
+
+    for (const serviceName of Object.keys(buildServices)) {
+      (remoteModel.services as Record<string, ComposeService>)[serviceName].image = serviceImage(serviceName, tagSuffix)
+    }
+  }
 
-    log.debug('Files to copy', filesToCopy)
+  const modelStr = yaml.stringify(remoteModel)
+  log.debug('model', modelStr)
+  const composeFilePath = await createCopiedFile(composeModelFilename, modelStr)
 
-    await uploadWithSpinner(exec, remoteDir, filesToCopy, skipUnchangedFiles)
+  const { exec } = connection
 
-    const compose = localComposeClient({
-      composeFiles: [composeFilePath.local],
-      projectName: userSpecifiedProjectName,
-    })
-    const composeArgs = calcComposeUpArgs({ userSpecifiedServices, debug, cwd })
+  await exec(`mkdir -p "${remoteDir}"`)
 
-    const withDockerSocket = wrapWithDockerSocket({ connection, log })
+  log.debug('Files to copy', filesToCopy)
 
-    log.info(`Running: docker compose up ${composeArgs.join(' ')}`)
-    await withDockerSocket(() => compose.spawnPromise(composeArgs, { stdio: 'inherit' }))
-  } finally {
-    await connection.close()
-  }
+  await uploadWithSpinner(exec, remoteDir, filesToCopy, skipUnchangedFiles)
+
+  const compose = localComposeClient({
+    composeFiles: [composeFilePath.local],
+    projectName: userSpecifiedProjectName,
+  })
+  const composeArgs = calcComposeUpArgs({ userSpecifiedServices, debug, cwd, build: !splitBuildSpec })
+
+  const withDockerSocket = wrapWithDockerSocket({ connection, log })
+
+  log.info(`Running: docker compose up ${composeArgs.join(' ')}`)
+  await withDockerSocket(() => compose.spawnPromise(composeArgs, { stdio: 'inherit' }))
 
   return { machine }
 }