split docker into base and normal image for speed

.github/workflows/docker-publish.yml

@@ -39,8 +39,11 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           context: .
+          file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
             ${{ env.IMAGE_NAME }}:latest
             ${{ env.IMAGE_NAME }}:${{ github.sha }}
+          cache-from: type=gha,scope=multiarch
+          cache-to: type=gha,mode=max,scope=multiarch

.github/workflows/nightly-base-image.yml

@@ -0,0 +1,45 @@
+name: Nightly Base Image Build
+
+on:
+  schedule:
+    # Runs at 2:00 AM UTC every day
+    - cron: '0 2 * * *'
+  workflow_dispatch:  # Allow manual triggering
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  IMAGE_NAME: superglueai/superglue-base
+
+jobs:
+  build-base-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Build and push multi-architecture base image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile.base
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.IMAGE_NAME }}:latest
+            ${{ env.IMAGE_NAME }}:${{ github.sha }}
+          cache-from: type=gha,scope=base-multiarch
+          cache-to: type=gha,mode=max,scope=base-multiarch 

docker/DOCKER.md

@@ -0,0 +1,50 @@
+# Docker Build Process
+
+We use a two stage build process to increase deploy speed.
+
+1. A base image containing all dependencies is built nightly
+2. The application image is built on top of that image
+
+## Base Image
+
+The base image contains:
+- Node.js and npm
+- All project dependencies
+- Build tools (TypeScript, Next.js, Turbo)
+- Playwright with browser dependencies
+
+This image is automatically built every night via GitHub Actions and pushed to DockerHub as a multi-architecture image at `superglueai/superglue-base:latest`.
+
+## Application Image
+
+The application image contains:
+- The base image
+- The application source code
+- Updated dependencies
+
+## Building Locally
+
+Because the application image now depends on the base image, we made a script
+which ensures that we can still build all images locally.
+
+```bash
+./docker/build-local-images.sh
+# OR, using the online base image
+./docker/build-local-images.sh --use-online-base
+```
+
+### Running the Application
+
+```bash
+docker run -p 3000:3000 -p 3001:3001 superglue:latest
+```
+
+## CI/CD Integration
+1. **Nightly Base Image Workflow** (`.github/workflows/nightly-base-image.yml`):
+   - Builds a multi-architecture base image daily (2am UTC)
+   - Pushes to DockerHub as `superglueai/superglue-base:latest`
+
+2. **Application Image Workflow** (`.github/workflows/docker-publish.yml`):
+   - Triggered on pushes to main, updates dependencies from base image
+   - Pushes to DockerHub as `superglueai/superglue:latest`
+

docker/Dockerfile

@@ -0,0 +1,32 @@
+# Build stage using pre-built base image
+
+# Use DockerHub image
+FROM superglueai/superglue-base:latest AS builder
+
+WORKDIR /usr/src/app
+
+# Copy all source code (including potentially updated package.json files)
+COPY . .
+
+# Update dependencies - ensure any new or updated dependencies are installed
+RUN npm install
+
+# Build the application
+RUN npm run build
+
+# Production stage using pre-built base image
+FROM superglueai/superglue-base:latest
+
+WORKDIR /usr/src/app
+
+# Copy built files from builder stage
+COPY --from=builder /usr/src/app/packages/core/dist ./packages/core/dist
+COPY --from=builder /usr/src/app/packages/web/.next ./packages/web/.next
+COPY --from=builder /usr/src/app/packages/web/public ./packages/web/public
+COPY --from=builder /usr/src/app/packages/shared/dist ./packages/shared/dist
+
+# Expose ports for both servers
+EXPOSE 3000 3001
+
+# Start both servers using turbo
+CMD ["npm", "run", "start"]

Dockerfile → docker/Dockerfile.base

@@ -3,7 +3,7 @@ FROM node:22-slim AS builder
 
 WORKDIR /usr/src/app
 
-# Copy package files first to leverage layer caching
+# Copy package files for dependency installation
 COPY package*.json ./
 COPY turbo.json ./
 COPY api.graphql ./
@@ -23,13 +23,7 @@ COPY packages/shared/tsconfig.json ./packages/shared/
 RUN npm install && \
     npm install -g typescript next turbo
 
-# Copy source code
-COPY . .
-
-# Build the application
-RUN npm run build
-
-# Production stage
+# Production stage with installed dependencies
 FROM node:22-slim
 
 WORKDIR /usr/src/app
@@ -47,14 +41,5 @@ RUN npm ci --omit=dev && \
     npm install -g next turbo cross-env && \
     npx playwright install --with-deps
 
-# Copy built files from builder stage
-COPY --from=builder /usr/src/app/packages/core/dist ./packages/core/dist
-COPY --from=builder /usr/src/app/packages/web/.next ./packages/web/.next
-COPY --from=builder /usr/src/app/packages/web/public ./packages/web/public
-COPY --from=builder /usr/src/app/packages/shared/dist ./packages/shared/dist
-
-# Expose ports for both servers
-EXPOSE 3000 3001
-
-# Start both servers using turbo
-CMD ["npm", "run", "start"]
+# This is a base image with all dependencies installed
+# It does not contain any application code 

docker/build-local-images.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+# Default to using local base image
+USE_ONLINE_BASE=false
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --use-online-base)
+      USE_ONLINE_BASE=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      echo "Usage: $0 [--use-online-base]"
+      exit 1
+      ;;
+  esac
+done
+
+# Determine the current architecture
+ARCH=$(uname -m | grep -q "x86_64" && echo "amd64" || echo "arm64")
+echo "Building Docker images for architecture: $ARCH"
+
+# Set image names and tags
+BASE_IMAGE_NAME="superglue-base"
+APP_IMAGE_NAME="superglue"
+ONLINE_BASE_IMAGE="superglueai/superglue-base"
+BASE_IMAGE_TAG="latest"
+APP_IMAGE_TAG="latest"
+
+echo "=== Step 1: Building base image ==="
+docker build \
+  -f docker/Dockerfile.base \
+  -t $BASE_IMAGE_NAME:$BASE_IMAGE_TAG \
+  -t $BASE_IMAGE_NAME:$BASE_IMAGE_TAG-$ARCH \
+  .
+
+# Create temporary Dockerfile based on whether to use online or local base image
+if [ "$USE_ONLINE_BASE" = true ]; then
+  echo "=== Step 2: Building application image using online base image ==="
+  BASE_IMAGE_REF="$ONLINE_BASE_IMAGE:$BASE_IMAGE_TAG"
+else
+  echo "=== Step 2: Building application image using local base image ==="
+  BASE_IMAGE_REF="$BASE_IMAGE_NAME:$BASE_IMAGE_TAG"
+
+  # Create a temporary Dockerfile that uses the local base image
+  TMP_DOCKERFILE=$(mktemp)
+  sed "s|FROM superglueai/superglue-base:latest|FROM $BASE_IMAGE_REF|g" docker/Dockerfile > $TMP_DOCKERFILE
+
+  # Build the application image using the temporary Dockerfile
+  docker build \
+    -f $TMP_DOCKERFILE \
+    -t $APP_IMAGE_NAME:$APP_IMAGE_TAG \
+    -t $APP_IMAGE_NAME:$APP_IMAGE_TAG-$ARCH \
+    .
+
+  # Remove the temporary Dockerfile
+  rm $TMP_DOCKERFILE
+
+  echo -e "\n=== Build Complete ==="
+  echo "Images built:"
+  echo "- $BASE_IMAGE_NAME:$BASE_IMAGE_TAG"
+  echo "- $BASE_IMAGE_NAME:$BASE_IMAGE_TAG-$ARCH"
+  echo "- $APP_IMAGE_NAME:$APP_IMAGE_TAG"
+  echo "- $APP_IMAGE_NAME:$APP_IMAGE_TAG-$ARCH"
+  echo -e "\nTo run the application:"
+  echo "docker run -p 3000:3000 -p 3001:3001 $APP_IMAGE_NAME:$APP_IMAGE_TAG-$ARCH"
+  exit 0
+fi
+
+# If using online base image, use the original Dockerfile
+docker build \
+  -f docker/Dockerfile \
+  -t $APP_IMAGE_NAME:$APP_IMAGE_TAG \
+  -t $APP_IMAGE_NAME:$APP_IMAGE_TAG-$ARCH \
+  .
+
+echo -e "\n=== Build Complete ==="
+echo "Images built:"
+echo "- $BASE_IMAGE_NAME:$BASE_IMAGE_TAG"
+echo "- $BASE_IMAGE_NAME:$BASE_IMAGE_TAG-$ARCH"
+echo "- $APP_IMAGE_NAME:$APP_IMAGE_TAG"
+echo "- $APP_IMAGE_NAME:$APP_IMAGE_TAG-$ARCH"
+echo -e "\nTo run the application:"
+echo "docker run -p 3000:3000 -p 3001:3001 $APP_IMAGE_NAME:$APP_IMAGE_TAG-$ARCH"