Merge branch 'docker' into docker-prod

Author: Mario-SO

.github/workflows/build_server.yml

@@ -22,7 +22,7 @@ jobs:
           else
             echo "ENV_TAG=staging" >> $GITHUB_OUTPUT
             echo "COMPOSE_FILE=docker-compose.staging.yml" >> $GITHUB_OUTPUT
-            echo "STACK_NAME=streameth-staging" >> $GITHUB_OUTPUT
+            echo "STACK_NAME=staging" >> $GITHUB_OUTPUT
           fi
   build-and-push-images:
     needs: prepare-env

.github/workflows/clean_caches.yml

@@ -0,0 +1,32 @@
+name: cleanup caches
+on:
+  workflow_dispatch:  # Adding manual trigger option
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup all caches
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For pull request events, clean specific PR branch caches
+            BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
+            echo "Cleaning caches for PR branch: $BRANCH"
+            cacheKeysForPR=$(gh cache list --ref $BRANCH --limit 100 --json id --jq '.[].id')
+          else
+            # For manual dispatch, clean all caches
+            echo "Cleaning all repository caches"
+            cacheKeysForPR=$(gh cache list --limit 100 --json id --jq '.[].id')
+          fi
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh cache delete $cacheKey --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.TOKEN }}
+          GH_REPO: ${{ github.repository }}

docker-compose.prod.yml

@@ -12,7 +12,7 @@ x-common-environment: &common-environment
   DB_NAME: streameth-prod
   JWT_SECRET_FILE: /run/secrets/jwt-secret
   JWT_EXPIRY: 30d
-  CORS_ORIGIN: "[]"
+  CORS_ORIGIN: "*"
   CORS_CREDENTIALS: "true"
   CORS_PROXY_URL: ""
   LIVEPEER_API_KEY_FILE: /run/secrets/prod-livepeer-api-key
@@ -43,7 +43,7 @@ x-common-environment: &common-environment
   REMOTION_WEBHOOK_SECRET_FILE: /run/secrets/remotion-webhook-secret
   REMOTION_ID: pebs
   REMOTION_SITE_NAME: https://streameth.org
-  REMOTION_WEBHOOK_URL: https://prod.api.streameth.org/remotion-webhook
+  REMOTION_WEBHOOK_URL: https://prod.api.streameth.org/webhook/remotion
   PIPEDREAM_AUTH_TOKEN_FILE: /run/secrets/pipedream-auth-token
   DEVCON_UPLOAD_ENDPOINT_FILE: /run/secrets/devcon-upload-endpoint
   LOG_DIR: logs
@@ -212,11 +212,11 @@ services:
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=traefik-public"
-        - "traefik.http.routers.reel-creator-prod.rule=Host(`prod.remotion.streameth.org`)"
+        - "traefik.http.routers.reel-creator-prod.rule=Host(`prod.api.streameth.org`)"
         - "traefik.http.routers.reel-creator-prod.service=reel-creator-prod"
         - "traefik.http.routers.reel-creator-prod.tls=true"
         - "traefik.http.routers.reel-creator-prod.tls.certresolver=myresolver"
-        - "traefik.http.services.reel-creator-prod.loadbalancer.server.port=4000"
+        - "traefik.http.services.reel-creator-prod.loadbalancer.server.port=3000"
     secrets:
       - aws-access-key
       - aws-secret-key
@@ -231,7 +231,6 @@ services:
       - redis
     environment: 
       <<: *common-environment
-      REMOTION_WEBHOOK_URL: https://prod.api.streameth.org/remotion-webhook
     deploy:
       <<: *common-deploy
       mode: replicated

docker-compose.router.yml

@@ -28,7 +28,7 @@ services:
       - "traefik.http.routers.to-staging.service=staging-server"
       - "traefik.http.routers.to-staging.tls=true"
       - "traefik.http.routers.to-staging.tls.certresolver=myresolver"
-      # CORS middleware
+      # CORS middleware for staging
       - "traefik.http.routers.to-staging.middlewares=to-staging-cors"
       - "traefik.http.middlewares.to-staging-cors.headers.accesscontrolalloworiginlist=*"
       - "traefik.http.middlewares.to-staging-cors.headers.accesscontrolalloworiginlist=http://localhost:3000,http://localhost,https://staging.api.streameth.org"
@@ -41,6 +41,14 @@ services:
       - "traefik.http.routers.to-prod.service=prod-server"
       - "traefik.http.routers.to-prod.tls=true"
       - "traefik.http.routers.to-prod.tls.certresolver=myresolver"
+      # CORS middleware for production
+      - "traefik.http.routers.to-prod.middlewares=to-prod-cors"
+      - "traefik.http.middlewares.to-prod-cors.headers.accesscontrolalloworiginlist=*"
+      - "traefik.http.middlewares.to-prod-cors.headers.accesscontrolalloworiginlist=http://localhost:3000,http://localhost,https://prod.api.streameth.org"
+      - "traefik.http.middlewares.to-prod-cors.headers.accesscontrolallowheaders=*"
+      - "traefik.http.middlewares.to-prod-cors.headers.accesscontrolallowmethods=GET,OPTIONS,PUT,POST,DELETE,PATCH"
+      - "traefik.http.middlewares.to-prod-cors.headers.accesscontrolmaxage=100"
+      - "traefik.http.middlewares.to-prod-cors.headers.addvaryheader=true"
     ports:
       - "80:80"
       - "443:443"

docker-compose.staging.yml

@@ -7,7 +7,7 @@ x-common-environment: &common-environment
   BASE_URL: https://staging.api.streameth.org
   PLAYER_URL: https://staging.player.streameth.org
   DB_PASSWORD_FILE: /run/secrets/db-password
-  DB_HOST: streameth-staging_mongodb
+  DB_HOST: staging_mongodb
   DB_USER: root
   DB_NAME: streameth-staging
   JWT_SECRET_FILE: /run/secrets/jwt-secret
@@ -43,7 +43,7 @@ x-common-environment: &common-environment
   REMOTION_WEBHOOK_SECRET_FILE: /run/secrets/remotion-webhook-secret
   REMOTION_ID: pebs
   REMOTION_SITE_NAME: rendering-engine
-  REMOTION_WEBHOOK_URL: https://staging.api.streameth.org/remotion-webhook
+  REMOTION_WEBHOOK_URL: https://staging.api.streameth.org/webhook/remotion
   PIPEDREAM_AUTH_TOKEN_FILE: /run/secrets/pipedream-auth-token
   DEVCON_UPLOAD_ENDPOINT_FILE: /run/secrets/devcon-upload-endpoint
   LOG_DIR: logs
@@ -53,8 +53,8 @@ x-common-environment: &common-environment
   REDIS_PORT: 6379
   OPENAI_API_KEY_FILE: /run/secrets/openai-api-key
   # AWS Configuration read from GitHub Actions secrets
-  AWS_ACCESS_KEY_ID_FILE: /run/secrets/aws-access-key
-  AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/aws-secret-key
+  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
 
 x-common-deploy: &common-deploy
   restart_policy:
@@ -104,7 +104,9 @@ services:
     networks:
       - traefik-public
       - default
-    environment: *common-environment
+    environment: 
+      NODE_ENV: staging
+      <<: *common-environment
     deploy:
       <<: *common-deploy
       labels:
@@ -162,13 +164,13 @@ services:
   reel-creator:
     image: ghcr.io/streamethorg/streameth-platform/reel-creator:staging
     environment:
-      NODE_ENV: development
+      NODE_ENV: staging
       SERVER_WEBHOOK_URL: https://staging.api.streameth.org/webhook/remotion
       SERVER_WEBHOOK_SECRET_FILE: /run/secrets/remotion-webhook-secret
       AWS_ACCESS_KEY_ID_FILE: /run/secrets/aws-access-key
       AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/aws-secret-key
       BUCKET_NAME: streameth-develop
-      BUCKET_URL: https://streameth-staging.ams3.digitaloceanspaces.com
+      BUCKET_URL: https://streameth-develop.ams3.digitaloceanspaces.com
       SPACES_KEY_FILE: /run/secrets/spaces-key
       SPACES_SECRET_FILE: /run/secrets/spaces-secret
       SITE_NAME: rendering-engine
@@ -181,11 +183,11 @@ services:
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=traefik-public"
-        - "traefik.http.routers.reel-creator-staging.rule=Host(`staging.remotion.streameth.org`)"
+        - "traefik.http.routers.reel-creator-staging.rule=Host(`staging.api.streameth.org`)"
+        - "traefik.http.services.reel-creator-staging.loadbalancer.server.port=3000"
         - "traefik.http.routers.reel-creator-staging.service=reel-creator-staging"
         - "traefik.http.routers.reel-creator-staging.tls=true"
         - "traefik.http.routers.reel-creator-staging.tls.certresolver=myresolver"
-        - "traefik.http.services.reel-creator-staging.loadbalancer.server.port=3000"
     secrets:
       - aws-access-key
       - aws-secret-key
@@ -198,7 +200,6 @@ services:
     volumes:
       - mongodb_data:/data/db
     environment:
-      NODE_ENV: development
       MONGO_INITDB_DATABASE: streameth-staging
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD_FILE: /run/secrets/db-password
@@ -244,7 +245,6 @@ services:
       - redis
     environment: 
       <<: *common-environment
-      REMOTION_WEBHOOK_URL: https://staging.api.streameth.org/remotion-webhook
     deploy:
       <<: *common-deploy
       mode: replicated

packages/reel-creator/app/api/lambda/progress/route.ts

@@ -13,17 +13,9 @@ import {
   TIMEOUT,
   WEBHOOK_SECRET,
   WEBHOOK_URL,
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
 } from "../../../../config.mjs";
-import * as fs from 'fs';
-
-// Read AWS credentials from secret files
-const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID_FILE ? 
-  fs.readFileSync(process.env.AWS_ACCESS_KEY_ID_FILE, 'utf8').trim() : 
-  process.env.AWS_ACCESS_KEY_ID;
-
-const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY_FILE ? 
-  fs.readFileSync(process.env.AWS_SECRET_ACCESS_KEY_FILE, 'utf8').trim() : 
-  process.env.AWS_SECRET_ACCESS_KEY;
 
 export const POST = executeApi<ProgressResponse, typeof ProgressRequest>(
   ProgressRequest,
@@ -35,8 +27,15 @@ export const POST = executeApi<ProgressResponse, typeof ProgressRequest>(
       !WEBHOOK_SECRET ||
       !SITE_NAME
     ) {
+      console.error('Missing required environment variables:', {
+        hasAwsKeyId: !!AWS_ACCESS_KEY_ID,
+        hasAwsSecretKey: !!AWS_SECRET_ACCESS_KEY,
+        hasWebhookUrl: !!WEBHOOK_URL,
+        hasWebhookSecret: !!WEBHOOK_SECRET,
+        hasSiteName: !!SITE_NAME
+      });
       throw new TypeError(
-        "Set up Remotion Lambda to render videos. See the README.md for how to do so."
+        "Missing required environment variables for Remotion Lambda. Check the logs for details."
       );
     }
 

packages/reel-creator/app/api/lambda/render/route.ts

@@ -23,13 +23,6 @@ import { RenderRequest } from "../../../../types/schema";
 export const POST = executeApi<RenderMediaOnLambdaOutput, typeof RenderRequest>(
 	RenderRequest,
 	async (req, body) => {
-		// console.log('Debug environment variables:');
-		// console.log('AWS_ACCESS_KEY_ID:', AWS_ACCESS_KEY_ID ? 'defined' : 'undefined');
-		// console.log('AWS_SECRET_ACCESS_KEY:', AWS_SECRET_ACCESS_KEY ? 'defined' : 'undefined');
-		// console.log('WEBHOOK_URL:', WEBHOOK_URL);
-		// console.log('WEBHOOK_SECRET:', WEBHOOK_SECRET ? 'defined' : 'undefined');
-		// console.log('SITE_NAME:', SITE_NAME);
-
 		if (
 			!AWS_ACCESS_KEY_ID ||
 			!AWS_SECRET_ACCESS_KEY ||

packages/reel-creator/config.mjs

@@ -1,27 +1,32 @@
 import { VERSION } from "remotion/version";
-
-console.log('Raw environment variables in config.mjs:');
-console.log('SERVER_WEBHOOK_URL:', process.env.SERVER_WEBHOOK_URL);
-console.log('SERVER_WEBHOOK_SECRET_FILE:', process.env.SERVER_WEBHOOK_SECRET_FILE);
-console.log('AWS_ACCESS_KEY_ID:', process.env.AWS_ACCESS_KEY_ID);
-console.log('AWS_SECRET_ACCESS_KEY:', process.env.AWS_SECRET_ACCESS_KEY);
+import * as fs from 'fs';
 
 /**
  * Use autocomplete to get a list of available regions.
  * @type {import('@remotion/lambda').AwsRegion}
  */
 
+
 const readSecretFile = (path) => {
-    if (process.env.NODE_ENV === 'development') {
-      return path;
-    }
-    try {
-      return fs.readFileSync(path, 'utf8').trim();
-    } catch (error) {
-      console.error(`Error reading secret file ${path}:`, error);
-      throw error;
-    }
-  };
+  // Add debug logging
+  console.log('Attempting to read secret file, path:', path);
+  
+  if (!path) {
+    console.warn('No path provided for secret file');
+    return undefined;
+  }
+
+  if (process.env.NODE_ENV === 'development') {
+    return path;
+  }
+
+  try {
+    return fs.readFileSync(path, 'utf8').trim();
+  } catch (error) {
+    console.error(`Error reading secret file ${path}:`, error);
+    return undefined;
+  }
+};
 
 export const REGION = "us-east-2";
 
@@ -31,7 +36,6 @@ export const DISK = 10240;
 export const TIMEOUT = 900;
 
 export const WEBHOOK_URL = process.env.SERVER_WEBHOOK_URL;
-export const WEBHOOK_SECRET = readSecretFile(process.env.SERVER_WEBHOOK_SECRET_FILE);
-export const AWS_ACCESS_KEY_ID = readSecretFile(process.env.AWS_ACCESS_KEY_ID_FILE);
-export const AWS_SECRET_ACCESS_KEY = readSecretFile(process.env.AWS_SECRET_ACCESS_KEY_FILE);
-
+export const WEBHOOK_SECRET = process.env.SERVER_WEBHOOK_SECRET_FILE;
+export const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
+export const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;

packages/server/src/app.ts

@@ -15,14 +15,15 @@ import { RegisterRoutes } from './routes/routes';
 import * as swaggerDocument from './swagger/swagger.json';
 import { logger } from './utils/logger';
 import ErrorMiddleware from './middlewares/error.middleware';
+
 class App {
   public app: express.Application;
   private env: string;
   private port: string | number;
 
   constructor() {
     this.app = express();
-    this.env = config.appEnv || 'development';
+    this.env = config.appEnv;
     this.port = config.port || 3400;
 
     this.connectToDatabase();
@@ -39,7 +40,7 @@ class App {
   listen() {
     this.app.listen(this.port, () => {
       logger.info(`=================================`);
-      logger.info(`======= ENV: ${this.env} =======`);
+      logger.info(`======= ENV: ${this.env} ========`);
       logger.info(`🚀 App listening on the port ${this.port}`);
       logger.info(`=================================`);
     });
@@ -57,7 +58,7 @@ class App {
   getServer() {
     return this.app;
   }
-
+  
   private async connectToDatabase() {
     if (this.env !== 'production') {
       set('debug', true);

packages/server/src/config/index.ts

@@ -87,7 +87,7 @@ export const config = {
   remotion: {
     id: validatedEnv.REMOTION_ID,
     host: validatedEnv.REMOTION_BASE_URL,
-    webhookSecretKey: validatedEnv.REMOTION_WEBHOOK_SECRET_FILE,
+    webhookSecretKey: readSecretFile(validatedEnv.REMOTION_WEBHOOK_SECRET_FILE),
   },
   redis: {
     host: validatedEnv.REDIS_HOST,

packages/server/src/databases/index.ts

@@ -1,5 +1,12 @@
 import { config } from '@config';
 const { host, user, name, password } = config.db;
+
+console.log('Database connection details:');
+console.log('Host:', host);
+console.log('User:', user);
+console.log('Database:', name);
+console.log('Password length:', password?.length);
+
 export const dbConnection = {
   url: `mongodb://${user}:${password}@${host}/${name}?authSource=admin&retryWrites=true&w=majority`,
   // For local development use this url

packages/server/src/services/clipEditor.service.ts

@@ -131,7 +131,6 @@ export class ClipEditorService extends SessionService {
 
     const data = await response.json();
 
-
     await ClipEditor.findByIdAndUpdate(
       clipEditor._id,
       {