self-hosted fixes

absorbb · absorbb · commit 015a6321a0ce · 2024-01-23T17:06:39.000+04:00
console: allow plain CONSOLE_AUTH_TOKENS
diff --git a/docker-start-console.sh b/docker-start-console.sh
@@ -86,7 +86,7 @@ main() {
   if [ -z "$cmd" ]; then
     if [ "$UPDATE_DB" = "1" ] || [ "$UPDATE_DB" = "yes" ] || [ "$UPDATE_DB" = "true" ]; then
       echo "UPDATE_DB is set, updating database schema..."
-      prisma --skip-generate --schema schema.prisma
+      prisma db push --skip-generate --schema schema.prisma
     fi
     echo "Starting the app"
     healthcheck $$ &
@@ -97,7 +97,7 @@ main() {
 
 
   elif [ "$cmd" = "db-prepare" ]; then
-    prisma --skip-generate --schema schema.prisma
+    prisma db push --skip-generate --schema schema.prisma
   else
     echo "ERROR! Unknown command '$cmd'"
   fi
diff --git a/docker/.env.example b/docker/.env.example
@@ -1,17 +1,23 @@
-# Required
+### Required ###
 GITHUB_CLIENT_ID=
 GITHUB_CLIENT_SECRET=
+# Domain name of your Jitsu server where events will be sent. E.g. jitsu.mycompany.com
+INGEST_DOMAIN=localhost
+# Public URL of your Jitsu UI is available. E.g. https://jitsu.mycompany.com:3000 or http://localhost:3000
+JITSU_PUBLIC_URL=http://localhost:3000
 
-# Secrets
-
+#### Secrets ###
+CONSOLE_PASSWORD=change_me
 POSTGRES_PASSWORD=change_me
 REDIS_PASSWORD=change_me
 
-# Optional
-JITSU_PORT=3000
-JITSU_PUBLIC_URL=
+### Optional ###
+JITSU_UI_PORT=3000
+JITSU_INGEST_PORT=8080
+# comma-separated list of test user credentials in format email:password, e.g.: email1@example.com:password1,email2@example.com:password2
+TEST_CREDENTIALS=
 
-# Connectors specific
+### Connectors specific ###
 SYNCS_ENABLED=false
 # Source Connectors License
 # Jitsu is licensed under MIT license, but some source connectors may use other licenses.
@@ -20,8 +26,6 @@ SYNCS_ENABLED=false
 # You can enable all source connectors by setting this variable to false
 # All source connectors will be enabled, but particular connectors license may conflict with license of your product
 MIT_COMPLIANT=true
-
-
 # bulker host how it is reachable from k8s cluster
 EXTERNAL_BULKER_HOST=
 #postgres host how it is reachable from k8s cluster
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -1,5 +1,24 @@
 version: "3.8"
 services:
+  kafka:
+    tty: true
+    image: "bitnami/kafka:3.6.0"
+    environment:
+      TERM: "xterm-256color"
+      KAFKA_CFG_NODE_ID: 0
+      KAFKA_CFG_PROCESS_ROLES: controller,broker
+      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+    healthcheck:
+      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --describe"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    volumes:
+      - ./data/kafka:/bitnami/kafka
+
   redis:
     tty: true
     image: redis:6.2-alpine
@@ -25,128 +44,147 @@ services:
       interval: 1s
       timeout: 10s
       retries: 10
-  #    volumes:
-  #      - ./data/postgres:/var/lib/postgresql/data
-  zookeeper:
+#    volumes:
+#      - ./data/postgres:/var/lib/postgresql/data
+
+
+  console:
     tty: true
+    image: jitsucom/console:beta
+    restart: "unless-stopped"
     platform: linux/amd64
-    image: wurstmeister/zookeeper:latest
-  kafka:
-    tty: true
-    image: wurstmeister/kafka:latest
-    depends_on:
-      - zookeeper
     environment:
-      TERM: "xterm-256color"
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENERS: INTERNAL://0.0.0.0:19093,OUTSIDE://0.0.0.0:19092
-      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:19093,OUTSIDE://localhost:19092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      DATA_DOMAIN: ${INGEST_DOMAIN}
+      ROTOR_URL: "http://rotor:3401"
+      BULKER_URL: "http://bulker:3042"
+      CONSOLE_AUTH_TOKENS: ${CONSOLE_PASSWORD}
+      BULKER_AUTH_KEY: ${BULKER_PASSWORD}
+      MIT_COMPLIANT: ${MIT_COMPLIANT:-true}
+      DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@postgres:5432/postgres?schema=newjitsu"
+      REDIS_URL: "redis://default:${REDIS_PASSWORD:-default}@redis:6379"
+      TEST_CREDENTIALS: ${TEST_CREDENTIALS}
+      TEST_CREDENTIALS_SHOW_LOGIN: "true"
+      GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID}
+      GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET}
+      SYNCS_ENABLED: ${SYNCS_ENABLED:-false}
+      SYNCCTL_URL: "http://syncctl:3043"
+      SYNCCTL_AUTH_KEY: ${SYNCCTL_PASSWORD}
+      GOOGLE_SCHEDULER_KEY: ${GOOGLE_SCHEDULER_KEY}
+      GOOGLE_SCHEDULER_LOCATION: ${GOOGLE_SCHEDULER_LOCATION:-us-central1}
+      NEXTAUTH_URL: "${JITSU_PUBLIC_URL:-${NEXTAUTH_URL:-http://localhost:${JITSU_UI_PORT:-3000}/}}"
+      UPDATE_DB: "true"
     healthcheck:
-      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:19092 --describe"]
-      interval: 5s
-      timeout: 3s
+      test: [ "CMD", "curl", "-f", "http://console:3000/api/healthcheck" ]
+      interval: 2s
+      timeout: 10s
       retries: 30
+    depends_on:
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
+    ports:
+      - "${JITSU_UI_PORT:-3000}:3000"
 
   bulker:
     tty: true
-    image: jitsucom/bulker:latest
+    image: jitsucom/bulker:beta
     platform: linux/amd64
     restart: "unless-stopped"
     environment:
-      BULKER_HTTP_PORT: "3042"
       TERM: "xterm-256color"
-      BULKER_KAFKA_BOOTSTRAP_SERVERS: "kafka:19093"
+      BULKER_KAFKA_BOOTSTRAP_SERVERS: "kafka:9092"
       BULKER_AUTH_TOKENS: ${BULKER_PASSWORD}
-      BULKER_CONFIG_SOURCE: "redis"
+      BULKER_CONFIG_SOURCE: "http://console:3000/api/admin/export/bulker-connections"
+      BULKER_CONFIG_SOURCE_HTTP_AUTH_TOKEN: "service-admin-account:${CONSOLE_PASSWORD}"
+      BULKER_CACHE_DIR: "/tmp/cache"
       REDIS_URL: "redis://default:${REDIS_PASSWORD:-default}@redis:6379"
       BULKER_INTERNAL_TASK_LOG: '{"id":"task_log","metricsKeyPrefix":"syncs","usesBulker":true,"type":"postgres","options":{"mode":"stream"},"credentials":{"host":"postgres","port":5432,"sslMode":"disable","database":"postgres","password":"${POSTGRES_PASSWORD:-default}","username":"postgres","defaultSchema":"public"}}'
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:3042/ready"]
+      test: ["CMD", "curl", "-f", "http://bulker:3042/health"]
       interval: 2s
       timeout: 10s
       retries: 15
     depends_on:
-      redis:
-        condition: service_started
-      kafka:
-        condition: service_healthy
-  syncctl:
-    tty: true
-    image: jitsucom/syncctl:latest
-    platform: linux/amd64
-    restart: "on-failure"
-    environment:
-      HTTP_PORT: "3043"
-      TERM: "xterm-256color"
-      SYNCCTL_SYNCS_ENABLED: ${SYNCS_ENABLED:-false}
-      SYNCCTL_AUTH_TOKENS: ${SYNCCTL_PASSWORD}
-      SYNCCTL_DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@postgres:5432/postgres"
-      SYNCCTL_SIDECAR_DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@${EXTERNAL_DATABASE_HOST}:${PG_PORT:-5437}/postgres"
-      SYNCCTL_BULKER_URL: "http://${EXTERNAL_BULKER_HOST}:${BULKER_PORT:-3045}"
-      SYNCCTL_BULKER_AUTH_TOKEN: ${BULKER_PASSWORD}
-      SYNCCTL_BULKER_LOGS_CONNECTION_ID: task_log
-      SYNCCTL_KUBERNETES_CLIENT_CONFIG: "${SYNCCTL_KUBERNETES_CLIENT_CONFIG:-local}"
-      SYNCCTL_KUBERNETES_CONTEXT: "${SYNCCTL_KUBERNETES_CONTEXT}"
-    volumes:
-      - ./data/syncctl:/etc/syncctl
-    extra_hosts:
-      - "kubernetes:host-gateway"
-    depends_on:
-      bulker:
+      console:
         condition: service_healthy
-      postgres:
+      kafka:
         condition: service_healthy
+
   rotor:
     tty: true
-    image: jitsucom/rotor:latest
+    image: jitsucom/rotor:beta
     platform: linux/amd64
     restart: "unless-stopped"
     environment:
       DISABLE_SERVICE_prisma: "true"
       DISABLE_SERVICE_pg: "true"
       BULKER_URL: "http://bulker:3042"
       BULKER_AUTH_KEY: ${BULKER_PASSWORD}
-      KAFKA_BOOTSTRAP_SERVERS: "kafka:19093"
+      KAFKA_BOOTSTRAP_SERVERS: "kafka:9092"
       REDIS_URL: "redis://default:${REDIS_PASSWORD:-default}@redis:6379"
-      CONFIG_STORE_DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@postgres:5432/postgres"
+      REPOSITORY_BASE_URL: "http://console:3000/api/admin/export/"
+      REPOSITORY_AUTH_TOKEN: "service-admin-account:${CONSOLE_PASSWORD}"
+      REPOSITORY_CACHE_DIR: "/tmp/cache"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://rotor:3401/health" ]
+      interval: 2s
+      timeout: 10s
+      retries: 15
     depends_on:
-      bulker:
+      console:
         condition: service_healthy
-      kafka:
+      bulker:
         condition: service_healthy
-      redis:
-        condition: service_started
-  console:
+
+  ingest:
     tty: true
-    image: jitsucom/console:latest
-    restart: "unless-stopped"
+    image: jitsucom/ingest:beta
     platform: linux/amd64
+    restart: "unless-stopped"
     environment:
-      ROTOR_URL: "http://rotor:3401"
-      BULKER_URL: "http://bulker:3042"
-      BULKER_AUTH_KEY: ${BULKER_PASSWORD}
-      MIT_COMPLIANT: ${MIT_COMPLIANT:-true}
-      DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@postgres:5432/postgres"
-      REDIS_URL: "redis://default:${REDIS_PASSWORD:-default}@redis:6379"
-      TEST_CREDENTIALS: ${TEST_CREDENTIALS}
-      TEST_CREDENTIALS_SHOW_LOGIN: "true"
-      GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID}
-      GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET}
-      SYNCS_ENABLED: ${SYNCS_ENABLED:-false}
-      SYNCCTL_URL: "http://syncctl:3043"
-      SYNCCTL_AUTH_KEY: ${SYNCCTL_PASSWORD}
-      GOOGLE_SCHEDULER_KEY: ${GOOGLE_SCHEDULER_KEY}
-      GOOGLE_SCHEDULER_LOCATION: ${GOOGLE_SCHEDULER_LOCATION:-us-central1}
-      NEXTAUTH_URL: "${JITSU_PUBLIC_URL:-${NEXTAUTH_URL:-http://localhost:${JITSU_PORT:-3000}/}}"
-      UPDATE_DB: "true"
+      TERM: "xterm-256color"
+      INGEST_DATA_DOMAIN: ${INGEST_DOMAIN}
+      INGEST_KAFKA_BOOTSTRAP_SERVERS: "kafka:9092"
+      INGEST_AUTH_TOKENS: ${BULKER_PASSWORD}
+      INGEST_REPOSITORY_URL: "http://console:3000/api/admin/export/streams-with-destinations"
+      INGEST_REPOSITORY_AUTH_TOKENS: "service-admin-account:${CONSOLE_PASSWORD}"
+      INGEST_CACHE_DIR: "/tmp/cache"
+      INGEST_REDIS_URL: "redis://default:${REDIS_PASSWORD:-default}@redis:6379"
+      INGEST_ROTOR_URL: "http://rotor:3401"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://ingest:3049/health" ]
+      interval: 2s
+      timeout: 10s
+      retries: 15
     depends_on:
-      redis:
-        condition: service_started
-      bulker:
+      console:
         condition: service_healthy
-      postgres:
+      rotor:
         condition: service_healthy
     ports:
-      - "${JITSU_PORT:-3000}:3000"
+      - "${JITSU_INGEST_PORT:-8080}:3049"
+
+  syncctl:
+    tty: true
+    image: jitsucom/syncctl:beta
+    platform: linux/amd64
+    restart: "on-failure"
+    environment:
+      TERM: "xterm-256color"
+      SYNCCTL_SYNCS_ENABLED: ${SYNCS_ENABLED:-false}
+      SYNCCTL_AUTH_TOKENS: ${SYNCCTL_PASSWORD}
+      SYNCCTL_DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@postgres:5432/postgres?search_path=newjitsu"
+      SYNCCTL_SIDECAR_DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@${EXTERNAL_DATABASE_HOST}:${PG_PORT:-5437}/postgres?search_path=newjitsu"
+      SYNCCTL_BULKER_URL: "http://${EXTERNAL_BULKER_HOST}:${BULKER_PORT:-3045}"
+      SYNCCTL_BULKER_AUTH_TOKEN: ${BULKER_PASSWORD}
+      SYNCCTL_BULKER_LOGS_CONNECTION_ID: task_log
+      SYNCCTL_KUBERNETES_CLIENT_CONFIG: "${SYNCCTL_KUBERNETES_CLIENT_CONFIG:-local}"
+      SYNCCTL_KUBERNETES_CONTEXT: "${SYNCCTL_KUBERNETES_CONTEXT}"
+    volumes:
+      - ./data/syncctl:/etc/syncctl
+    extra_hosts:
+      - "kubernetes:host-gateway"
+    depends_on:
+      bulker:
+        condition: service_healthy
diff --git a/libs/juava/__tests__/id.test.ts b/libs/juava/__tests__/id.test.ts
@@ -1,4 +1,4 @@
-import { checkHash, createHash, randomId } from "../src";
+import { randomId } from "../src";
 
 test("id test", () => {
   const id1 = randomId();
diff --git a/libs/juava/__tests__/security.test.ts b/libs/juava/__tests__/security.test.ts
@@ -1,11 +1,11 @@
-import { createHash, checkHash, createAuthorized } from "../src/security";
+import { createHash, checkToken, createAuthorized } from "../src/security";
 
 test("security", () => {
   const password = "secretPassword";
   const hashResult = createHash(password);
   console.log("Hash result = " + hashResult);
-  expect(checkHash(hashResult, password)).toBe(true);
-  expect(checkHash(hashResult.substring(2), password)).toBe(false);
+  expect(checkToken(hashResult, password)).toBe(true);
+  expect(checkToken(hashResult.substring(2), password)).toBe(false);
 });
 
 test("authorizer", () => {
diff --git a/libs/juava/src/security.ts b/libs/juava/src/security.ts
@@ -23,6 +23,13 @@ export function createHash(secret: string): string {
   return hashInternal(secret, randomSeed, globalSeed[0]);
 }
 
+export function checkToken(hashOrPlain: string, secret: string): boolean {
+  if (hashOrPlain.indexOf(".") === -1) {
+    return secret === hashOrPlain;
+  }
+  return checkHash(hashOrPlain, secret);
+}
+
 export function checkHash(hash: string, secret: string): boolean {
   const [randomSeed] = hash.split(".");
   return globalSeed.find(seed => hash === hashInternal(secret, randomSeed, seed)) !== undefined;
@@ -43,11 +50,7 @@ export function createAuthorized(tokens: string): Authorizer {
   const authorizers = tokens
     .split(",")
     .map(tok => tok.trim())
-    .map(hashOrPlain =>
-      hashOrPlain.indexOf(".") === -1
-        ? (secret: string) => secret === hashOrPlain
-        : (secret: string) => checkHash(hashOrPlain, secret)
-    );
+    .map(hashOrPlain => (secret: string) => checkToken(hashOrPlain, secret));
   return (secret: string) => authorizers.find(auth => auth(secret)) !== undefined;
 }
 
diff --git a/services/rotor/src/index.ts b/services/rotor/src/index.ts
@@ -42,7 +42,7 @@ async function main() {
   errorTypes.forEach(type => {
     process.once(type, err => {
       log.atError().withCause(err).log(`process.on ${type}`);
-      process.exit(1);
+      //  process.exit(1);
     });
   });
 
diff --git a/webapps/console/lib/api.ts b/webapps/console/lib/api.ts
@@ -1,6 +1,6 @@
 import { ZodType } from "zod";
 import { NextApiHandler, NextApiRequest, NextApiResponse } from "next";
-import { assertDefined, checkHash, getErrorMessage, requireDefined, tryJson } from "juava";
+import { assertDefined, checkToken, getErrorMessage, requireDefined, tryJson } from "juava";
 import { getServerSession, Session } from "next-auth";
 import { nextAuthConfig } from "./nextauth.config";
 import { SessionUser } from "./schema";
@@ -87,8 +87,8 @@ export function getAuthBearerToken(req: NextApiRequest): string | undefined {
 function findServiceAccount({ keyId, secret }): SessionUser | undefined {
   if (process.env.CONSOLE_AUTH_TOKENS) {
     const tokens = process.env.CONSOLE_AUTH_TOKENS.split(",");
-    for (const tokenHash of tokens) {
-      if (checkHash(tokenHash, secret)) {
+    for (const tokenHashOrPlain of tokens) {
+      if (checkToken(tokenHashOrPlain, secret)) {
         return {
           internalId: adminServiceAccountEmail,
           externalUsername: adminServiceAccountEmail,
@@ -120,7 +120,7 @@ export async function getUser(
       if (!token) {
         throw new ApiError(`Invalid API key id ${keyId}`, { keyId }, { status: 401 });
       }
-      if (!checkHash(token.hash, secret)) {
+      if (!checkToken(token.hash, secret)) {
         throw new ApiError(`Invalid API key secret for ${keyId}`, { keyId }, { status: 401 });
       }
       const user = requireDefined(
diff --git a/webapps/console/lib/nextauth.config.ts b/webapps/console/lib/nextauth.config.ts

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { checkHash, createHash, randomId } from "../src";`
	`1`	`+import { randomId } from "../src";`
`2`	`2`
`3`	`3`	`test("id test", () => {`
`4`	`4`	`const id1 = randomId();`