Merge branch 'main' of github.com:dust-tt/dust into thib/homerevamp

.github/actions/slack-notify/action.yml

@@ -26,6 +26,10 @@ inputs:
   region:
     description: "Region being deployed to"
     required: false
+  is_revert:
+    description: "Whether reverting or deploying"
+    required: false
+    default: "false"
 
 outputs:
   thread_ts:
@@ -81,7 +85,7 @@ runs:
         payload: |
           channel: ${{ inputs.channel }}
           text: |
-            🚀 Starting deployment of ${{ inputs.component }} `${{ inputs.image_tag }}`
+            ${{ inputs.is_revert == 'true' && '⏮️' || '🚀' }} ${{ inputs.is_revert == 'true' && 'Starting revert of' || 'Starting deployment of' }} ${{ inputs.component }} `${{ inputs.image_tag }}`
             • Region: `${{ inputs.region }}`
             • Commit: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ inputs.image_tag }}>
             • Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View build>
@@ -97,6 +101,6 @@ runs:
           channel: ${{ inputs.channel }}
           thread_ts: "${{ inputs.thread_ts }}"
           text: |
-            ❌ Build pipeline failed
+            ❌ ${{ inputs.is_revert == 'true' && 'Revert' || 'Build' }} pipeline failed
             ${{ inputs.blocked == 'true' && format('Is blocked by channel topic: {0}', inputs.blocked) }}
             • Check logs: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View details>

.github/workflows/list-tags.yaml

@@ -3,11 +3,12 @@ name: List Image Tags
 on:
   workflow_dispatch:
     inputs:
-      regions:
-        description: "Comma-separated list of regions to deploy"
+      region:
+        description: "Region to list images from"
         type: choice
         options:
           - "us-central1"
+          - "europe-west1"
         default: "us-central1"
         required: true
       component:
@@ -25,27 +26,45 @@ on:
           - prodbox
           - viz
         required: true
-
-env:
-  GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }}
+    secrets:
+      GCLOUD_US_PROJECT_ID:
+        required: true
+      GCLOUD_EU_PROJECT_ID:
+        required: true
 
 jobs:
   list-tags:
+    permissions:
+      contents: read
+      id-token: write
+
     runs-on: ubuntu-latest
     steps:
-      - name: "Authenticate with Google Cloud"
-        uses: "google-github-actions/auth@v1"
+      - name: Set project ID
+        id: project
+        run: |
+          if [ "${{ inputs.region }}" = "us-central1" ]; then
+            echo "PROJECT_ID=${{ secrets.GCLOUD_US_PROJECT_ID }}" >> $GITHUB_OUTPUT
+          else
+            echo "PROJECT_ID=${{ secrets.GCLOUD_EU_PROJECT_ID }}" >> $GITHUB_OUTPUT
+          fi
+
+      - id: "auth"
+        name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@v2"
         with:
-          credentials_json: "${{ secrets.GCLOUD_SA_KEY }}"
+          create_credentials_file: true
+          workload_identity_provider: "projects/357744735673/locations/global/workloadIdentityPools/github-pool-apps/providers/github-provider-apps"
+          service_account: "github-build-invoker@${{ steps.project.outputs.PROJECT_ID }}.iam.gserviceaccount.com"
 
       - name: "Set up Cloud SDK"
-        uses: "google-github-actions/setup-gcloud@v1"
+        uses: "google-github-actions/setup-gcloud@v2"
 
       - name: List available tags
         run: |
-          echo "Latest tags for ${{ inputs.component }} in ${{ inputs.regions }}:"
+          echo "Latest tags for ${{ inputs.component }} in ${{ inputs.region }}:"
           echo "-------------------------------------------"
-          gcloud container images list-tags ${{ inputs.regions }}-docker.pkg.dev/${{ env.GCLOUD_PROJECT_ID }}/dust-images/${{ inputs.component }} \
+          gcloud container images list-tags "${{ inputs.region }}-docker.pkg.dev/${{ steps.project.outputs.PROJECT_ID }}/dust-images/${{ inputs.component }}" \
             --limit=10 \
             --sort-by=~timestamp \
             --format="table(timestamp.date('%Y-%m-%d %H:%M:%S'),tags[0])"

.github/workflows/revert.yml

@@ -0,0 +1,118 @@
+name: Revert Workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      region:
+        description: "Region to revert"
+        required: true
+        default: "us-central1"
+        type: choice
+        options:
+          - "us-central1"
+          - "europe-west1"
+      image_tag:
+        description: "The image tag/SHA to deploy"
+        type: string
+        required: true
+      component:
+        description: "Component to revert"
+        type: choice
+        options:
+          - alerting-temporal
+          - connectors
+          - core
+          - front
+          - front-edge
+          - metabase
+          - oauth
+          - prodbox
+          - viz
+        required: true
+    secrets:
+      SLACK_CHANNEL_ID:
+        required: true
+      SLACK_BOT_TOKEN:
+        required: true
+      INFRA_DISPATCH_APP_ID:
+        required: true
+      INFRA_DISPATCH_APP_PRIVATE_KEY:
+        required: true
+
+jobs:
+  check-branch:
+    runs-on: ubuntu-latest
+    if: ${{ !inputs.enforce_main || github.ref == 'refs/heads/main' }}
+    steps:
+      - name: Check branch condition
+        run: |
+          if [[ "${{ inputs.enforce_main }}" == "true" && "${{ github.ref }}" != "refs/heads/main" ]]; then
+            echo "Reverts are only allowed from the main branch"
+            exit 1
+          fi
+
+  notify-start:
+    runs-on: ubuntu-latest
+    outputs:
+      thread_ts: ${{ steps.build_message.outputs.thread_ts }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Notify Start
+        id: build_message
+        uses: ./.github/actions/slack-notify
+        with:
+          step: "start"
+          channel: ${{ secrets.SLACK_CHANNEL_ID }}
+          component: ${{ inputs.component }}
+          image_tag: ${{ inputs.image_tag }}
+          region: ${{ inputs.region }}
+          slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
+          is_revert: true
+
+  revert:
+    needs: [notify-start]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Generate token
+        id: generate-token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.INFRA_DISPATCH_APP_ID }}
+          private-key: ${{ secrets.INFRA_DISPATCH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: dust-infra
+
+      - name: Trigger dust-infra workflow
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ steps.generate-token.outputs.token }}
+          script: |
+            await github.rest.repos.createDispatchEvent({
+              owner: '${{ github.repository_owner }}',
+              repo: 'dust-infra',
+              event_type: 'trigger-component-revert',
+              client_payload: {
+                regions: '${{ inputs.region }}',
+                component: '${{ inputs.component }}',
+                image_tag: '${{ inputs.image_tag }}',
+                slack_thread_ts: "${{ needs.notify-start.outputs.thread_ts }}",
+                slack_channel: '${{ secrets.SLACK_CHANNEL_ID }}',
+                run_playwright: false,
+              }
+            })
+
+      - name: Notify Failure
+        if: failure()
+        uses: ./.github/actions/slack-notify
+        with:
+          step: "failure"
+          channel: ${{ secrets.SLACK_CHANNEL_ID }}
+          component: ${{ inputs.component }}
+          image_tag: ${{ inputs.image_tag }}
+          region: ${{ inputs.region }}
+          slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
+          thread_ts: "${{ needs.notify-start.outputs.thread_ts }}"
+          is_revert: true

connectors/migrations/db/migration_47.sql

@@ -0,0 +1,3 @@
+-- Migration created on Jan 21, 2025
+ALTER TABLE "public"."remote_databases" ADD COLUMN "permission" VARCHAR(255) NOT NULL DEFAULT 'selected';
+ALTER TABLE "public"."remote_schemas" ADD COLUMN "permission" VARCHAR(255) NOT NULL DEFAULT 'selected';

connectors/src/connectors/snowflake/index.ts

@@ -11,8 +11,10 @@ import type {
   RetrievePermissionsErrorCode,
   UpdateConnectorErrorCode,
 } from "@connectors/connectors/interface";
-import { ConnectorManagerError } from "@connectors/connectors/interface";
-import { BaseConnectorManager } from "@connectors/connectors/interface";
+import {
+  BaseConnectorManager,
+  ConnectorManagerError,
+} from "@connectors/connectors/interface";
 import {
   fetchAvailableChildrenInSnowflake,
   fetchReadNodes,
@@ -166,7 +168,6 @@ export class SnowflakeConnectorManager extends BaseConnectorManager<null> {
       },
       { where: { connectorId: c.id } }
     );
-    await launchSnowflakeSyncWorkflow(c.id);
     // We launch the workflow again so it syncs immediately.
     await launchSnowflakeSyncWorkflow(c.id);
 
@@ -340,9 +341,38 @@ export class SnowflakeConnectorManager extends BaseConnectorManager<null> {
   }: {
     permissions: Record<string, ConnectorPermission>;
   }): Promise<Result<void, Error>> {
+    const connectorAndCredentialsRes = await getConnectorAndCredentials({
+      connectorId: this.connectorId,
+      logger,
+    });
+    if (connectorAndCredentialsRes.isErr()) {
+      switch (connectorAndCredentialsRes.error.code) {
+        case "connector_not_found":
+          throw new Error("Snowflake connector not found");
+        case "invalid_credentials":
+          return new Err(
+            new ConnectorManagerError(
+              "EXTERNAL_OAUTH_TOKEN_ERROR",
+              "Snowflake authorization error, please re-authorize."
+            )
+          );
+        default:
+          assertNever(connectorAndCredentialsRes.error.code);
+      }
+    }
+
+    const { credentials } = connectorAndCredentialsRes.value;
+
+    if (!isSnowflakeCredentials(credentials)) {
+      throw new Error(
+        "Invalid credentials type - expected snowflake credentials"
+      );
+    }
+
     await saveNodesFromPermissions({
       connectorId: this.connectorId,
       permissions,
+      credentials,
       logger,
     });