ci: update nightly workflow

anct-cartographie-nationale · Jan 8, 2025 · 9bd3575 · 9bd3575
1 parent 81df816
commit 9bd3575
Showing 1 changed file with 66 additions and 150 deletions.
diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml
@@ -6,164 +6,80 @@ on:
     - cron: '15 4 * * *' # every day at 04:15 AM
 
 jobs:
-  transform-and-publish:
-    name: Transform and publish to data.gouv
-    strategy:
-      matrix:
-        source:
-          [
-            aidants-connect,
-            aix-en-provence,
-            angers,
-            bus-france-services-charente,
-            charente-maritime,
-            conseiller-numerique,
-            coop-numerique,
-            corse,
-            dora,
-            epernay,
-            etapes-numerique,
-            fibre-64,
-            france-services,
-            francil-in,
-            gironde,
-            grand-paris-sud,
-            haute-vienne,
-            hinaura,
-            hub-bretagne,
-            hub-lo,
-            la-creuse,
-            le-havre,
-            les-landes,
-            loire-atlantique,
-            maine-et-loire,
-            mednum-bfc,
-            mulhouse,
-            nouvelle-caledonie,
-            vendee,
-            paca,
-            rhin-occ,
-            siilab
-          ]
-    runs-on: ubuntu-latest
-    environment: production
-    steps:
-      - name: Checkout mednum-cli repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: latest
-          cache: 'pnpm'
-      - name: Install dependencies
-        run: pnpm install
-      - name: 'Create env file'
-        run: |
-          touch .env
-          echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env
-          echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env
-          echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
-          echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
-          echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
-          echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env
-          echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env
-      - name: Transform
-        run: pnpm transformer.${{ matrix.source }}
-      - name: Publish
-        run: pnpm publier.${{ matrix.source }}
-      - name: Deduplicate
-        run: pnpm dedupliquer.${{ matrix.source }}
+  process-data:
+    name: Process data
+    uses: ./.github/workflows/process-data.reusable.yml
+    with:
+      sources: '[
+        "aidants-connect",
+        "aix-en-provence",
+        "angers",
+        "bus-france-services-charente",
+        "charente-maritime",
+        "conseiller-numerique",
+        "coop-numerique",
+        "corse",
+        "dora",
+        "epernay",
+        "etapes-numerique",
+        "fibre-64",
+        "france-services",
+        "francil-in",
+        "gironde",
+        "grand-paris-sud",
+        "haute-vienne",
+        "hinaura",
+        "hub-bretagne",
+        "hub-lo",
+        "la-creuse",
+        "le-havre",
+        "les-landes",
+        "loire-atlantique",
+        "maine-et-loire",
+        "mednum-bfc",
+        "mulhouse",
+        "nouvelle-caledonie",
+        "vendee",
+        "paca",
+        "rhin-occ",
+        "siilab"
+        ]'
+      environment: production
+    secrets:
+      DATA_INCLUSION_API_KEY: ${{ secrets.DATA_INCLUSION_API_KEY }}
+      DATA_GOUV_API_URL: ${{ secrets.DATA_GOUV_API_URL }}
+      DATA_GOUV_API_KEY: ${{ secrets.DATA_GOUV_API_KEY }}
+      DATA_GOUV_REFERENCE_ID: ${{ secrets.DATA_GOUV_REFERENCE_ID }}
+      DATA_GOUV_REFERENCE_TYPE: ${{ secrets.DATA_GOUV_REFERENCE_TYPE }}
 
-  deduplicate:
-    name: Merge all duplicates between sources
+  upload-to-database:
+    name: Upload data to database
     runs-on: ubuntu-latest
-    environment: production
-    needs:
-      - transform-and-publish
+    needs: process-data
     steps:
-      - name: Checkout mednum-cli repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
+      - name: Download deduplicated data
+        uses: actions/download-artifact@v4
         with:
-          node-version: latest
-          cache: 'pnpm'
-      - name: Install dependencies
-        run: pnpm install
-      - name: 'Create env file'
-        run: |
-          touch .env
-          echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env
-          echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env
-          echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
-          echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
-          echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
-          echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env
-      - name: Deduplicate
-        run: pnpm dedupliquer.all
-
-  extract-and-publish:
-    name: Extract and publish to data.gouv
-    strategy:
-      matrix:
-        source: [extract-eure-et-loir, extract-mednum-hub-antilles, extract-numi, extract-paca, extract-savoie, extract-vendee]
-    runs-on: ubuntu-latest
-    environment: production
-    needs:
-      - deduplicate
-    steps:
-      - name: Wait for deduplication end
-        run: sleep 300s
-        shell: bash
-      - name: Checkout mednum-cli repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9
-      - name: Set up Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: latest
-          cache: 'pnpm'
-      - name: Install dependencies
-        run: pnpm install
-      - name: 'Create env file'
-        run: |
-          touch .env
-          echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env
-          echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env
-          echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
-          echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
-          echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env
-      - name: Extract
-        run: pnpm extract.${{ matrix.source }}
-      - name: Publish
-        run: pnpm publier.${{ matrix.source }}
+          path: deduplicated
+          merge-multiple: true
+      - name: Publish to dynamoDB via S3
+        uses: kersvers/s3-sync-with-cloudfront-invalidation@v1.0.0
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_DYNAMODB_BUCKET }}
+          DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_DISTRIBUTION_ID }}
+          SOURCE_DIR: 'deduplicated/dynamo-import'
 
   publish-to-data-gouv:
     name: Get lieux from API and publish to data.gouv
     runs-on: ubuntu-latest
     environment: production
     needs:
-      - deduplicate
+      - upload-to-database
     steps:
-      - name: Wait for deduplication end
-        run: sleep 300s
+      - name: Sleep for 4 minutes
+        run: sleep 240s
         shell: bash
       - name: Checkout mednum-cli repository
         uses: actions/checkout@v4
@@ -197,10 +113,10 @@ jobs:
     name: Get lieux from API and publish to S3
     runs-on: ubuntu-latest
     needs:
-      - deduplicate
+      - upload-to-database
     steps:
-      - name: Sleep for 5 minutes
-        run: sleep 300s
+      - name: Sleep for 4 minutes
+        run: sleep 240s
         shell: bash
       - name: Prepare lieux inclusion numerique file
         run: |