diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index c1c32e60..57c8b7bf 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -6,164 +6,80 @@ on: - cron: '15 4 * * *' # every day at 04:15 AM jobs: - transform-and-publish: - name: Transform and publish to data.gouv - strategy: - matrix: - source: - [ - aidants-connect, - aix-en-provence, - angers, - bus-france-services-charente, - charente-maritime, - conseiller-numerique, - coop-numerique, - corse, - dora, - epernay, - etapes-numerique, - fibre-64, - france-services, - francil-in, - gironde, - grand-paris-sud, - haute-vienne, - hinaura, - hub-bretagne, - hub-lo, - la-creuse, - le-havre, - les-landes, - loire-atlantique, - maine-et-loire, - mednum-bfc, - mulhouse, - nouvelle-caledonie, - vendee, - paca, - rhin-occ, - siilab - ] - runs-on: ubuntu-latest - environment: production - steps: - - name: Checkout mednum-cli repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 9 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: latest - cache: 'pnpm' - - name: Install dependencies - run: pnpm install - - name: 'Create env file' - run: | - touch .env - echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env - echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env - echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env - echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env - echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env - - name: Transform - run: pnpm transformer.${{ matrix.source }} - - name: Publish - run: pnpm publier.${{ matrix.source }} - - name: Deduplicate - run: pnpm dedupliquer.${{ matrix.source }} + process-data: + name: Process data + uses: ./.github/workflows/process-data.reusable.yml + with: + sources: '[ + "aidants-connect", + "aix-en-provence", + "angers", + "bus-france-services-charente", + "charente-maritime", + "conseiller-numerique", + "coop-numerique", + "corse", + "dora", + "epernay", + "etapes-numerique", + "fibre-64", + "france-services", + "francil-in", + "gironde", + "grand-paris-sud", + "haute-vienne", + "hinaura", + "hub-bretagne", + "hub-lo", + "la-creuse", + "le-havre", + "les-landes", + "loire-atlantique", + "maine-et-loire", + "mednum-bfc", + "mulhouse", + "nouvelle-caledonie", + "vendee", + "paca", + "rhin-occ", + "siilab" + ]' + environment: production + secrets: + DATA_INCLUSION_API_KEY: ${{ secrets.DATA_INCLUSION_API_KEY }} + DATA_GOUV_API_URL: ${{ secrets.DATA_GOUV_API_URL }} + DATA_GOUV_API_KEY: ${{ secrets.DATA_GOUV_API_KEY }} + DATA_GOUV_REFERENCE_ID: ${{ secrets.DATA_GOUV_REFERENCE_ID }} + DATA_GOUV_REFERENCE_TYPE: ${{ secrets.DATA_GOUV_REFERENCE_TYPE }} - deduplicate: - name: Merge all duplicates between sources + upload-to-database: + name: Upload data to database runs-on: ubuntu-latest - environment: production - needs: - - transform-and-publish + needs: process-data steps: - - name: Checkout mednum-cli repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 9 - - name: Set up Node.js - uses: actions/setup-node@v4 + - name: Download deduplicated data + uses: actions/download-artifact@v4 with: - node-version: latest - cache: 'pnpm' - - name: Install dependencies - run: pnpm install - - name: 'Create env file' - run: | - touch .env - echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env - echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env - echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env - echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_KEY="${{ secrets.CARTOGRAPHIE_NATIONALE_API_KEY }}" >> .env - - name: Deduplicate - run: pnpm dedupliquer.all - - extract-and-publish: - name: Extract and publish to data.gouv - strategy: - matrix: - source: [extract-eure-et-loir, extract-mednum-hub-antilles, extract-numi, extract-paca, extract-savoie, extract-vendee] - runs-on: ubuntu-latest - environment: production - needs: - - deduplicate - steps: - - name: Wait for deduplication end - run: sleep 300s - shell: bash - - name: Checkout mednum-cli repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 9 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: latest - cache: 'pnpm' - - name: Install dependencies - run: pnpm install - - name: 'Create env file' - run: | - touch .env - echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env - echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env - echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env - echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env - echo CARTOGRAPHIE_NATIONALE_API_URL="${{ secrets.CARTOGRAPHIE_NATIONALE_API_URL }}" >> .env - - name: Extract - run: pnpm extract.${{ matrix.source }} - - name: Publish - run: pnpm publier.${{ matrix.source }} + path: deduplicated + merge-multiple: true + - name: Publish to dynamoDB via S3 + uses: kersvers/s3-sync-with-cloudfront-invalidation@v1.0.0 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_DYNAMODB_BUCKET }} + DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_DISTRIBUTION_ID }} + SOURCE_DIR: 'deduplicated/dynamo-import' publish-to-data-gouv: name: Get lieux from API and publish to data.gouv runs-on: ubuntu-latest environment: production needs: - - deduplicate + - upload-to-database steps: - - name: Wait for deduplication end - run: sleep 300s + - name: Sleep for 4 minutes + run: sleep 240s shell: bash - name: Checkout mednum-cli repository uses: actions/checkout@v4 @@ -197,10 +113,10 @@ jobs: name: Get lieux from API and publish to S3 runs-on: ubuntu-latest needs: - - deduplicate + - upload-to-database steps: - - name: Sleep for 5 minutes - run: sleep 300s + - name: Sleep for 4 minutes + run: sleep 240s shell: bash - name: Prepare lieux inclusion numerique file run: |