Expose collection API (#173)

.github/workflows/deploy.yml

@@ -1,22 +1,39 @@
 name: Deploy
+
 on:
   push:
     branches: [ main ]
 
 jobs:
+  test:
+    uses: "./.github/workflows/test.yml"
+
   deploy:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - name: Get target server hostname
+        id: get_hostname
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq '.all.hosts.* | key' deployment/inventory.yml
+      - name: Get target server fingerprint
+        id: get_fingerprint
+        uses: mikefarah/yq@master
         with:
-          repository: ambanum/OpenTermsArchive
-      - name: Install SSH key
+          cmd: yq '.all.hosts.*.ed25519_fingerprint' deployment/inventory.yml
+      - name: Set up SSH
         uses: shimataro/ssh-key-action@v2
         with:
           key: ${{ secrets.SERVER_SSH_KEY }}
-          known_hosts: ${{ secrets.SERVER_FINGERPRINT }}
-      - run: echo "${{ secrets.ANSIBLE_VAULT_KEY }}" > vault.key
+          known_hosts: ${{ steps.get_hostname.outputs.result }} ssh-ed25519 ${{ steps.get_fingerprint.outputs.result }}
+      - run: echo "${{ secrets.ANSIBLE_VAULT_KEY }}" > deployment/vault.key
       - run: pip install --upgrade setuptools
-      - run: pip install 'ansible ~= 2.9'
-      - run: ansible-playbook ops/app.yml --inventory ops/inventories/production.yml --tags update-declarations --limit france_elections
+      - run: pip install 'ansible ~= 9.4.0'
+      - name: Install dependencies
+        working-directory: ./deployment
+        run: ansible-galaxy collection install -p ./ -r requirements.yml --force
+      - name: Deploy
+        working-directory: ./deployment
+        run: ansible-playbook ./ansible_collections/opentermsarchive/deployment/playbooks/deploy.yml

.github/workflows/test.yml

@@ -1,29 +1,47 @@
 name: Validate declarations
 
 on:
-  push:
   pull_request:
-    types: [ opened, reopened ]
+  workflow_call:
 
 jobs:
   validate_schema:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 20
       - run: npm install
       - run: npm run test:schema
 
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+      - run: npm install
+      - run: npm run lint
+
   validate_modified_declarations:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
-          fetch-depth: 0
-      - uses: actions/setup-node@v2
+          fetch-depth: 0 # fetch all history for all branches and tags
+      - uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 20
       - run: npm install
       - run: npm run test:modified
+      - name: Generate artifacts
+        if: ${{ failure() }}
+        run: npm run start:modified
+      - uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: snapshots_and_versions
+          path: ./data
+          if-no-files-found: error

.gitignore

@@ -1,4 +1,4 @@
 .DS_Store
 node_modules
-package-lock.json
 data
+vault.key

config/production.json

@@ -0,0 +1,48 @@
+{
+  "@opentermsarchive/engine": {
+    "recorder": {
+      "versions": {
+        "storage": {
+          "git": {
+            "snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/france-elections-snapshots/commit/%SNAPSHOT_ID",
+            "repository": "git@github.com:OpenTermsArchive/france-elections-versions.git",
+            "path": "./data/versions"
+          }
+        }
+      },
+      "snapshots": {
+        "storage": {
+          "git": {
+            "repository": "git@github.com:OpenTermsArchive/france-elections-snapshots.git",
+            "path": "./data/snapshots"
+          }
+        }
+      }
+    },
+    "fetcher": {
+      "language": "en"
+    },
+    "notifier": {
+      "sendInBlue": {
+        "updatesListId": 861,
+        "updateTemplateId": 56
+      }
+    },
+    "logger": {
+      "timestampPrefix": false
+    },
+    "tracker": {
+      "githubIssues": {
+        "repository": "OpenTermsArchive/france-elections-declarations"
+      }
+    },
+    "dataset": {
+      "title": "france-elections",
+      "versionsRepositoryURL": "https://github.com/OpenTermsArchive/france-elections-versions"
+    },
+    "collection-api": {
+      "port": 3000,
+      "basePath": "/collection-api"
+    }
+  }
+}

deployment/.env

@@ -0,0 +1,11 @@
+$ANSIBLE_VAULT;1.1;AES256
+66663831663730303932353632313863636434656336333733353233323430346337333161313037
+6165383434373636643634303965653835353233313562320a363566616665666362383930356662
+34663639626539666334626161306137346462656434333561356433393834623965633165636635
+6632316332313031360a366465633935323033663930303431363936656139643837386164646332
+32613236303930633732303239653138623266333937346634383230643161393861616664373038
+36666238646363616232643666343162376163623638613133616635623633613233643037393962
+36306230393563653165613364656636653132326237363434363663376263316266626132313761
+30643531616631333331623638343636613465666663623239646564366262333833663132616435
+34383964326638306461653662346630326134653638663036656462383636373264613235616261
+3130643938393738623465373434336331636161646662623164

deployment/ansible.cfg

@@ -0,0 +1,12 @@
+# ansible.cfg
+[defaults]
+roles_path = ./ansible_collections/opentermsarchive/deployment/roles
+inventory = inventory.yml
+
+# The two following lines allow to have human readable output
+# Use the YAML callback plugin.
+stdout_callback = yaml
+# Use the stdout_callback when running ad-hoc commands.
+bin_ansible_callbacks = true
+
+vault_password_file = vault.key

deployment/github-bot-private-key

@@ -0,0 +1,26 @@
+$ANSIBLE_VAULT;1.1;AES256
+32623837323336656466663634353134643866633362336335386664353931666531386161303337
+3839393334336233303832386235303330303862303930660a626362396135323962396661393737
+34323835656264353236666633313531653563336433666665303637643564346465646431373061
+6332353963356538380a366261353736356637663135316531343464336636643465393339336363
+33396264613838643036656137393239323464663036323330343963313730633638386365363534
+61616433313161653565376537366463366166663964313336623561353431313630303734356134
+36646561393134346534636230313261366161393632393865616636623638653163393739363636
+62643366613231613934333637313638366364666630626632636132323030326633623736633165
+39333062313436343362663865363037346230323938383364316433376530336337363838366332
+35326639633562613234656663313561303734386433373034333061653034346437656666303039
+31666630376266626534323236336635393461346132346231613531373962356536376437363330
+61346235373965333636373232393839316266633433323332306431386162356438323230346563
+30356231613064333233623439343263383432636366373563653361633639303338306135383462
+35346632393931366237613165653637396135666336643366346634386434656165356136353239
+36376561643263376461666130303262396536653861323737326135313333363738666230656661
+31333765636364323463643933666461366234313134306439303264353631373765623366663362
+63366531376437333330363263623630326232656230653565373339653232623336633430326334
+66346139633736343034636663383066386366316238343138386563326538316466653630636138
+63633063313433393130396631613739653532313132353038643963363662326664303835656330
+30326266393432336537306561323235353533393364336433373964326232313031313063666466
+62656135356535633633326462383764623334643836313062383261383766303430663734613332
+61383039633930613130636531313265336663323339323135373065643634393866376231356130
+30646665316663616331666236373339303934616333373631653037386134393133643261373936
+62616362386534643839313435363666623633353165376362373762316665643832396165313031
+35376131306461306663333964653662626266376661306565366463383431623265

deployment/inventory.yml

@@ -0,0 +1,6 @@
+all:
+  hosts:
+    51.89.225.198:
+      ansible_user: debian
+      ed25519_fingerprint: AAAAC3NzaC1lZDI1NTE5AAAAIKKjqn0w980q4yeqogV0p06MQavEw7fzPR6rplT3kiLj
+      ota_source_repository: https://github.com/OpenTermsArchive/france-elections-declarations.git

deployment/pm2.config.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  apps: [
+    {
+      name: 'ota-collection-api',
+      script: 'npm',
+      args: 'run start:collection-api',
+      min_uptime: '10s',
+      max_restarts: 10,
+      restart_delay: 1000,
+      exponential_backoff_restart_delay: true,
+      log_date_format: "YYYY-MM-DDTHH:mm:ssZ"
+    }
+  ],
+};

deployment/requirements.yml

@@ -0,0 +1,4 @@
+---
+collections:
+- name: opentermsarchive.deployment
+  version: 2.0.2