-
Notifications
You must be signed in to change notification settings - Fork 0
174 lines (169 loc) · 4.87 KB
/
process-data.reusable.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
---
name: Reusable Data transform, merge, deduplicate and publish
on:
workflow_call:
inputs:
sources:
description: List of sources to process
required: true
type: string
environment:
description: Environment to deploy to
required: true
type: string
max_transform:
description: Maximum number of lines to transform
required: false
type: number
secrets:
DATA_INCLUSION_API_KEY:
required: true
DATA_GOUV_API_URL:
required: true
DATA_GOUV_API_KEY:
required: true
DATA_GOUV_REFERENCE_ID:
required: true
DATA_GOUV_REFERENCE_TYPE:
required: true
jobs:
transform:
name: Transform
strategy:
matrix:
source: ${{ fromJSON(inputs.sources) }}
runs-on: ubuntu-latest
environment: ${{ inputs.environment }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
version: 9
run_install: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: 22
cache: 'pnpm'
- name: Install dependencies
run: pnpm install
- name: 'Create env file'
run: |
touch .env
echo DATA_INCLUSION_API_KEY="${{ secrets.DATA_INCLUSION_API_KEY }}" >> .env
- name: Transform
run: pnpm transformer.${{ matrix.source }} -f
- name: Deduplicate
run: pnpm dedupliquer.${{ matrix.source }}
- name: Upload transformed data
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.source }}
path: assets/output/${{ matrix.source }}
publish:
name: Publish to data.gouv
strategy:
matrix:
source: ${{ fromJSON(inputs.sources) }}
runs-on: ubuntu-latest
environment: ${{ inputs.environment }}
needs: transform
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
version: 9
run_install: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: 22
cache: 'pnpm'
- name: Install dependencies
run: pnpm install
- name: 'Create env file'
run: |
touch .env
echo DATA_GOUV_API_URL="${{ secrets.DATA_GOUV_API_URL }}" >> .env
echo DATA_GOUV_API_KEY="${{ secrets.DATA_GOUV_API_KEY }}" >> .env
echo DATA_GOUV_REFERENCE_ID="${{ secrets.DATA_GOUV_REFERENCE_ID }}" >> .env
echo DATA_GOUV_REFERENCE_TYPE="${{ secrets.DATA_GOUV_REFERENCE_TYPE }}" >> .env
- name: Download transformed data
uses: actions/download-artifact@v4
with:
name: ${{ matrix.source }}
path: assets/output/${{ matrix.source }}
- name: Rename files to publish
run: sed -i 's/-sans-doublons//g' ./assets/output/${{ matrix.source }}/publier.json
- name: Publish
run: pnpm publier.${{ matrix.source }}
merge:
name: Merge
runs-on: ubuntu-latest
needs: transform
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
version: 9
run_install: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: 22
cache: 'pnpm'
- name: Install dependencies
run: pnpm install
- name: Download all transformed data
uses: actions/download-artifact@v4
with:
path: to-merge
merge-multiple: true
- name: Remove useless files
run: |
rm to-merge/services-*
rm to-merge/structures-*
- name: Merge transformed data
run: pnpm fusionner
- name: Upload merged data
uses: actions/upload-artifact@v4
with:
name: merged
path: merged
deduplicate:
name: Deduplicate
runs-on: ubuntu-latest
needs: merge
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install pnpm
uses: pnpm/action-setup@v4
with:
version: 9
run_install: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: 22
cache: 'pnpm'
- name: Install dependencies
run: pnpm install
- name: Download merged data
uses: actions/download-artifact@v4
with:
name: merged
path: assets/to-deduplicate
- name: Deduplicate merged data
run: pnpm dedupliquer.merged-json
- name: Upload deduplicated data
uses: actions/upload-artifact@v4
with:
name: deduplicated
path: assets/deduplicated