-
Notifications
You must be signed in to change notification settings - Fork 7
198 lines (195 loc) · 8.51 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
name: IBEX Knowledge-Base Data Validation and Update
on:
push:
branches:
- main
paths-ignore: # ignore auto-generated files, only run when source files change
- docs/contrib.md
- docs/fluorescent_probes.md
- docs/index.md
- docs/protocols.md
- docs/publications.md
- docs/reagent_resources.md
- docs/the_who.md
- docs/videos.md
- docs/data_and_software.md
pull_request:
branches:
- main
paths-ignore: # ignore auto-generated files, only run when source files change
- docs/contrib.md
- docs/fluorescent_probes.md
- docs/index.md
- docs/protocols.md
- docs/publications.md
- docs/reagent_resources.md
- docs/the_who.md
- docs/videos.md
- docs/data_and_software.md
# Enable manual running of workflow, so that if the validation or
# markdown generation code changes we can force execution
workflow_dispatch:
jobs:
preliminary_test: # run all pre-commit tests (see .pre-commit-config.yaml)
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Run pre-commit tests
run: |
pip install pre-commit
pre-commit run --all-files
validate_data:
needs: preliminary_test
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install dependencies and validate data
run: | # validate the various data files, update the wheel version here whenever the validation scripts are updated
pip install --upgrade pip
pip install https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base_utilities/releases/download/v0.8.1/ibex_imaging_knowledge_base_utilities-0.8.1-py3-none-any.whl
validate_zenodo_json .zenodo.json
validate_bibfile data/publications.bib
validate_basic data/protocols.csv .github/data_validation_configs/protocols.json
validate_basic data/vendor_urls.csv .github/data_validation_configs/vendors.json
validate_basic data/datasets.csv .github/data_validation_configs/datasets.json
validate_basic data/software.csv .github/data_validation_configs/software.json
validate_basic data/fluorescent_probes.csv .github/data_validation_configs/fluorescent_probes.json
validate_videos data/videos.csv .github/data_validation_configs/videos.json .zenodo.json
validate_image_resources data/image_resources.csv .github/data_validation_configs/image_resources.json docs/supporting_material
validate_reagent_resources data/reagent_resources.csv .github/data_validation_configs/reagent_resources.json .zenodo.json docs/supporting_material
data_2_md: # run the data to markdown updates only when pushing to the main branch or forced execution via workflow_dispatch
needs: validate_data
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'}}
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install dependencies
run: |
brew update
brew install pandoc
pip install --upgrade pip
pip install https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base_utilities/releases/download/v0.7.0/ibex_imaging_knowledge_base_utilities-0.7.0-py3-none-any.whl
- name: Fine tune data to markdown conversion
id: ft
shell: bash
run: |
convert_bib=0
convert_reagents=0
update_index=0
convert_fluorescent=0
convert_the_who=0
update_how_to_contrib=0
convert_protocols=0
convert_videos=0
convert_data_software=0
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
convert_bib=1
convert_reagents=1
update_index=1
convert_fluorescent=1
convert_the_who=1
update_how_to_contrib=1
convert_protocols=1
convert_videos=1
convert_data_software=1
else
if ! git diff --quiet HEAD\^ HEAD -- data/publications.bib data/ibex.csl; then
convert_bib=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/reagent_resources.md.in data/reagent_resources.csv data/vendor_urls.csv; then
convert_reagents=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/index.md.in data/reagent_resources.csv; then
update_index=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/fluorescent_probes.md.in data/fluorescent_probes.csv; then
convert_fluorescent=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/the_who.md.in .zenodo.json; then
convert_the_who=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/contrib.md.in data/reagent_data_dict.csv data/reagent_glossary.csv; then
update_how_to_contrib=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/protocols.md.in data/protocols.csv; then
convert_protocols=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/videos.md.in data/videos.csv; then
convert_videos=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/data_and_software.md.in data/datasets.csv data/software.csv; then
convert_data_software=1
fi
fi
echo "convert_bib=$convert_bib" >> $GITHUB_OUTPUT
echo "convert_reagents=$convert_reagents" >> $GITHUB_OUTPUT
echo "update_index=$update_index" >> $GITHUB_OUTPUT
echo "convert_fluorescent=$convert_fluorescent" >> $GITHUB_OUTPUT
echo "convert_the_who=$convert_the_who" >> $GITHUB_OUTPUT
echo "update_how_to_contrib=$update_how_to_contrib" >> $GITHUB_OUTPUT
echo "convert_protocols=$convert_protocols" >> $GITHUB_OUTPUT
echo "convert_videos=$convert_videos" >> $GITHUB_OUTPUT
echo "convert_data_software=$convert_data_software" >> $GITHUB_OUTPUT
- name: Convert bibliography to markdown
if: ${{ steps.ft.outputs.convert_bib == '1' }}
run: |
pandoc --version
bib2md data/publications.bib data/ibex.csl docs/publications.md
- name: Convert reagent resources csv to markdown
if: ${{ steps.ft.outputs.convert_reagents == '1' }}
run: |
reagent_resources_csv_2_md_url docs_in/reagent_resources.md.in data/reagent_resources.csv data/vendor_urls.csv docs/supporting_material
- name: Update the index.md file statistics
if: ${{ steps.ft.outputs.update_index == '1' }}
run: |
update_index_md_stats docs_in/index.md.in data/reagent_resources.csv docs/
- name: Convert fluorescent probes csv to markdown
if: ${{ steps.ft.outputs.convert_fluorescent == '1' }}
run: |
fluorescent_probes_csv_2_md docs_in/fluorescent_probes.md.in data/fluorescent_probes.csv docs/
- name: Convert Zenodo data to contributors list markdown
if: ${{ steps.ft.outputs.convert_the_who == '1' }}
run: |
zenodo_json_2_thewho_md docs_in/the_who.md.in .zenodo.json docs/
- name: Update how to contribute markdown
if: ${{ steps.ft.outputs.update_how_to_contrib == '1' }}
run: |
data_dict_glossary_2_contrib_md docs_in/contrib.md.in data/reagent_data_dict.csv data/reagent_glossary.csv docs/
- name: Convert protocols csv to markdown
if: ${{ steps.ft.outputs.convert_protocols == '1' }}
run: |
protocols_csv_2_md docs_in/protocols.md.in data/protocols.csv docs/
- name: Convert videos csv to markdown
if: ${{ steps.ft.outputs.convert_videos == '1' }}
run: |
videos_csv_2_md docs_in/videos.md.in data/videos.csv docs/
- name: Convert datasets and software csv files to markdown
if: ${{ steps.ft.outputs.convert_data_software == '1' }}
run: |
data_software_csv_2_md docs_in/data_and_software.md.in data/datasets.csv data/software.csv docs/
- name: Commit and push
shell: bash
run: |
if ! git diff --quiet; then
git config --local user.email "$(git log --format='%ae' HEAD^!)"
git config --local user.name "$(git log --format='%an' HEAD^!)"
git add docs/*.md
git commit -m "Adding converted markdowns."
git push
fi