Merge pull request #230 from zivy/FLData #314
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: IBEX Knowledge-Base Data Validation and Update | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: # ignore auto-generated files, only run when source files change | |
| - docs/contrib.md | |
| - docs/fluorescent_probes.md | |
| - docs/index.md | |
| - docs/protocols.md | |
| - docs/publications.md | |
| - docs/reagent_resources.md | |
| - docs/the_who.md | |
| - docs/videos.md | |
| - docs/data_and_software.md | |
| pull_request: | |
| branches: | |
| - main | |
| # Enable manual running of workflow, so that if the validation or | |
| # markdown generation code changes we can force execution | |
| workflow_dispatch: | |
| jobs: | |
| preliminary_test: | |
| # run all pre-commit tests (see .pre-commit-config.yaml) | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Run pre-commit tests | |
| run: | | |
| pip install pre-commit | |
| pre-commit run --all-files | |
| pr_preliminary_test: | |
| # check that no auto-generated file was modified in a PR on main so just check we're dealing with a PR | |
| # (workflow "on" event settings are all on main branch) | |
| needs: preliminary_test | |
| if: github.event_name == 'pull_request' | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| autogenerated_docs: | |
| - docs/contrib.md | |
| - docs/fluorescent_probes.md | |
| - docs/index.md | |
| - docs/protocols.md | |
| - docs/publications.md | |
| - docs/reagent_resources.md | |
| - docs/the_who.md | |
| - docs/videos.md | |
| - docs/data_and_software.md | |
| - name: modified autogenerated docs # succeed if no auto-generated files modified, otherwise fail | |
| if: steps.filter.outputs.autogenerated_docs != 'false' | |
| run: | | |
| echo "The pull request includes changes to auto-generated files, this is not allowed. Please modify the relevant files in the docs_in and data directories." | |
| exit 1 | |
| validate_data: | |
| needs: preliminary_test | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies and validate data | |
| run: | # validate the various data files, update the wheel version here whenever the validation scripts are updated | |
| pip install --upgrade pip | |
| pip install https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base_utilities/releases/download/v0.8.2/ibex_imaging_knowledge_base_utilities-0.8.2-py3-none-any.whl | |
| validate_zenodo_json .zenodo.json | |
| validate_bibfile data/publications.bib | |
| validate_basic data/protocols.csv .github/data_validation_configs/protocols.json | |
| validate_basic data/vendor_urls.csv .github/data_validation_configs/vendors.json | |
| validate_basic data/datasets.csv .github/data_validation_configs/datasets.json | |
| validate_basic data/software.csv .github/data_validation_configs/software.json | |
| validate_basic data/fluorescent_probes.csv .github/data_validation_configs/fluorescent_probes.json | |
| validate_videos data/videos.csv .github/data_validation_configs/videos.json .zenodo.json | |
| validate_image_resources data/image_resources.csv .github/data_validation_configs/image_resources.json docs/supporting_material | |
| validate_reagent_resources data/reagent_resources.csv .github/data_validation_configs/reagent_resources.json .zenodo.json data/vendor_urls.csv docs/supporting_material | |
| data_2_md: | |
| # run the data to markdown updates only when pushing to the main branch or forced execution via workflow_dispatch | |
| needs: validate_data | |
| if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'}} | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| brew update | |
| brew install pandoc | |
| pip install --upgrade pip | |
| pip install https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base_utilities/releases/download/v0.7.0/ibex_imaging_knowledge_base_utilities-0.7.0-py3-none-any.whl | |
| - name: Fine tune data to markdown conversion | |
| id: ft | |
| shell: bash | |
| run: | | |
| convert_bib=0 | |
| convert_reagents=0 | |
| update_index=0 | |
| convert_fluorescent=0 | |
| convert_the_who=0 | |
| update_how_to_contrib=0 | |
| convert_protocols=0 | |
| convert_videos=0 | |
| convert_data_software=0 | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| convert_bib=1 | |
| convert_reagents=1 | |
| update_index=1 | |
| convert_fluorescent=1 | |
| convert_the_who=1 | |
| update_how_to_contrib=1 | |
| convert_protocols=1 | |
| convert_videos=1 | |
| convert_data_software=1 | |
| else | |
| if ! git diff --quiet HEAD\^ HEAD -- data/publications.bib data/ibex.csl; then | |
| convert_bib=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/reagent_resources.md.in data/reagent_resources.csv data/vendor_urls.csv; then | |
| convert_reagents=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/index.md.in data/reagent_resources.csv; then | |
| update_index=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/fluorescent_probes.md.in data/fluorescent_probes.csv; then | |
| convert_fluorescent=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/the_who.md.in .zenodo.json; then | |
| convert_the_who=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/contrib.md.in data/reagent_data_dict.csv data/reagent_glossary.csv; then | |
| update_how_to_contrib=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/protocols.md.in data/protocols.csv; then | |
| convert_protocols=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/videos.md.in data/videos.csv; then | |
| convert_videos=1 | |
| fi | |
| if ! git diff --quiet HEAD\^ HEAD -- docs_in/data_and_software.md.in data/datasets.csv data/software.csv; then | |
| convert_data_software=1 | |
| fi | |
| fi | |
| echo "convert_bib=$convert_bib" >> $GITHUB_OUTPUT | |
| echo "convert_reagents=$convert_reagents" >> $GITHUB_OUTPUT | |
| echo "update_index=$update_index" >> $GITHUB_OUTPUT | |
| echo "convert_fluorescent=$convert_fluorescent" >> $GITHUB_OUTPUT | |
| echo "convert_the_who=$convert_the_who" >> $GITHUB_OUTPUT | |
| echo "update_how_to_contrib=$update_how_to_contrib" >> $GITHUB_OUTPUT | |
| echo "convert_protocols=$convert_protocols" >> $GITHUB_OUTPUT | |
| echo "convert_videos=$convert_videos" >> $GITHUB_OUTPUT | |
| echo "convert_data_software=$convert_data_software" >> $GITHUB_OUTPUT | |
| - name: Convert bibliography to markdown | |
| if: ${{ steps.ft.outputs.convert_bib == '1' }} | |
| run: | | |
| pandoc --version | |
| bib2md data/publications.bib data/ibex.csl docs/publications.md | |
| - name: Convert reagent resources csv to markdown | |
| if: ${{ steps.ft.outputs.convert_reagents == '1' }} | |
| run: | | |
| reagent_resources_csv_2_md_url docs_in/reagent_resources.md.in data/reagent_resources.csv data/vendor_urls.csv docs/supporting_material | |
| - name: Update the index.md file statistics | |
| if: ${{ steps.ft.outputs.update_index == '1' }} | |
| run: | | |
| update_index_md_stats docs_in/index.md.in data/reagent_resources.csv docs/ | |
| - name: Convert fluorescent probes csv to markdown | |
| if: ${{ steps.ft.outputs.convert_fluorescent == '1' }} | |
| run: | | |
| fluorescent_probes_csv_2_md docs_in/fluorescent_probes.md.in data/fluorescent_probes.csv docs/ | |
| - name: Convert Zenodo data to contributors list markdown | |
| if: ${{ steps.ft.outputs.convert_the_who == '1' }} | |
| run: | | |
| zenodo_json_2_thewho_md docs_in/the_who.md.in .zenodo.json docs/ | |
| - name: Update how to contribute markdown | |
| if: ${{ steps.ft.outputs.update_how_to_contrib == '1' }} | |
| run: | | |
| data_dict_glossary_2_contrib_md docs_in/contrib.md.in data/reagent_data_dict.csv data/reagent_glossary.csv docs/ | |
| - name: Convert protocols csv to markdown | |
| if: ${{ steps.ft.outputs.convert_protocols == '1' }} | |
| run: | | |
| protocols_csv_2_md docs_in/protocols.md.in data/protocols.csv docs/ | |
| - name: Convert videos csv to markdown | |
| if: ${{ steps.ft.outputs.convert_videos == '1' }} | |
| run: | | |
| videos_csv_2_md docs_in/videos.md.in data/videos.csv docs/ | |
| - name: Convert datasets and software csv files to markdown | |
| if: ${{ steps.ft.outputs.convert_data_software == '1' }} | |
| run: | | |
| data_software_csv_2_md docs_in/data_and_software.md.in data/datasets.csv data/software.csv docs/ | |
| - name: Commit and push | |
| shell: bash | |
| run: | | |
| if ! git diff --quiet; then | |
| git config --local user.email "$(git log --format='%ae' HEAD^!)" | |
| git config --local user.name "$(git log --format='%an' HEAD^!)" | |
| git add docs/*.md | |
| git commit -m "Adding converted markdowns." | |
| git push | |
| fi |