Upload results to results branch

.github/workflows/fetch_all_tools.yaml

@@ -16,42 +16,30 @@ permissions:
   contents: write
 
 jobs:
-  fetch-available-servers:
-    runs-on: ubuntu-20.04
-    name: Fetch list of all available servers
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-      - name: Install requirement
-        run: python -m pip install -r requirements.txt
-      - name: Run script 
-        run: | 
-          python bin/get_public_galaxy_servers.py -o data/available_public_servers.csv
-      - name: Commit servers 
-        # add or commit any changes in results if there was a change, merge with main, and push as bot
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git pull --no-rebase -s recursive -X ours
-          git add data
-          git status
-          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step get available servers") 
-          git push
 
   fetch-all-tools-stepwise:
     runs-on: ubuntu-20.04
     environment: fetch-tools
     name: Fetch all tool stepwise
     strategy:
+      max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge)
       matrix:
-        python-version: [3.8]
+        python-version: [3.11]
         subset:
          - repositories01.list
          - repositories02.list
          - repositories03.list
          - repositories04.list
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+      - name: Checkout results
+        uses: actions/checkout@v4
+        with:
+          ref: results
+          path: results
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -63,43 +51,50 @@ jobs:
         env: 
           GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
       - name: Commit all tools 
-        # add or commit any changes in results if there was a change, merge with main and push as bot
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git pull --no-rebase -s recursive -X ours
-          git add results
-          git status
-          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
-          git push
+        uses: s0/git-publish-subdir-action@develop
+        env:
+          BRANCH: results
+          FOLDER: results
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: self
+          SKIP_EMPTY_COMMITS: true
+          MESSAGE: "Update results step: fetch all tool stepwise - commmit {sha}"
 
   fetch-all-tools-merge:
     runs-on: ubuntu-20.04
     needs: fetch-all-tools-stepwise
     name: Fetch all tools merge
     steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
-      - uses: actions/setup-python@v5
-      - name: Install requirement
-        run: |
-          python -m pip install -r requirements.txt
-          sudo apt-get install jq
-      - name: Merge all tools
-        run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
-          awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
-          jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json 
-      - name: Wordcloud and interactive table
-        run: |
-          bash bin/format_tools.sh
-      - name: Commit all tools
-        # add or commit any changes in results if there was a change, merge with main and push as bot
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git pull --no-rebase -s recursive -X ours
-          git add results
-          git status
-          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step merge") 
-          git push
+    - name: Checkout main
+      uses: actions/checkout@v4
+      with:
+        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+    - name: Checkout results
+      uses: actions/checkout@v4
+      with:
+        ref: results
+        path: results
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Install requirement
+      run: |
+        python -m pip install -r requirements.txt
+        sudo apt-get install jq
+    - name: Merge all tools
+      run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
+        awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
+        jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json 
+    - name: Wordcloud and interactive table
+      run: |
+        bash bin/format_tools.sh
+    - name: Commit all tools
+      # add or commit any changes in results if there was a change, merge with main and push as bot
+      uses: s0/git-publish-subdir-action@develop
+      env:
+        BRANCH: results
+        FOLDER: results
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        REPO: self
+        SKIP_EMPTY_COMMITS: true
+        MESSAGE: "Update results step: fetch all tools merge - commmit {sha}"

.github/workflows/fetch_all_tutorials.yaml

@@ -21,7 +21,15 @@ jobs:
     environment: fetch-tutorials
     name: Fetch all tutorials
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+      - name: Checkout results
+        uses: actions/checkout@v4
+        with:
+          ref: results
+          path: results
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
@@ -33,12 +41,11 @@ jobs:
         env: 
           PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
       - name: Commit all tools 
-        # add or commit any changes in results if there was a change, merge with main and push as bot
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git pull --no-rebase -s recursive -X ours
-          git add results
-          git status
-          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials bot  - step fetch") 
-          git push
+        uses: s0/git-publish-subdir-action@develop
+        env:
+          BRANCH: results
+          FOLDER: results
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: self
+          SKIP_EMPTY_COMMITS: true
+          MESSAGE: "Update results step: fetch all tutorials - commmit {sha}"

.github/workflows/filter_communities.yaml

@@ -26,73 +26,99 @@ permissions:
 
 jobs:
   filter-all-tutorials:
+    name: Filter tutorials
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout main
+      uses: actions/checkout@v4
+      with:
+        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+    - name: Checkout results
+      uses: actions/checkout@v4
+      with:
+        ref: results
+        path: results
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8' 
+        python-version: '3.11' 
     - name: Install requirement
       run: python -m pip install -r requirements.txt
     - name: Run script
       run: | 
         bash bin/get_community_tutorials.sh
     - name: Commit results
       # commit the new filtered data, only if stuff was changed
-      run: |
-        git config user.name github-actions
-        git config user.email github-actions@github.com
-        git pull --no-rebase -s recursive -X ours
-        git add results
-        git status
-        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials / tools bot  - step tutorial filter") 
-        git push
+      uses: s0/git-publish-subdir-action@develop
+      env:
+        BRANCH: results
+        FOLDER: results
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        REPO: self
+        SKIP_EMPTY_COMMITS: true
+        MESSAGE: "Update results step: filter tutorials - commmit {sha}"
 
   update-tools-to-keep-exclude:
+    name: Update tool to keep and exclude
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout main
+      uses: actions/checkout@v4
+      with:
+        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+    - name: Checkout results
+      uses: actions/checkout@v4
+      with:
+        ref: results
+        path: results
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8' 
+        python-version: '3.11' 
     - name: Install requirement
       run: python -m pip install -r requirements.txt
     - name: Run script
       run: | 
         bash bin/update_tools_to_keep_exclude.sh
     - name: Commit results
       # commit the new filtered data, only if stuff was changed
-      run: |
-        git config user.name github-actions
-        git config user.email github-actions@github.com
-        git pull --no-rebase -s recursive -X ours
-        git add results
-        git status
-        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials / tools bot  - step exluded/kept tool list update") 
-        git push
+      uses: s0/git-publish-subdir-action@develop
+      env:
+        BRANCH: results
+        FOLDER: results
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        REPO: self
+        SKIP_EMPTY_COMMITS: true
+        MESSAGE: "Update results step: update tool to keep and exclude - commmit {sha}"
 
   filter-all-tools:
+    name: Filter all tools
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout main
+      uses: actions/checkout@v4
+      with:
+        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+    - name: Checkout results
+      uses: actions/checkout@v4
+      with:
+        ref: results
+        path: results
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8' 
+        python-version: '3.11' 
     - name: Install requirement
       run: python -m pip install -r requirements.txt
     - name: Run script
       run: | 
         bash bin/get_community_tools.sh
     - name: Commit results
       # commit the new filtered data, only if stuff was changed
-      run: |
-        git config user.name github-actions
-        git config user.email github-actions@github.com
-        git pull --no-rebase -s recursive -X ours
-        git add results
-        git status
-        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step tool filter") 
-        git push
-
+      uses: s0/git-publish-subdir-action@develop
+      env:
+        BRANCH: results
+        FOLDER: results
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        REPO: self
+        SKIP_EMPTY_COMMITS: true
+        MESSAGE: "Update results step: filter all tools - commmit {sha}"
 
 

.github/workflows/run_tests.yaml

@@ -1,12 +1,20 @@
 name: Run tests
-on: [push, pull_request]
+on: [push, pull_request_target]
 
 jobs:
   test-tools:
     runs-on: ubuntu-20.04
     environment: fetch-tools
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+      - name: Checkout results
+        uses: actions/checkout@v4
+        with:
+          ref: results
+          path: results
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
@@ -26,7 +34,15 @@ jobs:
   test-tutorials:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+      - name: Checkout results
+        uses: actions/checkout@v4
+        with:
+          ref: results
+          path: results
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'

.github/workflows/static.yml

@@ -35,6 +35,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with: 
+          ref: results
       - name: Setup Pages
         uses: actions/configure-pages@v4
       - name: Upload artifact