From 17e759f3d4121a25a5ab26b8539ed285d68a583c Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 15 Jul 2024 11:26:40 +0200
Subject: [PATCH 01/10] ci: add run-very-expensive-tests checks

---
 .github/pull_request_template.md |  1 +
 .github/workflows/test.yml       | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 8dad1795843..0c1a58b8e08 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -26,3 +26,4 @@ Before you mark the PR ready for review, please make sure that:
   - [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
 - [ ] Tests exist for new functionality or change in behavior
 - [ ] CI is green
+- [ ] Add the label `run-very-expensive-tests` if the PR requires running very expensive tests
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b2a1551b70d..ef20b6b6a3a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,6 +7,8 @@ on:
       - master
       - release/*
   workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *' # Runs nightly at 0AM UTC
 
 defaults:
   run:
@@ -25,10 +27,24 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       groups: ${{ steps.test.outputs.groups }}
+      run_very_expensive_tests: ${{ steps.envs.outputs.run_very_expensive_tests }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
+      - id: envs
+        env:
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          HAS_RUN_VERY_EXPENSIVE_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'run-very-expensive-tests') }}
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" = "pull_request" && "${HAS_RUN_VERY_EXPENSIVE_LABEL}" = "true" ]]; then
+            echo "run_very_expensive_tests=1" >> $GITHUB_OUTPUT
+          elif [[ "${GITHUB_EVENT_NAME}" = "schedule" ]]; then
+            echo "run_very_expensive_tests=1" >> $GITHUB_OUTPUT
+          else
+            echo "run_very_expensive_tests=0" >> $GITHUB_OUTPUT
+          fi
+
       - id: test
         env:
           # Unit test groups other than unit-rest
@@ -242,6 +258,9 @@ jobs:
       fail-fast: false
       matrix:
         include: ${{ fromJson(needs.discover.outputs.groups) }}
+    env:
+      LOTUS_RUN_EXPENSIVE_TESTS: 1
+      LOTUS_RUN_VERY_EXPENSIVE_TESTS: ${{ needs.discover.outputs.run_very_expensive_tests }}
     steps:
       - uses: actions/checkout@v4
         with:

From 240fad31919af2eda4b45531f96d736d887ee2a0 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 15 Jul 2024 15:05:01 +0200
Subject: [PATCH 02/10] ci: update tag name

---
 .github/pull_request_template.md | 2 +-
 .github/workflows/test.yml       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 0c1a58b8e08..27963be4cd3 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -21,9 +21,9 @@ Before you mark the PR ready for review, please make sure that:
   - If the change does not require a CHANGELOG.md entry, do one of the following:
     - Add `[skip changelog]` to the PR title
     - Add the label `skip/changelog` to the PR
+- [ ] Add the label `need/very-expensive-tests` if the PR requires running very expensive tests
 - [ ] New features have usage guidelines and / or documentation updates in
   - [ ] [Lotus Documentation](https://lotus.filecoin.io)
   - [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)
 - [ ] Tests exist for new functionality or change in behavior
 - [ ] CI is green
-- [ ] Add the label `run-very-expensive-tests` if the PR requires running very expensive tests
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ef20b6b6a3a..0caec79eb2e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -35,7 +35,7 @@ jobs:
       - id: envs
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
-          HAS_RUN_VERY_EXPENSIVE_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'run-very-expensive-tests') }}
+          HAS_RUN_VERY_EXPENSIVE_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'need/very-expensive-tests') }}
         run: |
           if [[ "${GITHUB_EVENT_NAME}" = "pull_request" && "${HAS_RUN_VERY_EXPENSIVE_LABEL}" = "true" ]]; then
             echo "run_very_expensive_tests=1" >> $GITHUB_OUTPUT

From ee308226e4a744a05dba779d0fbc6b33a35d1316 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 15 Jul 2024 15:14:14 +0200
Subject: [PATCH 03/10] ci: labelling events

---
 .github/workflows/test.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0caec79eb2e..fd09dd0ae85 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,6 +2,13 @@ name: Test
 
 on:
   pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+      - labeled
+      - unlabeled
   push:
     branches:
       - master

From 919535fd6a1416cb78b36b0464741a322160de46 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 22 Jul 2024 11:06:47 +0200
Subject: [PATCH 04/10] ci: tweak niporep timeout

---
 .github/workflows/test.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fd09dd0ae85..9cc78c687a5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -54,6 +54,7 @@ jobs:
 
       - id: test
         env:
+          LOTUS_RUN_VERY_EXPENSIVE_TESTS: ${{ steps.envs.outputs.run_very_expensive_tests }}
           # Unit test groups other than unit-rest
           utests: |
             [
@@ -76,6 +77,11 @@ jobs:
                 "skip_conformance": "0"
               }
             ]
+          # Tests that are very expensive to run and need a higher timeout when activated
+          very_expensive_tests: |
+            [
+              "itest-niporep_manual"
+            ]
           # Mapping from test group names to custom runner labels
           # The jobs default to running on the default hosted runners (4 CPU, 16 RAM).
           # We use self-hosted xlarge (4 CPU, 8 RAM; and large - 2 CPU, 4 RAM) runners
@@ -190,6 +196,21 @@ jobs:
           # Apply the needs_parameters flag to the groups
           groups="$(jq -n --argjson g "$groups" --argjson p "$parameters" '$g | map(. + {"needs_parameters": ([.name] | inside($p)) })')"
 
+          # Update the timeout for very expensive tests
+          if [ "${LOTUS_RUN_VERY_EXPENSIVE_TESTS}" == "1" ]; then
+            groups="$(jq -n --argjson g "$groups" --argjson e "$very_expensive_tests" '$g | map(
+              if ([.name] | inside($e)) then
+                if .go_test_flags then
+                  .go_test_flags += " -timeout=60m"
+                else
+                  . + {"go_test_flags": "-timeout=60m"}
+                end
+              else
+                .
+              end
+            )')"
+          fi
+
           # Output the groups
           echo "groups=$groups"
           echo "groups=$(jq -nc --argjson g "$groups" '$g')" >> $GITHUB_OUTPUT

From 158bcce4bfd195b0bd335c1e24f2df9bd4164ae9 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 22 Jul 2024 15:35:16 +0200
Subject: [PATCH 05/10] ci: simplif timeout

---
 .github/workflows/test.yml | 28 ++++++----------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9cc78c687a5..af4c6e0a542 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -54,7 +54,6 @@ jobs:
 
       - id: test
         env:
-          LOTUS_RUN_VERY_EXPENSIVE_TESTS: ${{ steps.envs.outputs.run_very_expensive_tests }}
           # Unit test groups other than unit-rest
           utests: |
             [
@@ -77,11 +76,6 @@ jobs:
                 "skip_conformance": "0"
               }
             ]
-          # Tests that are very expensive to run and need a higher timeout when activated
-          very_expensive_tests: |
-            [
-              "itest-niporep_manual"
-            ]
           # Mapping from test group names to custom runner labels
           # The jobs default to running on the default hosted runners (4 CPU, 16 RAM).
           # We use self-hosted xlarge (4 CPU, 8 RAM; and large - 2 CPU, 4 RAM) runners
@@ -196,21 +190,6 @@ jobs:
           # Apply the needs_parameters flag to the groups
           groups="$(jq -n --argjson g "$groups" --argjson p "$parameters" '$g | map(. + {"needs_parameters": ([.name] | inside($p)) })')"
 
-          # Update the timeout for very expensive tests
-          if [ "${LOTUS_RUN_VERY_EXPENSIVE_TESTS}" == "1" ]; then
-            groups="$(jq -n --argjson g "$groups" --argjson e "$very_expensive_tests" '$g | map(
-              if ([.name] | inside($e)) then
-                if .go_test_flags then
-                  .go_test_flags += " -timeout=60m"
-                else
-                  . + {"go_test_flags": "-timeout=60m"}
-                end
-              else
-                .
-              end
-            )')"
-          fi
-
           # Output the groups
           echo "groups=$groups"
           echo "groups=$(jq -nc --argjson g "$groups" '$g')" >> $GITHUB_OUTPUT
@@ -325,13 +304,18 @@ jobs:
           TEST_RUSTPROOFS_LOGS: ${{ matrix.test_rustproofs_logs || '0' }}
           FORMAT: ${{ matrix.format || 'standard-verbose' }}
           PACKAGES: ${{ join(matrix.packages, ' ') }}
+          GO_TEST_FLAGS: ${{ matrix.go_test_flags || '' }}
         run: |
+          if [ "$LOTUS_RUN_VERY_EXPENSIVE_TESTS" == "1" ] && [[ ! "$GO_TEST_FLAGS" =~ "-timeout" ]]; then
+            GO_TEST_FLAGS="$GO_TEST_FLAGS -timeout 30m"
+          fi
+
           gotestsum \
             --format "$FORMAT" \
             --junitfile "$REPORTS_PATH/$NAME.xml" \
             --jsonfile "$REPORTS_PATH/$NAME.json" \
             --packages="$PACKAGES" \
-            -- ${{ matrix.go_test_flags || '' }}
+            -- ${GO_TEST_FLAGS}
       - if: success() || failure()
         uses: actions/upload-artifact@v4
         with:

From 8dea5b839e700955b61cb8f9c0bd285b44110655 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 22 Jul 2024 16:08:41 +0200
Subject: [PATCH 06/10] ci: add doc

---
 .github/workflows/test.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index af4c6e0a542..3debe7330c0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -43,6 +43,10 @@ jobs:
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
           HAS_RUN_VERY_EXPENSIVE_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'need/very-expensive-tests') }}
+        # set the run_very_expensive_tests flag based on a few criteras:
+        # - if we're in a PR with the label 'need/very-expensive-tests'
+        # - if we're in the nightly cron job (schedule)
+        # this flag is used later to setup the env variable LOTUS_RUN_VERY_EXPENSIVE_TESTS
         run: |
           if [[ "${GITHUB_EVENT_NAME}" = "pull_request" && "${HAS_RUN_VERY_EXPENSIVE_LABEL}" = "true" ]]; then
             echo "run_very_expensive_tests=1" >> $GITHUB_OUTPUT

From ad641c5599efd3f2aaed4ed7fe0c4f32fa0ffa54 Mon Sep 17 00:00:00 2001
From: Laurent Senta <laurent@singulargarden.com>
Date: Mon, 22 Jul 2024 17:14:16 +0200
Subject: [PATCH 07/10] ci: bump to 60 min

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3debe7330c0..ea026492dd5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -311,7 +311,7 @@ jobs:
           GO_TEST_FLAGS: ${{ matrix.go_test_flags || '' }}
         run: |
           if [ "$LOTUS_RUN_VERY_EXPENSIVE_TESTS" == "1" ] && [[ ! "$GO_TEST_FLAGS" =~ "-timeout" ]]; then
-            GO_TEST_FLAGS="$GO_TEST_FLAGS -timeout 30m"
+            GO_TEST_FLAGS="$GO_TEST_FLAGS -timeout 60m"
           fi
 
           gotestsum \

From ed6044592cbabd0c75878a514d2ac85f2eb423f5 Mon Sep 17 00:00:00 2001
From: Laurent Senta <Laurent.Senta@gmail.com>
Date: Tue, 23 Jul 2024 09:33:52 +0200
Subject: [PATCH 08/10] ci: refine .github/pull_request_template.md

Co-authored-by: Steve Loeppky <biglep@protocol.ai>
---
 .github/pull_request_template.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 27963be4cd3..4640b68b8c0 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -21,7 +21,9 @@ Before you mark the PR ready for review, please make sure that:
   - If the change does not require a CHANGELOG.md entry, do one of the following:
     - Add `[skip changelog]` to the PR title
     - Add the label `skip/changelog` to the PR
-- [ ] Add the label `need/very-expensive-tests` if the PR requires running very expensive tests
+- [ ] Run [very expensive tests](https://github.com/search?q=repo%3Afilecoin-project%2Flotus+VeryExpensive&type=code) if useful or skip
+   - If touching codepaths affecting "very expensive tests", add the label `need/very-expensive-tests`.
+   - Otherwise skip (do nothing)
 - [ ] New features have usage guidelines and / or documentation updates in
   - [ ] [Lotus Documentation](https://lotus.filecoin.io)
   - [ ] [Discussion Tutorials](https://github.com/filecoin-project/lotus/discussions/categories/tutorials)

From dd43d305a17e37f7afa1dac800756a3c2185c995 Mon Sep 17 00:00:00 2001
From: Laurent Senta <Laurent.Senta@gmail.com>
Date: Tue, 23 Jul 2024 14:39:57 +0200
Subject: [PATCH 09/10] ci: tests: create issue on very expensive test failure
 (#12277)

---
 .github/workflows/test.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ea026492dd5..708c49e4e8d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,6 +27,7 @@ concurrency:
 
 permissions:
   contents: read
+  issues: write
 
 jobs:
   discover:
@@ -328,3 +329,12 @@ jobs:
             ${{ steps.reports.outputs.path }}/${{ matrix.name }}.xml
             ${{ steps.reports.outputs.path }}/${{ matrix.name }}.json
         continue-on-error: true
+
+      - name: Create issue on failure
+        if: failure() && github.event_name == 'schedule'
+        uses: ipdxco/create-or-update-issue@0265e9148d6c26f6cdb0abb9c47df048a585c2fe # v1.0.0
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          title: "Fix Expensive Tests"
+          label: "expensive-test-failure"
+          body: "During a scheduled run, the test ${{ matrix.name }} failed."

From 0b62e28a71cd9338bf25fef916f530073f5001d3 Mon Sep 17 00:00:00 2001
From: Laurent Senta <Laurent.Senta@gmail.com>
Date: Wed, 24 Jul 2024 09:58:53 +0200
Subject: [PATCH 10/10] ci: tweak the create or update behavior

Co-authored-by: Steve Loeppky <biglep@protocol.ai>
---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 708c49e4e8d..694a9a501c7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -332,9 +332,9 @@ jobs:
 
       - name: Create issue on failure
         if: failure() && github.event_name == 'schedule'
-        uses: ipdxco/create-or-update-issue@0265e9148d6c26f6cdb0abb9c47df048a585c2fe # v1.0.0
+        uses: ipdxco/create-or-update-issue@1c3635e06dd2f09272e49c6f1ad4619ba949120b # v1.0.1
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           title: "Fix Expensive Tests"
-          label: "expensive-test-failure"
+          label: "area/expensive-test-failure"
           body: "During a scheduled run, the test ${{ matrix.name }} failed."