more sample_data updates

mozilla · Mar 1, 2025 · 1e44e66 · 1e44e66
1 parent 81e7be4
commit 1e44e66
Show file tree

Hide file tree

Showing 31 changed files with 3,812 additions and 998 deletions.
diff --git a/docs/testing.md b/docs/testing.md
@@ -96,4 +96,23 @@ Just the `test_ingest_pending_pulse_job` within the `/etl` tests
 docker-compose run backend pytest tests/ -k test_ingest_pending_pulse_job
 ```
 
+### Updating backend python test data
+
+There are many parts of the backend data, this section will continue to be updated as we document this process.
+
+For the `sample_data/` there is `tests/sample_data/transform.py`:
+
+- `push_data.json`: list of commits
+- `job_data.txt`: list of job data as returned from the TH jobs api
+- `pulse_consumer/job_data.json`: specific data that pulse would have for related jobs and pushes
+- `pulse_consumer/transformed_job_data.json`: what we transform the pulse data to
+
+The will update the data used for `etl/` using recent live data from autoland.
+
+There are a lot of taskid, revisions, and expected fields to update in tests.  Future work could be done to:
+
+- create a revision list and reference it instead of raw revisions
+- create an input file as a start date, end date, and use that instead of hard coded dates in many tests
+- ensure variety of platforms, builds, tests, pass/fail, etc. are included
+
 [eslint]: https://eslint.org
diff --git a/schemas/pulse-job.yml b/schemas/pulse-job.yml
@@ -146,6 +146,7 @@ properties:
     description: |
       fail: A failure
       exception: An infrastructure error/exception
+      retry: Task has a known exception and is automatically retried
       success: Build/Test executed without error or failure
       canceled: The job was cancelled by a user
       unknown: When the job is not yet completed
@@ -155,6 +156,7 @@ properties:
       - success
       - fail
       - exception
+      - retry
       - canceled
       - superseded
       - unknown

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -421,7 +421,8 @@ def try_push_stored(try_repository, sample_push):
 def eleven_job_blobs(sample_data, sample_push, test_repository, mock_log_parser):
     store_push_data(test_repository, sample_push)
 
-    num_jobs = 11
+    # NOTE: when generating new data, we appear to need more jobs to find similar jobs
+    num_jobs = 100
     jobs = sample_data.job_data[0:num_jobs]
 
     max_index = len(sample_push) - 1
@@ -1146,7 +1147,7 @@ def bug_data(eleven_jobs_stored, test_repository, test_push, bugs):
     bug_id = bugs[0].bugzilla_id
     job_id = jobs[0].id
     th_models.BugJobMap.create(job_id=job_id, bugzilla_id=bug_id)
-    query_string = f"?startday=2012-05-09&endday=2018-05-10&tree={test_repository.name}"
+    query_string = f"?startday=2022-05-09&endday=2025-05-10&tree={test_repository.name}"
 
     return {
         "tree": test_repository.name,
@@ -1188,7 +1189,7 @@ def test_run_data(bug_data):
 
 @pytest.fixture
 def group_data(transactional_db, eleven_job_blobs, create_jobs):
-    query_string = "?manifest=/test&date=2022-10-01"
+    query_string = "?manifest=/test&date=2025-02-19"
 
     jt = []
     jt.append(

diff --git a/tests/etl/test_job_ingestion.py b/tests/etl/test_job_ingestion.py
@@ -19,7 +19,7 @@ def test_ingest_single_sample_job(
     job = Job.objects.get(id=1)
     # Ensure we don't inadvertently change the way we generate job-related hashes.
     assert job.option_collection_hash == "32faaecac742100f7753f0c1d0aa0add01b4046b"
-    assert job.signature.signature == "d900aca1e93a9ef2d9e00c1877c838ea920abca1"
+    assert job.signature.signature == "d30205e5d3010b250095d52983ad7fd17d10ad34"
 
 
 def test_ingest_all_sample_jobs(

diff --git a/tests/etl/test_job_loader.py b/tests/etl/test_job_loader.py
@@ -120,29 +120,50 @@ def test_ingest_pulse_jobs(
         jl.process_job(job, "https://firefox-ci-tc.services.mozilla.com")
 
     jobs = Job.objects.all()
-    assert len(jobs) == 5
+    assert len(jobs) == 30
 
     assert [job.taskcluster_metadata for job in jobs]
     assert set(TaskclusterMetadata.objects.values_list("task_id", flat=True)) == set(
         [
-            "IYyscnNMTLuxzna7PNqUJQ",
-            "XJCbbRQ6Sp-UL1lL-tw5ng",
-            "ZsSzJQu3Q7q2MfehIBAzKQ",
-            "bIzVZt9jQQKgvQYD3a2HQw",
+            "AI3Nrr3gSDSpZ9E9aBA3rg",
+            "BAG7ifS1QbGCDwiOP7NklQ",
+            "CaK6NlfBSf6F-NAVrrKJDQ",
+            "CilZCnmiTKmagJe_h6Hq5A",
+            "FNT3BLiQRHO14NNgonjQQg",
+            "FcbIUoVbS4utxFES84wrPw",
+            "FclD6gA-TTGgvq_r9-LSDg",
+            "GPLk78m6Sz6TTLJFVca4Xw",
+            "GcvHP6HLSeO_rKYDN2y_Tg",
+            "I-Hg7bM4TUOq4JqnX0pt0g",
+            "I2Y-TBNcQPSJzsKlB95rfQ",
+            "M1ECjPJBTlmwJxZq5pWyvg",
+            "MKq8mMM-RIOxztXO5ng-_A",
+            "MrrbifzBQJefUbS2ym4Qag",
+            "ORYYNMhET0yxGMvel4Jujg",
+            "TqWDDGoWSbCH93RTTxPAWg",
+            "V8rtIDroRV-G9bzjJglS0A",
+            "VVa2amzMS-2cSDbig9RHsw",
+            "YIOK401yR2GvygIFcfPVBg",
+            "b_QCzMjVQmKPyO5Il0Jedw",
+            "bljbLRFdT4KGCWJ2_C6RsQ",
+            "c2dxYucCSMWPlTkb70r89g",
+            "cPe8y071Spat09dlAzCGug",
+            "cZ7gc9JYQa2UPEC_EIxIug",
+            "dB8R5AXORZeCpDfQlYUlow",
+            "e1YPllz6TMawISpugkRx1g",
+            "eJ9PG41tSaWzNU1uY7-uSQ",
+            "edzgzCphTAS-QN_TAnf7eA",
+            "ekQaeC_yR0K8jPKx28E7EA",
+            "ftXsRyOwRgeiiYHyITXUOA",
         ]
     )
 
     job_logs = JobLog.objects.filter(job_id=1)
-    assert job_logs.count() == 2
+    assert job_logs.count() == 1
     logs_expected = [
-        {
-            "name": "errorsummary_json",
-            "url": "http://example.com/blobs/Mozilla-Inbound-Non-PGO/sha512/05c7f57df6583c6351c6b49e439e2678e0f43c2e5b66695ea7d096a7519e1805f441448b5ffd4cc3b80b8b2c74b244288fda644f55ed0e226ef4e25ba02ca466",
-            "parse_status": 0,
-        },
         {
             "name": "live_backing_log",
-            "url": "http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/tinderbox-builds/mozilla-inbound-linux64/mozilla-inbound_linux64_spidermonkey-warnaserr-bm57-build1-build352.txt.gz",
+            "url": "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AI3Nrr3gSDSpZ9E9aBA3rg/runs/0/artifacts/public/logs/live_backing.log",
             "parse_status": 0,
         },
     ]
@@ -191,10 +212,10 @@ def test_ingest_pending_pulse_job(
 
     job = jobs[0]
     assert job.taskcluster_metadata
-    assert job.taskcluster_metadata.task_id == "IYyscnNMTLuxzna7PNqUJQ"
+    assert job.taskcluster_metadata.task_id == "AI3Nrr3gSDSpZ9E9aBA3rg"
 
     # should not have processed any log or details for pending jobs
-    assert JobLog.objects.count() == 2
+    assert JobLog.objects.count() == 1
 
 
 def test_ingest_pulse_jobs_bad_project(
@@ -214,7 +235,7 @@ def test_ingest_pulse_jobs_bad_project(
         jl.process_job(pulse_job, "https://firefox-ci-tc.services.mozilla.com")
 
     # length of pulse jobs is 5, so one will be skipped due to bad project
-    assert Job.objects.count() == 4
+    assert Job.objects.count() == 29
 
 
 @responses.activate
@@ -228,7 +249,7 @@ def test_ingest_pulse_jobs_with_missing_push(pulse_jobs):
     job["origin"]["revision"] = "1234567890123456789012345678901234567890"
     responses.add(
         responses.GET,
-        "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/IYyscnNMTLuxzna7PNqUJQ",
+        "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AI3Nrr3gSDSpZ9E9aBA3rg",
         json={},
         content_type="application/json",
         status=200,

diff --git a/tests/intermittents_commenter/expected_comment.text b/tests/intermittents_commenter/expected_comment.text
@@ -1,19 +1,19 @@
 ## Summary
-1 failures in 10 pushes (0.1 failures/push) were associated with this bug in the last 7 days.
+1 failures in 23 pushes (0.043 failures/push) were associated with this bug in the last 7 days.
 
 This is the #1 most frequent failure this week.
 
 ## Repository breakdown:
 * mozilla-central: 1
 
 ## Platform and build breakdown:
-* linux1804: 1
+* mac64: 1
   * debug: 1
 
 ## Table
-| |**no_variant**|
+| |**spi-nw**|
 |---|:-:|
-|**linux1804-x86/debug**|1|
+|**mac64-x86_64/debug**|1|
 
 ## For more details, see:
-https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all
+https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2022-05-09&endday=2025-05-10&tree=all
diff --git a/tests/intermittents_commenter/expected_comment_with_5_failures.text b/tests/intermittents_commenter/expected_comment_with_5_failures.text
@@ -1,30 +1,22 @@
 ## Summary
-5 failures in 10 pushes (0.5 failures/push) were associated with this bug in the last 7 days.
+5 failures in 23 pushes (0.217 failures/push) were associated with this bug in the last 7 days.
 
 This is the #1 most frequent failure this week.
 
 ## Repository breakdown:
 * mozilla-central: 5
 
 ## Platform and build breakdown:
-* linux1804: 2
-  * debug: 1
-  * opt: 1
-* mac1015: 1
-  * debug: 1
-* mac1120: 1
-  * debug: 1
-* windows7-32: 1
-  * debug: 1
+* linux64: 3
+  * debug: 3
+* mac64: 2
+  * debug: 2
 
 ## Table
-| |**headless**|**no_variant**|
-|---|:-:|:-:|
-|**linux1804-x86/debug**| |1|
-|**linux1804-x86/opt**| |1|
-|**mac1015-x86_64/debug**| |1|
-|**mac1120-x86_64/debug**|1| |
-|**windows7-32-x86/debug**|1| |
+| |**headless**|**spi-nw**|**swr**|
+|---|:-:|:-:|:-:|
+|**linux64-x86_64/debug**|2|1| |
+|**mac64-x86_64/debug**| |1|1|
 
 ## For more details, see:
-https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all
+https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2022-05-09&endday=2025-05-10&tree=all
diff --git a/tests/intermittents_commenter/test_commenter.py b/tests/intermittents_commenter/test_commenter.py
@@ -6,8 +6,8 @@
 
 @responses.activate
 def test_intermittents_commenter(bug_data, mock_test_variants_firefoxci_request):
-    startday = "2012-05-09"
-    endday = "2018-05-10"
+    startday = "2022-05-09"
+    endday = "2025-05-10"
     alt_startday = startday
     alt_endday = endday
 
@@ -46,8 +46,8 @@ def test_intermittents_commenter(bug_data, mock_test_variants_firefoxci_request)
 def test_intermittents_commenter_with_failures(
     bug_data_with_5_failures, mock_test_variants_firefoxci_request
 ):
-    startday = "2012-05-09"
-    endday = "2018-05-10"
+    startday = "2022-05-09"
+    endday = "2025-05-10"
     alt_startday = startday
     alt_endday = endday
 

diff --git a/tests/log_parser/test_tasks.py b/tests/log_parser/test_tasks.py
@@ -15,13 +15,14 @@
 def jobs_with_local_log(activate_responses):
     sample_data = SampleData()
     url = add_log_response(
-        "mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.txt.gz"
+        "mozilla-central_mountainlion_test-mochitest-2-bm80-tests1-macosx-build138.txt.gz"
     )
 
     job = sample_data.job_data[0]
 
     # substitute the log url with a local url
     job["job"]["log_references"][0]["url"] = url
+    job["job"]["log_references"][0]["name"] = "live_backing_log"
     return [job]
 
 
@@ -66,6 +67,7 @@ def test_create_error_summary(
         assert set(failure_line.keys()) == expected_keys
 
 
+
 @pytest.mark.django_db
 @patch(
     "treeherder.model.error_summary.get_error_search_term_and_path",

diff --git a/tests/perf/auto_perf_sheriffing/test_backfill_reports/test_identify_retriggerables.py b/tests/perf/auto_perf_sheriffing/test_backfill_reports/test_identify_retriggerables.py
@@ -66,8 +66,8 @@ def test_identify_retriggerables_selects_all_data_points(gapped_performance_data
     min_push_timestamp = min(push_timestamps)
     max_push_timestamp = max(push_timestamps)
 
-    assert datetime.datetime(year=2013, month=11, day=12) <= min_push_timestamp
-    assert max_push_timestamp <= datetime.datetime(year=2013, month=11, day=14)
+    assert datetime.datetime(year=2025, month=2, day=27) <= min_push_timestamp
+    assert max_push_timestamp <= datetime.datetime(year=2025, month=2, day=27, hour=14, minute=41, second=1)
 
 
 def test_identify_retriggerables_selects_even_single_data_point(
@@ -92,6 +92,6 @@ def test_identify_retriggerables_doesnt_select_out_of_range_data_points(
 
     job_ids_to_retrigger = set(map(get_key("job_id"), data_points_to_retrigger))
 
-    assert len(data_points_to_retrigger) == 1
+    assert len(data_points_to_retrigger) == 2
     assert NON_RETRIGGERABLE_JOB_ID not in job_ids_to_retrigger
-    assert {4} == job_ids_to_retrigger
+    assert {4, 31} == job_ids_to_retrigger
diff --git a/tests/push_health/test_builds.py b/tests/push_health/test_builds.py
@@ -17,5 +17,5 @@ def test_get_build_failures(
     result, build_failures, in_progress = get_build_failures(test_push)
 
     assert in_progress == 0
-    assert result == "fail"
-    assert len(build_failures) == 2
+    assert result == "none"
+    assert len(build_failures) == 0
diff --git a/tests/sample_data/artifacts/text_log_summary.json b/tests/sample_data/artifacts/text_log_summary.json
@@ -18,7 +18,7 @@
         "linenumber": 8157
       }
     ],
-    "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/2015/04/2015-04-15-03-02-06-mozilla-central/mozilla-central_snowleopard_test-mochitest-2-bm107-tests1-macosx-build128.txt.gz"
+    "logurl": "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/YxJCaWl3Shu4FBJrKG2K1Q/runs/0/artifacts/public/logs/live_backing.log.gz"
   },
   "type": "json",
   "id": 8217463,