Update the name to test-report and add README for report workflow (#3864

)

Signed-off-by: Zelin Hao <zelinhao@amazon.com>

src/manifests/test_run_manifest.py → src/manifests/test_report_manifest.py

@@ -10,9 +10,9 @@
 from manifests.component_manifest import Component, ComponentManifest, Components
 
 
-class TestRunManifest(ComponentManifest['TestRunManifest', 'TestComponents']):
+class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents']):
     """
-    TestRunManifest contains the aggregated test results for the components.
+    TestReportManifest contains the aggregated test results for the components.
 
     The format for schema version 1.0 is:
         schema-version: '1.0'
@@ -149,7 +149,7 @@ def __to_dict__(self) -> dict:
                 }
 
 
-TestRunManifest.VERSIONS = {"1.0": TestRunManifest}
+TestReportManifest.VERSIONS = {"1.0": TestReportManifest}
 
 TestComponent.__test__ = False  # type: ignore[attr-defined]
-TestRunManifest.__test__ = False  # type: ignore[attr-defined]
+TestReportManifest.__test__ = False  # type: ignore[attr-defined]

src/report_workflow/README.md

@@ -0,0 +1,24 @@
+#### Generate test-report manifest for each test.
+As the name specifies, the test report workflow helps to automatically generate a consolidated report of the tests run at distribution level along with commands to reproduce the error and associated failures.
+
+*Usage*
+```
+./report.sh <test-manifest-path> --artifact-paths opensearch=<...> opensearch-dashboards=<...> --test-run-id <...> --test-type integ-test --base-path <...>
+```
+e.g.
+```
+./report.sh manifests/2.9.0/opensearch-2.9.0-test.yml -p opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8172/linux/x64/tar --test-run-id 5328 --test-type integ-test --base-path https://ci.opensearch.org/ci/dbc/integ-test/2.9.0/8172/linux/x64/tar
+```
+The following options are available.
+
+| name                            | description                                                                    |
+|---------------------------------|--------------------------------------------------------------------------------|
+| test-manifest-path   <required> | Specify a test manifest path.                                                  |
+| -p, --artifact-paths <required> | Artifact paths of distributions used for testing.                              |
+| --base-path          <required> | Base paths of testing logs.                                                    |
+| --test-type          <required> | Type of tests report generates on.                                             |
+| --output-path        <optional> | Specify the path location for the test-report manifest.                        |
+| --test-run-id        <required> | Specify the unique execution id that matches the id of the test.               |
+| --component          <optional> | Specify a specific component or components instead of the entire distribution. |
+| --verbose            <optional> | Show more verbose output.                                                      |
+

src/report_workflow/report_args.py

@@ -30,7 +30,7 @@ def __init__(self) -> None:
         parser.add_argument("--base-path", type=str, default="",
                             help="Specify base paths for the integration test logs.")
         parser.add_argument("--test-type", type=str, default="integ-test", help="Specify test type of this.")
-        parser.add_argument("--output-path", type=str, help="Specify the path location for the test-run manifest.")
+        parser.add_argument("--output-path", type=str, help="Specify the path location for the test-report manifest.")
         parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test")
         parser.add_argument("--component", type=str, dest="components", nargs='*', help="Test a specific component or components instead of the entire distribution.")
         parser.add_argument(

src/report_workflow/test_run_runner.py → src/report_workflow/test_report_runner.py

@@ -15,22 +15,22 @@
 import yaml
 
 from manifests.test_manifest import TestManifest
-from manifests.test_run_manifest import TestRunManifest
+from manifests.test_report_manifest import TestReportManifest
 from report_workflow.report_args import ReportArgs
 
 
-class TestRunRunner:
+class TestReportRunner:
     args: ReportArgs
     test_manifest: TestManifest
     tests_dir: str
-    test_run_manifest: TestRunManifest
+    test_report_manifest: TestReportManifest
     test_run_data: dict
 
     def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None:
         self.args = args
         self.base_path = args.base_path
         self.test_manifest = test_manifest
-        self.test_run_data = self.test_run_manifest_data_template("manifest")
+        self.test_run_data = self.test_report_manifest_data_template("manifest")
         self.product_name = test_manifest.__to_dict__().get("name")
         self.name = self.product_name.replace(" ", "-").lower()
         self.components = self.args.components
@@ -63,13 +63,13 @@ def update_test_run_data(self) -> dict:
         return test_run_data
 
     def generate_report(self, data: dict, output_dir: str) -> Any:
-        test_run_manifest = TestRunManifest(data)
-        test_run_manifetest_run_manifest_file = os.path.join(output_dir, "test-run.yml")
-        logging.info(f"Generating test-run.yml in {output_dir}")
-        return test_run_manifest.to_file(test_run_manifetest_run_manifest_file)
+        test_report_manifest = TestReportManifest(data)
+        test_report_manifest_file = os.path.join(output_dir, "test-report.yml")
+        logging.info(f"Generating test-report.yml in {output_dir}")
+        return test_report_manifest.to_file(test_report_manifest_file)
 
     def component_entry(self, component_name: str) -> Any:
-        component = self.test_run_manifest_data_template("component")
+        component = self.test_report_manifest_data_template("component")
         component["name"] = component_name
         component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name)
 
@@ -102,7 +102,7 @@ def component_entry(self, component_name: str) -> Any:
             component["configs"].append(config_dict)
         return component
 
-    def test_run_manifest_data_template(self, template_type: str) -> Any:
+    def test_report_manifest_data_template(self, template_type: str) -> Any:
         templates = {
             "manifest": {
                 "schema-version": "1.0",
@@ -134,4 +134,4 @@ def generate_test_command(test_type: str, test_manifest_path: str, artifacts_pat
     return command
 
 
-TestRunRunner.__test__ = False  # type:ignore
+TestReportRunner.__test__ = False  # type:ignore

src/run_test_report.py

@@ -10,7 +10,7 @@
 
 from manifests.test_manifest import TestManifest
 from report_workflow.report_args import ReportArgs
-from report_workflow.test_run_runner import TestRunRunner
+from report_workflow.test_report_runner import TestReportRunner
 from system import console
 
 
@@ -21,11 +21,11 @@ def main() -> Any:
 
     test_manifest = TestManifest.from_path(args.test_manifest_path)
 
-    test_run_runner = TestRunRunner(args, test_manifest)
+    test_report_runner = TestReportRunner(args, test_manifest)
 
-    test_run_data = test_run_runner.update_data()
+    test_report_data = test_report_runner.update_data()
 
-    test_run = test_run_runner.generate_report(test_run_data, args.output_path or os.getcwd())
+    test_run = test_report_runner.generate_report(test_report_data, args.output_path or os.getcwd())
 
     return test_run
 

tests/test_run_test_report.py

@@ -39,7 +39,7 @@ def test_usage(self, *mocks: Any) -> None:
         self.assertTrue(out.startswith("usage:"))
 
     @patch("argparse._sys.argv", ["run_test_report.py", TEST_MANIFEST_PATH, "-p", "opensearch=foo"])
-    @patch('run_test_report.TestRunRunner')
+    @patch('run_test_report.TestReportRunner')
     def test_main(self, runner_mock: Mock, *mocks: Any) -> None:
 
         main()

tests/tests_manifests/test_test_run_manifest.py

@@ -10,16 +10,16 @@
 
 import yaml
 
-from manifests.test_run_manifest import TestRunManifest
+from manifests.test_report_manifest import TestReportManifest
 
 
-class TestTestRunManifest(unittest.TestCase):
+class TestTestReportManifest(unittest.TestCase):
 
     def setUp(self) -> None:
         self.maxDiff = None
         self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data"))
         self.manifest_filename = os.path.join(self.data_path, "test-run.yml")
-        self.manifest = TestRunManifest.from_path(self.manifest_filename)
+        self.manifest = TestReportManifest.from_path(self.manifest_filename)
 
     def test_test_run(self) -> None:
         self.assertEqual(self.manifest.name, "OpenSearch")

tests/tests_report_workflow/test_test_run_runner.py → tests/tests_report_workflow/test_test_report_runner.py

@@ -11,10 +11,11 @@
 from unittest.mock import MagicMock, call, mock_open, patch
 
 from manifests.test_manifest import TestManifest
-from report_workflow.test_run_runner import TestRunRunner
+from report_workflow.test_report_runner import TestReportRunner
+from system.temporary_directory import TemporaryDirectory
 
 
-class TestTestRunRunner(unittest.TestCase):
+class TestTestReportRunner(unittest.TestCase):
     TEST_MANIFEST_PATH = os.path.join(
         os.path.dirname(__file__), "data", "test_manifest.yml"
     )
@@ -35,12 +36,35 @@ def test_runner_init(self, report_args_mock: MagicMock, test_manifest_mock: Magi
         report_args_mock.test_run_id = 123
         report_args_mock.test_type = "integ-test"
 
-        test_run_runner = TestRunRunner(report_args_mock, self.TEST_MANIFEST)
+        test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST)
         self.assertEqual(test_run_runner.name, "opensearch")
         self.assertEqual(test_run_runner.test_run_id, 123)
         self.assertEqual(test_run_runner.test_type, "integ-test")
         self.assertEqual(test_run_runner.test_manifest_path, self.TEST_MANIFEST_PATH)
 
+    @patch("yaml.safe_load")
+    @patch("urllib.request.urlopen")
+    @patch("validators.url")
+    @patch("report_workflow.report_args.ReportArgs")
+    def test_generate_file(self, report_args_mock: MagicMock, validators_mock: MagicMock, urlopen_mock: MagicMock, yaml_safe_load_mock: MagicMock) -> None:
+        report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH
+        report_args_mock.artifact_paths = {"opensearch": "foo/bar"}
+        report_args_mock.test_run_id = 123
+        report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock"
+        report_args_mock.test_type = "integ-test"
+
+        validators_mock.return_value = True
+        yaml_safe_load_mock.return_value = {"test_result": "PASS"}
+        urlopen_mock.return_value = MagicMock()
+
+        test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST)
+        test_run_runner_data = test_run_runner.update_data()
+
+        with TemporaryDirectory() as path:
+            output_path = os.path.join(path.name, "test-report.yml")
+            test_run_runner.generate_report(test_run_runner_data, path.name)
+            self.assertTrue(os.path.isfile(output_path))
+
     @patch("report_workflow.report_args.ReportArgs")
     @patch("manifests.test_manifest.TestManifest")
     def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, test_manifest_mock: MagicMock) -> None:
@@ -49,7 +73,7 @@ def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, te
         report_args_mock.test_run_id = 123
         report_args_mock.test_type = "integ-test"
 
-        test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data()
+        test_run_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data()
         self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=foo/bar"]))
         self.assertEqual(test_run_dict.get("TestType"), "integ-test")
         self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH)
@@ -64,7 +88,7 @@ def test_runner_update_test_run_data_url(self, report_args_mock: MagicMock, test
         report_args_mock.test_run_id = 123
         report_args_mock.test_type = "integ-test"
 
-        test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data()
+        test_run_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data()
         self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=https://foo/bar"]))
         self.assertEqual(test_run_dict.get("TestType"), "integ-test")
         self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH)
@@ -86,7 +110,7 @@ def test_runner_component_entry_url(self, report_args_mock: MagicMock, validator
         yaml_safe_load_mock.return_value = {"test_result": "PASS"}
         urlopen_mock.return_value = MagicMock()
 
-        test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
+        test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
         urlopen_mock.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml')])
         self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS")
         self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security")
@@ -107,7 +131,7 @@ def test_runner_component_entry_local(self, report_args_mock: MagicMock, validat
         yaml_safe_load_mock.return_value = {"test_result": "PASS"}
         mock_open.return_value = MagicMock()
 
-        test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
+        test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
         mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')])
         self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS")
         self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security")
@@ -125,7 +149,7 @@ def test_runner_component_entry_url_invalid(self, report_args_mock: MagicMock, v
 
         validators_mock.return_value = True
 
-        test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
+        test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
         self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available")
         self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security")
         self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "URL not available")
@@ -145,7 +169,7 @@ def test_runner_component_entry_local_invalid(self, report_args_mock: MagicMock,
         yaml_safe_load_mock.return_value = {"test_result": "PASS"}
         mock_open.side_effect = FileNotFoundError
 
-        test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
+        test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial")
         mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')])
         self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available")
         self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security")