Reference check machanism

soumeh01 · Jul 11, 2024 · 2bc4f34 · 2bc4f34
1 parent fa4cc3d
commit 2bc4f34
Show file tree

Hide file tree

Showing 17 changed files with 226 additions and 40 deletions.
diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml
@@ -72,7 +72,10 @@ jobs:
       - name: Run Test
         shell: bash
         run: |
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -229,9 +229,14 @@ jobs:
 
       - name: Run Test
         shell: bash
+        continue-on-error: true
         run: |
           export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
+            --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()
@@ -245,6 +250,7 @@ jobs:
     needs: [run-tests]
     runs-on: ubuntu-latest
     permissions: write-all
+
     steps:
     - uses: actions/checkout@v4
 
@@ -267,6 +273,7 @@ jobs:
 
     - name: Consolidate robot test results
       working-directory: artifacts
+      continue-on-error: true
       run: |
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-windows-amd64/output.xml \
@@ -276,7 +283,10 @@ jobs:
     - name: Generate Summary report
       if: always()
       run: |
-        python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md
+        python ./test/lib/execution_summary.py artifacts \
+          -r ./test/reference.md \
+          -o artifacts/collective_robot_results/output.xml \
+          -m summary_report.md
 
     - name: Print E2E Report
       if: always()

diff --git a/test/data/build-asm/solution.csolution.yml b/test/data/build-asm/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/build-c/solution.csolution.yml b/test/data/build-c/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/build-cpp/solution.csolution.yml b/test/data/build-cpp/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/include-define/solution.csolution.yml b/test/data/include-define/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/language-scope/solution.csolution.yml b/test/data/language-scope/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/linker-pre-processing/solution.csolution.yml b/test/data/linker-pre-processing/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/pre-include/solution.csolution.yml b/test/data/pre-include/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/whitespace/solution.csolution.yml b/test/data/whitespace/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/lib/execution_summary.py b/test/lib/execution_summary.py
@@ -13,6 +13,7 @@
 from robot.api import ExecutionResult, ResultVisitor
 from robot.result.model import TestCase
 from robot.result.executionresult import Result 
+from reference_compare import *
 
 class ResultVisitorEx(ResultVisitor):
     def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str):
@@ -89,6 +90,7 @@ def end_result(self, result: Result):
             self.__write_test_section(f, self.failed_tests, "Failed Tests", "|Tag|Test|Message|:clock1030: Duration|Suite|\n")
             self.__write_test_section(f, self.skipped_tests, "Skipped Tests", "|Tag|Test|Suite|\n")
 
+
     def __write_test_section(self, file, test_dict, section_header, table_header):
         if len(test_dict) != 0:
             file.write(f"\n## {section_header}\n\n")
@@ -113,20 +115,21 @@ def __write_test_section(self, file, test_dict, section_header, table_header):
 def main():
     parser = argparse.ArgumentParser(description='Consolidate test summary report')
     parser.add_argument('test_env_files_path', type=str, help='Path to the test environment files')
+    parser.add_argument('-r', '--reference_file', type=str, help='Path to reference file')
     parser.add_argument('-o', '--output_file', type=str, nargs='?', default='output.xml', help='Path to output xml file')
     parser.add_argument('-m', '--markdown_file', type=str, nargs='?', default='summary_report.md', help='Path to consolidated summary markdown file')
     args = parser.parse_args()
 
-    test_env_files_path = args.test_env_files_path
-    output_file = args.output_file
-    markdown_file = args.markdown_file
+    # generate summary report
+    result = ExecutionResult(args.output_file)
+    result.visit(ResultVisitorEx(args.test_env_files_path, args.output_file, args.markdown_file))
 
-    result = ExecutionResult(output_file)
-    result.visit(ResultVisitorEx(test_env_files_path, output_file, markdown_file))
+    # compare test execution summary and reference results 
+    return compare_summaries(args.markdown_file, args.reference_file)
 
 if __name__ == '__main__':
     try:
-        main()
+        sys.exit(main())
     except Exception as e:
         print(f'An error occurred: {e}', file=sys.stderr)
         sys.exit(1)
diff --git a/test/lib/reference_compare.py b/test/lib/reference_compare.py
@@ -0,0 +1,85 @@
+import argparse
+import sys
+import re
+
+class SummaryResult:
+    def __init__(self, passed, failed, skipped, total):
+        self.passed = passed
+        self.failed = failed
+        self.skipped = skipped
+        self.total = total
+
+    def __eq__(self, other):
+        if isinstance(other, SummaryResult):
+            return (self.passed == other.passed and
+                    self.failed == other.failed and
+                    self.skipped == other.skipped and
+                    self.total == other.total)
+        return False
+
+    def __str__(self):
+        return f'SummaryResult(passed={self.passed}, failed={self.failed}, skipped={self.skipped}, total={self.total})'
+
+class MarkdownReader:
+    def __init__(self, input_file_path: str):
+        self.input_file = input_file_path
+        self.content = ""
+
+    def read(self):
+        # Read the Markdown file
+        try:
+            with open(self.input_file, 'r') as file:
+                self.content = file.read()
+        except FileNotFoundError:
+            print(f"File not found: {self.input_file}", file=sys.stderr)
+            raise
+        except Exception as e:
+            print(f"Error reading file {self.input_file}: {e}", file=sys.stderr)
+            raise
+
+    def extract_summary(self):
+        self.read()
+        # Use regex to find the Summary section and extract the table values
+        summary_pattern = re.compile(
+            r"## Summary\n\n\|:white_check_mark: Passed\|:x: Failed\|:fast_forward: Skipped\|Total\|\n"
+            r"\|:----:\|:----:\|:-----:\|:---:\|\n\|(\d+)\|(\d+)\|(\d+)\|(\d+)\|"
+        )
+        match = summary_pattern.search(self.content)
+        if match:
+            return SummaryResult(*map(int, match.groups()))
+        else:
+            print(f"No summary found in file: {self.input_file}", file=sys.stderr)
+            return None
+
+def compare_summaries(markdown_file: str, reference_file: str):
+    md_summary = MarkdownReader(markdown_file).extract_summary()
+    ref_summary = MarkdownReader(reference_file).extract_summary()
+
+    if md_summary is None or ref_summary is None:
+        print("Comparison could not be performed due to missing summary data.", file=sys.stderr)
+        return False
+
+    if md_summary != ref_summary:
+       print(f"Test results do not match the reference.\n"
+          f"Expected: Passed: {ref_summary.passed}, Failed: {ref_summary.failed}, Skipped: {ref_summary.skipped}, Total: {ref_summary.total}\n"
+          f"Actual: Passed: {md_summary.passed}, Failed: {md_summary.failed}, Skipped: {md_summary.skipped}, Total: {md_summary.total}")
+       return False
+    return True
+
+def main():
+    parser = argparse.ArgumentParser(description='Compare summaries in Markdown files.')
+    parser.add_argument('-r', '--reference_file', type=str, required=True, help='Path to reference file')
+    parser.add_argument('-m', '--markdown_file', type=str, default='summary_report.md', help='Path to consolidated summary markdown file')
+    args = parser.parse_args()
+
+    if compare_summaries(args.markdown_file, args.reference_file):
+        print("Summaries are equal.")
+    else:
+        print("Summaries are not equal.")
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        print(f'An error occurred: {e}', file=sys.stderr)
+        sys.exit(1)
diff --git a/test/reference.md b/test/reference.md
@@ -0,0 +1,81 @@
+# Robot Framework Report
+
+## Test Environment
+
+|Name|Version|
+|:---|:------|
+|cbuild|2.4.0-43-gd715d2f|
+|cpackget|2.1.3-1-g0aaeca9|
+|csolution|2.4.0+p63-g2fdb3c85|
+|cbuild2cmake|0.9.1-26-g1120292|
+|cbuildgen|2.4.0+p62-g2fdb3c85|
+
+## Summary
+
+|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|
+|:----:|:----:|:-----:|:---:|
+|39|15|0|53|
+
+## Passed Tests
+
+|Tag|Test|:clock1030: Duration|Suite|
+|:---:|:---:|:---:|:---:|
+|windows-amd64|Validate build-c Example|6.01 s|Local Example Tests|
+|windows-amd64|Validate build-cpp Example|9.21 s|Local Example Tests|
+|windows-amd64|Validate linker-pre-processing Example|6.39 s|Local Example Tests|
+|windows-amd64|Validate pre-include Example|6.10 s|Local Example Tests|
+|windows-amd64|Validate whitespace Example|6.04 s|Local Example Tests|
+|windows-amd64|Validate trustzone Example|48.31 s|Local Example Tests|
+|windows-amd64|Hello_B-U585I-IOT02A|44.34 s|Remote Example Tests|
+|windows-amd64|Hello_FRDM-K32L3A6|27.11 s|Remote Example Tests|
+|windows-amd64|Hello_IMXRT1050-EVKB|33.85 s|Remote Example Tests|
+|windows-amd64|Hello_LPCXpresso55S69|29.09 s|Remote Example Tests|
+|windows-amd64|Blinky_FRDM-K32L3A6|24.95 s|Remote Example Tests|
+|windows-amd64|keil-studio-get-started|7.20 s|Remote Example Tests|
+|windows-amd64|Hello_AVH|72.43 s|Remote Example Tests|
+|linux-amd64|Validate build-c Example|1.59 s|Local Example Tests|
+|linux-amd64|Validate build-cpp Example|2.45 s|Local Example Tests|
+|linux-amd64|Validate linker-pre-processing Example|1.62 s|Local Example Tests|
+|linux-amd64|Validate pre-include Example|1.69 s|Local Example Tests|
+|linux-amd64|Validate whitespace Example|1.61 s|Local Example Tests|
+|linux-amd64|Validate trustzone Example|7.59 s|Local Example Tests|
+|linux-amd64|Hello_B-U585I-IOT02A|19.91 s|Remote Example Tests|
+|linux-amd64|Hello_FRDM-K32L3A6|8.79 s|Remote Example Tests|
+|linux-amd64|Hello_IMXRT1050-EVKB|11.89 s|Remote Example Tests|
+|linux-amd64|Hello_LPCXpresso55S69|15.33 s|Remote Example Tests|
+|linux-amd64|Blinky_FRDM-K32L3A6|9.20 s|Remote Example Tests|
+|linux-amd64|keil-studio-get-started|3.63 s|Remote Example Tests|
+|linux-amd64|Hello_AVH|28.00 s|Remote Example Tests|
+|darwin-amd64|Validate build-c Example|4.44 s|Local Example Tests|
+|darwin-amd64|Validate build-cpp Example|6.57 s|Local Example Tests|
+|darwin-amd64|Validate linker-pre-processing Example|4.56 s|Local Example Tests|
+|darwin-amd64|Validate pre-include Example|4.72 s|Local Example Tests|
+|darwin-amd64|Validate whitespace Example|4.82 s|Local Example Tests|
+|darwin-amd64|Validate trustzone Example|19.98 s|Local Example Tests|
+|darwin-amd64|Hello_B-U585I-IOT02A|32.40 s|Remote Example Tests|
+|darwin-amd64|Hello_FRDM-K32L3A6|14.36 s|Remote Example Tests|
+|darwin-amd64|Hello_IMXRT1050-EVKB|15.43 s|Remote Example Tests|
+|darwin-amd64|Hello_LPCXpresso55S69|15.10 s|Remote Example Tests|
+|darwin-amd64|Blinky_FRDM-K32L3A6|13.49 s|Remote Example Tests|
+|darwin-amd64|keil-studio-get-started|5.79 s|Remote Example Tests|
+|darwin-amd64|Hello_AVH|47.37 s|Remote Example Tests|
+
+## Failed Tests
+
+|Tag|Test|Message|:clock1030: Duration|Suite|
+|:---:|:---:|:---:|:---:|:---:|
+|windows-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|10.03 s|Local Example Tests|
+|windows-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|5.12 s|Local Example Tests|
+|windows-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.51 s|Local Example Tests|
+|windows-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|22.25 s|Remote Example Tests|
+|windows-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|27.58 s|Remote Example Tests|
+|linux-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.25 s|Local Example Tests|
+|linux-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|1.42 s|Local Example Tests|
+|linux-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|1.00 s|Local Example Tests|
+|linux-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|10.52 s|Remote Example Tests|
+|linux-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|11.91 s|Remote Example Tests|
+|darwin-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|8.99 s|Local Example Tests|
+|darwin-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|3.58 s|Local Example Tests|
+|darwin-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|2.35 s|Local Example Tests|
+|darwin-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|14.92 s|Remote Example Tests|
+|darwin-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|21.39 s|Remote Example Tests|
diff --git a/test/remote_example_tests.csv b/test/remote_example_tests.csv
@@ -0,0 +1,10 @@
+*** Test Cases ***;${github_url};${expect};[Tags];[Documentation]
+Hello_B-U585I-IOT02A;https://github.com/Arm-Examples/Hello_B-U585I-IOT02A;${0};;
+Hello_FRDM-K32L3A6;https://github.com/Arm-Examples/Hello_FRDM-K32L3A6;${0};;
+Hello_IMXRT1050-EVKB;https://github.com/Arm-Examples/Hello_IMXRT1050-EVKB;${0};;
+Hello_LPCXpresso55S69;https://github.com/Arm-Examples/Hello_LPCXpresso55S69;${0};;
+Blinky_FRDM-K32L3A6;https://github.com/Arm-Examples/Blinky_FRDM-K32L3A6;${0};;
+Blinky_NUCLEO-G0B1RE;https://github.com/Arm-Examples/Blinky_NUCLEO-G0B1RE;${0};;
+Blinky_NUCLEO-F756ZG;https://github.com/Arm-Examples/Blinky_NUCLEO-F756ZG;${0};;
+keil-studio-get-started;https://github.com/Arm-Examples/keil-studio-get-started;${0};;
+Hello_AVH;https://github.com/Arm-Examples/Hello_AVH;${0};;
diff --git a/test/remote_example_tests.robot b/test/remote_example_tests.robot
@@ -5,39 +5,28 @@ Suite Teardown          Global Teardown
 Library                 lib${/}utils.py 
 Library                 String
 Library                 Collections
-Library                 lib${/}elf_compare.py 
+Library                 lib${/}elf_compare.py
+Library                 DataDriver
 Resource                resources${/}global.resource
 Resource                resources${/}utils.resource
 Test Template           Test github examples
 
-
-*** Variables ***
-${csolution-examples}       https://github.com/Open-CMSIS-Pack/csolution-examples
-${Hello_B-U585I-IOT02A}     https://github.com/Arm-Examples/Hello_B-U585I-IOT02A
-${Hello_FRDM-K32L3A6}       https://github.com/Arm-Examples/Hello_FRDM-K32L3A6
-
+# *** Variables ***
 
 *** Test Cases ***
-# Test Csolution example
-#     ${csolution-examples}    ${Pass}
-
-# Test Hello_1 example
-#     ${Hello_B-U585I-IOT02A}    ${Pass}
-
-# Test Hello_2 example
-#     ${Hello_FRDM-K32L3A6}    ${Pass}
+Test remote example ${github_url} and expect ${expect}    Default    UserData
 
 *** Keywords ***
 Test github examples
-    [Arguments]    ${github_url}        ${expect}
-    ${dest_dir}=    Evaluate    "${github_url}".split('/')[-1]
-    ${dest_dir}=    Set Variable    ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
-    Checkout GitHub Repository    ${github_url}    ${dest_dir}       
+    [Arguments]     ${github_url}        ${expect}
+    ${dest_dir}=    Get Destination Path    ${github_url}
+    # ${dest_dir}=    Evaluate             "${github_url}".split('/')[-1]
+    # ${dest_dir}=    Set Variable         ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
+    Checkout GitHub Repository           ${github_url}    ${dest_dir}       
     ${files}    Glob Files In Directory    ${dest_dir}    *.csolution.*    ${True}
     FOR    ${file}    IN    @{files}
-        ${example_name}=    Get Parent Directory Name    ${file}
-        Run Keyword If    '${example_name}' != 'CubeMX'    
-        ...    Run Csolution Project    ${file}    ${expect}
+        # ${example_name}=    Get Parent Directory Name    ${file}
+        Run Csolution Project    ${file}    ${expect}
     END
 
 Run Csolution Project

diff --git a/test/requirements.txt b/test/requirements.txt
@@ -1,2 +1,8 @@
 # Requirements needed
 robotframework >= 7.0
+robotframework-datadriver >= 1.11.2
+markdown >= 3.6
+pandas >= 2.2.2
+lxml >= 5.0.0
+html5lib >= 1.1
+beautifulsoup4 >= 4.12.3