Reference check machanism

soumeh01 · Jul 11, 2024 · 6cbac35 · 6cbac35
1 parent fa4cc3d
commit 6cbac35
Show file tree

Hide file tree

Showing 17 changed files with 222 additions and 35 deletions.
diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml
@@ -72,7 +72,10 @@ jobs:
       - name: Run Test
         shell: bash
         run: |
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -229,9 +229,14 @@ jobs:
 
       - name: Run Test
         shell: bash
+        continue-on-error: true
         run: |
           export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
+            --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()
@@ -245,6 +250,7 @@ jobs:
     needs: [run-tests]
     runs-on: ubuntu-latest
     permissions: write-all
+
     steps:
     - uses: actions/checkout@v4
 
@@ -267,6 +273,7 @@ jobs:
 
     - name: Consolidate robot test results
       working-directory: artifacts
+      continue-on-error: true
       run: |
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-windows-amd64/output.xml \
@@ -276,7 +283,10 @@ jobs:
     - name: Generate Summary report
       if: always()
       run: |
-        python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md
+        python ./test/lib/execution_summary.py artifacts \
+          -o artifacts/collective_robot_results/output.xml \
+          -r ./test/reference.md \
+          -m summary_report.md
 
     - name: Print E2E Report
       if: always()

diff --git a/test/data/build-asm/solution.csolution.yml b/test/data/build-asm/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/build-c/solution.csolution.yml b/test/data/build-c/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/build-cpp/solution.csolution.yml b/test/data/build-cpp/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/include-define/solution.csolution.yml b/test/data/include-define/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/language-scope/solution.csolution.yml b/test/data/language-scope/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/linker-pre-processing/solution.csolution.yml b/test/data/linker-pre-processing/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/pre-include/solution.csolution.yml b/test/data/pre-include/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/whitespace/solution.csolution.yml b/test/data/whitespace/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/lib/execution_summary.py b/test/lib/execution_summary.py
@@ -13,6 +13,7 @@
 from robot.api import ExecutionResult, ResultVisitor
 from robot.result.model import TestCase
 from robot.result.executionresult import Result 
+from reference_compare import *
 
 class ResultVisitorEx(ResultVisitor):
     def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str):
@@ -89,6 +90,7 @@ def end_result(self, result: Result):
             self.__write_test_section(f, self.failed_tests, "Failed Tests", "|Tag|Test|Message|:clock1030: Duration|Suite|\n")
             self.__write_test_section(f, self.skipped_tests, "Skipped Tests", "|Tag|Test|Suite|\n")
 
+
     def __write_test_section(self, file, test_dict, section_header, table_header):
         if len(test_dict) != 0:
             file.write(f"\n## {section_header}\n\n")
@@ -113,20 +115,31 @@ def __write_test_section(self, file, test_dict, section_header, table_header):
 def main():
     parser = argparse.ArgumentParser(description='Consolidate test summary report')
     parser.add_argument('test_env_files_path', type=str, help='Path to the test environment files')
+    parser.add_argument('-r', '--reference_file', type=str, help='Path to reference file')
     parser.add_argument('-o', '--output_file', type=str, nargs='?', default='output.xml', help='Path to output xml file')
     parser.add_argument('-m', '--markdown_file', type=str, nargs='?', default='summary_report.md', help='Path to consolidated summary markdown file')
+
     args = parser.parse_args()
 
     test_env_files_path = args.test_env_files_path
     output_file = args.output_file
     markdown_file = args.markdown_file
+    reference_file = args.reference_file
 
     result = ExecutionResult(output_file)
     result.visit(ResultVisitorEx(test_env_files_path, output_file, markdown_file))
 
+    if os.path.exists(markdown_file):
+        print(f"The file '{markdown_file}' exists.")
+    else:
+        print(f"The file '{markdown_file}' does not exist.")
+
+    print("MD: {}, REF: {}".format(markdown_file, reference_file))
+    return compare_summary(markdown_file, reference_file)
+
 if __name__ == '__main__':
     try:
-        main()
+        sys.exit(main())
     except Exception as e:
         print(f'An error occurred: {e}', file=sys.stderr)
         sys.exit(1)
diff --git a/test/lib/reference_compare.py b/test/lib/reference_compare.py
@@ -0,0 +1,76 @@
+import markdown
+import argparse
+import sys
+import pandas as pd
+from io import StringIO
+
+class SummaryResult:
+    def __init__(self, passed, failed, skipped, total):
+        self.passed = passed
+        self.failed = failed
+        self.skipped = skipped
+        self.total = total
+
+class MarkdownReader:
+    def __init__(self, input_file_path:str):
+      self.input_file=input_file_path
+      self.html_content = ""
+
+    def read(self):
+      # Read the Markdown file
+      with open(self.input_file, 'r') as file:
+        md_content = file.read()
+
+      print(f"The file content '{md_content}'")
+
+      # Convert the Markdown to HTML
+      self.html_content = markdown.markdown(md_content)
+
+    def extract_summary_table(self):
+       self.read()
+       # Extract the tables from the HTML
+       tables = pd.read_html(StringIO(self.html_content))
+       # The second table is the summary table
+       summary_table = tables[1]
+       # Extract the summary results
+       passed = int(summary_table.iloc[0, 0])
+       failed = int(summary_table.iloc[0, 1])
+       skipped = int(summary_table.iloc[0, 2])
+       total = int(summary_table.iloc[0, 3])
+
+       return SummaryResult(passed, failed, skipped, total) 
+
+
+def compare_summary(markdownFile:str, referenceFile:str):
+    mdFileSummary = MarkdownReader(markdownFile).extract_summary_table()
+    refFileSummary = MarkdownReader(referenceFile).extract_summary_table()
+    # Compare the results
+    results_match = (
+        mdFileSummary.passed == refFileSummary.passed and
+        mdFileSummary.failed == refFileSummary.failed and
+        mdFileSummary.skipped == refFileSummary.skipped and
+        mdFileSummary.total == refFileSummary.total
+    )
+
+    if results_match == False:
+       print(f"Test results do not match the reference.\n"
+          f"Expected: Passed: {refFileSummary.passed}, Failed: {refFileSummary.failed}, Skipped: {refFileSummary.skipped}, Total: {refFileSummary.total}\n"
+          f"Actual: Passed: {mdFileSummary.passed}, Failed: {mdFileSummary.failed}, Skipped: {mdFileSummary.skipped}, Total: {mdFileSummary.total}")
+    return results_match
+
+def main():
+    parser = argparse.ArgumentParser(description='Reference check')
+    parser.add_argument('-r', '--reference_file', type=str, help='Path to reference file')
+    parser.add_argument('-m', '--markdown_file', type=str, nargs='?', default='summary_report.md', help='Path to consolidated summary markdown file')
+    args = parser.parse_args()
+    markdown_file = args.markdown_file
+    reference_file = args.reference_file
+
+    compare_summary(markdown_file, reference_file)
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except Exception as e:
+        print(f'An error occurred: {e}', file=sys.stderr)
+        sys.exit(1)
diff --git a/test/reference.md b/test/reference.md
@@ -0,0 +1,81 @@
+# Robot Framework Report
+
+## Test Environment
+
+|Name|Version|
+|:---|:------|
+|cbuild|2.4.0-43-gd715d2f|
+|cpackget|2.1.3-1-g0aaeca9|
+|csolution|2.4.0+p63-g2fdb3c85|
+|cbuild2cmake|0.9.1-26-g1120292|
+|cbuildgen|2.4.0+p62-g2fdb3c85|
+
+## Summary
+
+|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|
+|:----:|:----:|:-----:|:---:|
+|39|15|0|53|
+
+## Passed Tests
+
+|Tag|Test|:clock1030: Duration|Suite|
+|:---:|:---:|:---:|:---:|
+|windows-amd64|Validate build-c Example|6.01 s|Local Example Tests|
+|windows-amd64|Validate build-cpp Example|9.21 s|Local Example Tests|
+|windows-amd64|Validate linker-pre-processing Example|6.39 s|Local Example Tests|
+|windows-amd64|Validate pre-include Example|6.10 s|Local Example Tests|
+|windows-amd64|Validate whitespace Example|6.04 s|Local Example Tests|
+|windows-amd64|Validate trustzone Example|48.31 s|Local Example Tests|
+|windows-amd64|Hello_B-U585I-IOT02A|44.34 s|Remote Example Tests|
+|windows-amd64|Hello_FRDM-K32L3A6|27.11 s|Remote Example Tests|
+|windows-amd64|Hello_IMXRT1050-EVKB|33.85 s|Remote Example Tests|
+|windows-amd64|Hello_LPCXpresso55S69|29.09 s|Remote Example Tests|
+|windows-amd64|Blinky_FRDM-K32L3A6|24.95 s|Remote Example Tests|
+|windows-amd64|keil-studio-get-started|7.20 s|Remote Example Tests|
+|windows-amd64|Hello_AVH|72.43 s|Remote Example Tests|
+|linux-amd64|Validate build-c Example|1.59 s|Local Example Tests|
+|linux-amd64|Validate build-cpp Example|2.45 s|Local Example Tests|
+|linux-amd64|Validate linker-pre-processing Example|1.62 s|Local Example Tests|
+|linux-amd64|Validate pre-include Example|1.69 s|Local Example Tests|
+|linux-amd64|Validate whitespace Example|1.61 s|Local Example Tests|
+|linux-amd64|Validate trustzone Example|7.59 s|Local Example Tests|
+|linux-amd64|Hello_B-U585I-IOT02A|19.91 s|Remote Example Tests|
+|linux-amd64|Hello_FRDM-K32L3A6|8.79 s|Remote Example Tests|
+|linux-amd64|Hello_IMXRT1050-EVKB|11.89 s|Remote Example Tests|
+|linux-amd64|Hello_LPCXpresso55S69|15.33 s|Remote Example Tests|
+|linux-amd64|Blinky_FRDM-K32L3A6|9.20 s|Remote Example Tests|
+|linux-amd64|keil-studio-get-started|3.63 s|Remote Example Tests|
+|linux-amd64|Hello_AVH|28.00 s|Remote Example Tests|
+|darwin-amd64|Validate build-c Example|4.44 s|Local Example Tests|
+|darwin-amd64|Validate build-cpp Example|6.57 s|Local Example Tests|
+|darwin-amd64|Validate linker-pre-processing Example|4.56 s|Local Example Tests|
+|darwin-amd64|Validate pre-include Example|4.72 s|Local Example Tests|
+|darwin-amd64|Validate whitespace Example|4.82 s|Local Example Tests|
+|darwin-amd64|Validate trustzone Example|19.98 s|Local Example Tests|
+|darwin-amd64|Hello_B-U585I-IOT02A|32.40 s|Remote Example Tests|
+|darwin-amd64|Hello_FRDM-K32L3A6|14.36 s|Remote Example Tests|
+|darwin-amd64|Hello_IMXRT1050-EVKB|15.43 s|Remote Example Tests|
+|darwin-amd64|Hello_LPCXpresso55S69|15.10 s|Remote Example Tests|
+|darwin-amd64|Blinky_FRDM-K32L3A6|13.49 s|Remote Example Tests|
+|darwin-amd64|keil-studio-get-started|5.79 s|Remote Example Tests|
+|darwin-amd64|Hello_AVH|47.37 s|Remote Example Tests|
+
+## Failed Tests
+
+|Tag|Test|Message|:clock1030: Duration|Suite|
+|:---:|:---:|:---:|:---:|:---:|
+|windows-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|10.03 s|Local Example Tests|
+|windows-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|5.12 s|Local Example Tests|
+|windows-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.51 s|Local Example Tests|
+|windows-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|22.25 s|Remote Example Tests|
+|windows-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|27.58 s|Remote Example Tests|
+|linux-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.25 s|Local Example Tests|
+|linux-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|1.42 s|Local Example Tests|
+|linux-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|1.00 s|Local Example Tests|
+|linux-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|10.52 s|Remote Example Tests|
+|linux-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|11.91 s|Remote Example Tests|
+|darwin-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|8.99 s|Local Example Tests|
+|darwin-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|3.58 s|Local Example Tests|
+|darwin-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|2.35 s|Local Example Tests|
+|darwin-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|14.92 s|Remote Example Tests|
+|darwin-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|21.39 s|Remote Example Tests|
diff --git a/test/remote_example_tests.csv b/test/remote_example_tests.csv
@@ -0,0 +1,10 @@
+*** Test Cases ***;${github_url};${expect};[Tags];[Documentation]
+Hello_B-U585I-IOT02A;https://github.com/Arm-Examples/Hello_B-U585I-IOT02A;${0};;
+Hello_FRDM-K32L3A6;https://github.com/Arm-Examples/Hello_FRDM-K32L3A6;${0};;
+Hello_IMXRT1050-EVKB;https://github.com/Arm-Examples/Hello_IMXRT1050-EVKB;${0};;
+Hello_LPCXpresso55S69;https://github.com/Arm-Examples/Hello_LPCXpresso55S69;${0};;
+Blinky_FRDM-K32L3A6;https://github.com/Arm-Examples/Blinky_FRDM-K32L3A6;${0};;
+Blinky_NUCLEO-G0B1RE;https://github.com/Arm-Examples/Blinky_NUCLEO-G0B1RE;${0};;
+Blinky_NUCLEO-F756ZG;https://github.com/Arm-Examples/Blinky_NUCLEO-F756ZG;${0};;
+keil-studio-get-started;https://github.com/Arm-Examples/keil-studio-get-started;${0};;
+Hello_AVH;https://github.com/Arm-Examples/Hello_AVH;${0};;
diff --git a/test/remote_example_tests.robot b/test/remote_example_tests.robot
@@ -5,39 +5,28 @@ Suite Teardown          Global Teardown
 Library                 lib${/}utils.py 
 Library                 String
 Library                 Collections
-Library                 lib${/}elf_compare.py 
+Library                 lib${/}elf_compare.py
+Library                 DataDriver
 Resource                resources${/}global.resource
 Resource                resources${/}utils.resource
 Test Template           Test github examples
 
-
-*** Variables ***
-${csolution-examples}       https://github.com/Open-CMSIS-Pack/csolution-examples
-${Hello_B-U585I-IOT02A}     https://github.com/Arm-Examples/Hello_B-U585I-IOT02A
-${Hello_FRDM-K32L3A6}       https://github.com/Arm-Examples/Hello_FRDM-K32L3A6
-
+# *** Variables ***
 
 *** Test Cases ***
-# Test Csolution example
-#     ${csolution-examples}    ${Pass}
-
-# Test Hello_1 example
-#     ${Hello_B-U585I-IOT02A}    ${Pass}
-
-# Test Hello_2 example
-#     ${Hello_FRDM-K32L3A6}    ${Pass}
+Test remote example ${github_url} and expect ${expect}    Default    UserData
 
 *** Keywords ***
 Test github examples
-    [Arguments]    ${github_url}        ${expect}
-    ${dest_dir}=    Evaluate    "${github_url}".split('/')[-1]
-    ${dest_dir}=    Set Variable    ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
-    Checkout GitHub Repository    ${github_url}    ${dest_dir}       
+    [Arguments]     ${github_url}        ${expect}
+    ${dest_dir}=    Get Destination Path    ${github_url}
+    # ${dest_dir}=    Evaluate             "${github_url}".split('/')[-1]
+    # ${dest_dir}=    Set Variable         ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
+    Checkout GitHub Repository           ${github_url}    ${dest_dir}       
     ${files}    Glob Files In Directory    ${dest_dir}    *.csolution.*    ${True}
     FOR    ${file}    IN    @{files}
-        ${example_name}=    Get Parent Directory Name    ${file}
-        Run Keyword If    '${example_name}' != 'CubeMX'    
-        ...    Run Csolution Project    ${file}    ${expect}
+        # ${example_name}=    Get Parent Directory Name    ${file}
+        Run Csolution Project    ${file}    ${expect}
     END
 
 Run Csolution Project

diff --git a/test/requirements.txt b/test/requirements.txt
@@ -1,2 +1,8 @@
 # Requirements needed
 robotframework >= 7.0
+robotframework-datadriver >= 1.11.2
+markdown >= 3.6
+pandas >= 2.2.2
+lxml >= 5.0.0
+html5lib >= 1.1
+beautifulsoup4 >= 4.12.3