Reference check machanism

soumeh01 · Jul 10, 2024 · c7d1f50 · c7d1f50
1 parent fa4cc3d
commit c7d1f50
Show file tree

Hide file tree

Showing 17 changed files with 194 additions and 35 deletions.
diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml
@@ -72,7 +72,10 @@ jobs:
       - name: Run Test
         shell: bash
         run: |
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -229,9 +229,14 @@ jobs:
 
       - name: Run Test
         shell: bash
+        continue-on-error: true
         run: |
           export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
+            --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
+            --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
+            --name ${{ matrix.target }}-${{ matrix.arch }} \
+            ./test
 
       - name: Archieve test results
         if: always()
@@ -245,6 +250,7 @@ jobs:
     needs: [run-tests]
     runs-on: ubuntu-latest
     permissions: write-all
+
     steps:
     - uses: actions/checkout@v4
 
@@ -267,6 +273,7 @@ jobs:
 
     - name: Consolidate robot test results
       working-directory: artifacts
+      continue-on-error: true
       run: |
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-windows-amd64/output.xml \
@@ -276,7 +283,10 @@ jobs:
     - name: Generate Summary report
       if: always()
       run: |
-        python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md
+        python ./test/lib/execution_summary.py artifacts \
+          -o artifacts/collective_robot_results/output.xml \
+          -r ./test/reference.md \
+          -m summary_report.md
 
     - name: Print E2E Report
       if: always()

diff --git a/test/data/build-asm/solution.csolution.yml b/test/data/build-asm/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/build-c/solution.csolution.yml b/test/data/build-c/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/build-cpp/solution.csolution.yml b/test/data/build-cpp/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/include-define/solution.csolution.yml b/test/data/include-define/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/language-scope/solution.csolution.yml b/test/data/language-scope/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/linker-pre-processing/solution.csolution.yml b/test/data/linker-pre-processing/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/pre-include/solution.csolution.yml b/test/data/pre-include/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/data/whitespace/solution.csolution.yml b/test/data/whitespace/solution.csolution.yml
@@ -1,5 +1,4 @@
 solution:
-  created-for: CMSIS-Toolbox@2.2.1
   cdefault:
 
   packs:

diff --git a/test/lib/execution_summary.py b/test/lib/execution_summary.py
@@ -13,6 +13,7 @@
 from robot.api import ExecutionResult, ResultVisitor
 from robot.result.model import TestCase
 from robot.result.executionresult import Result 
+from reference_compare import *
 
 class ResultVisitorEx(ResultVisitor):
     def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str):
@@ -113,20 +114,26 @@ def __write_test_section(self, file, test_dict, section_header, table_header):
 def main():
     parser = argparse.ArgumentParser(description='Consolidate test summary report')
     parser.add_argument('test_env_files_path', type=str, help='Path to the test environment files')
+    parser.add_argument('-r', '--reference_file', type=str, help='Path to reference file')
     parser.add_argument('-o', '--output_file', type=str, nargs='?', default='output.xml', help='Path to output xml file')
     parser.add_argument('-m', '--markdown_file', type=str, nargs='?', default='summary_report.md', help='Path to consolidated summary markdown file')
+
     args = parser.parse_args()
 
     test_env_files_path = args.test_env_files_path
     output_file = args.output_file
     markdown_file = args.markdown_file
+    reference_file = args.reference_file
 
     result = ExecutionResult(output_file)
     result.visit(ResultVisitorEx(test_env_files_path, output_file, markdown_file))
 
+    print("MD: {}, REF: {}".format(markdown_file, reference_file))
+    return compare_summary(markdown_file, reference_file)
+
 if __name__ == '__main__':
     try:
-        main()
+        sys.exit(main())
     except Exception as e:
         print(f'An error occurred: {e}', file=sys.stderr)
         sys.exit(1)
diff --git a/test/lib/reference_compare.py b/test/lib/reference_compare.py
@@ -0,0 +1,56 @@
+import markdown
+import pandas as pd
+from io import StringIO
+
+class SummaryResult:
+    def __init__(self, passed, failed, skipped, total):
+        self.passed = passed
+        self.failed = failed
+        self.skipped = skipped
+        self.total = total
+
+class MarkdownReader:
+    def __init__(self, input_file_path:str):
+      self.input_file=input_file_path
+      self.html_content = ""
+
+    def read(self):
+      # Read the Markdown file
+      with open(self.input_file, 'r') as file:
+        md_content = file.read()
+
+      # Convert the Markdown to HTML
+      self.html_content = markdown.markdown(md_content)
+
+    def extract_summary_table(self):
+       # Extract the tables from the HTML
+       tables = pd.read_html(StringIO(self.html_content))
+       # The second table is the summary table
+       summary_table = tables[1]
+       # Extract the summary results
+       passed = int(summary_table.iloc[0, 0])
+       failed = int(summary_table.iloc[0, 1])
+       skipped = int(summary_table.iloc[0, 2])
+       total = int(summary_table.iloc[0, 3])
+
+       return SummaryResult(passed, failed, skipped, total) 
+
+
+def compare_summary(markdownFile:str, referenceFile:str):
+    mdFileSummary = MarkdownReader(markdownFile).extract_summary_table()
+    refFileSummary = MarkdownReader(referenceFile).extract_summary_table()
+    # Compare the results
+    results_match = (
+        mdFileSummary.passed == refFileSummary.passed and
+        mdFileSummary.failed == refFileSummary.failed and
+        mdFileSummary.skipped == refFileSummary.skipped and
+        mdFileSummary.total == refFileSummary.total
+    )
+
+    return results_match
+
+# print("Summary results match expected values:", results_match)
+# print(f"Passed: {passed}, Expected: {expected_passed}")
+# print(f"Failed: {failed}, Expected: {expected_failed}")
+# print(f"Skipped: {skipped}, Expected: {expected_skipped}")
+# print(f"Total: {total}, Expected: {expected_total}")
diff --git a/test/reference.md b/test/reference.md
@@ -0,0 +1,81 @@
+# Robot Framework Report
+
+## Test Environment
+
+|Name|Version|
+|:---|:------|
+|cbuild|2.4.0-43-gd715d2f|
+|cpackget|2.1.3-1-g0aaeca9|
+|csolution|2.4.0+p63-g2fdb3c85|
+|cbuild2cmake|0.9.1-26-g1120292|
+|cbuildgen|2.4.0+p62-g2fdb3c85|
+
+## Summary
+
+|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|
+|:----:|:----:|:-----:|:---:|
+|39|15|0|53|
+
+## Passed Tests
+
+|Tag|Test|:clock1030: Duration|Suite|
+|:---:|:---:|:---:|:---:|
+|windows-amd64|Validate build-c Example|6.01 s|Local Example Tests|
+|windows-amd64|Validate build-cpp Example|9.21 s|Local Example Tests|
+|windows-amd64|Validate linker-pre-processing Example|6.39 s|Local Example Tests|
+|windows-amd64|Validate pre-include Example|6.10 s|Local Example Tests|
+|windows-amd64|Validate whitespace Example|6.04 s|Local Example Tests|
+|windows-amd64|Validate trustzone Example|48.31 s|Local Example Tests|
+|windows-amd64|Hello_B-U585I-IOT02A|44.34 s|Remote Example Tests|
+|windows-amd64|Hello_FRDM-K32L3A6|27.11 s|Remote Example Tests|
+|windows-amd64|Hello_IMXRT1050-EVKB|33.85 s|Remote Example Tests|
+|windows-amd64|Hello_LPCXpresso55S69|29.09 s|Remote Example Tests|
+|windows-amd64|Blinky_FRDM-K32L3A6|24.95 s|Remote Example Tests|
+|windows-amd64|keil-studio-get-started|7.20 s|Remote Example Tests|
+|windows-amd64|Hello_AVH|72.43 s|Remote Example Tests|
+|linux-amd64|Validate build-c Example|1.59 s|Local Example Tests|
+|linux-amd64|Validate build-cpp Example|2.45 s|Local Example Tests|
+|linux-amd64|Validate linker-pre-processing Example|1.62 s|Local Example Tests|
+|linux-amd64|Validate pre-include Example|1.69 s|Local Example Tests|
+|linux-amd64|Validate whitespace Example|1.61 s|Local Example Tests|
+|linux-amd64|Validate trustzone Example|7.59 s|Local Example Tests|
+|linux-amd64|Hello_B-U585I-IOT02A|19.91 s|Remote Example Tests|
+|linux-amd64|Hello_FRDM-K32L3A6|8.79 s|Remote Example Tests|
+|linux-amd64|Hello_IMXRT1050-EVKB|11.89 s|Remote Example Tests|
+|linux-amd64|Hello_LPCXpresso55S69|15.33 s|Remote Example Tests|
+|linux-amd64|Blinky_FRDM-K32L3A6|9.20 s|Remote Example Tests|
+|linux-amd64|keil-studio-get-started|3.63 s|Remote Example Tests|
+|linux-amd64|Hello_AVH|28.00 s|Remote Example Tests|
+|darwin-amd64|Validate build-c Example|4.44 s|Local Example Tests|
+|darwin-amd64|Validate build-cpp Example|6.57 s|Local Example Tests|
+|darwin-amd64|Validate linker-pre-processing Example|4.56 s|Local Example Tests|
+|darwin-amd64|Validate pre-include Example|4.72 s|Local Example Tests|
+|darwin-amd64|Validate whitespace Example|4.82 s|Local Example Tests|
+|darwin-amd64|Validate trustzone Example|19.98 s|Local Example Tests|
+|darwin-amd64|Hello_B-U585I-IOT02A|32.40 s|Remote Example Tests|
+|darwin-amd64|Hello_FRDM-K32L3A6|14.36 s|Remote Example Tests|
+|darwin-amd64|Hello_IMXRT1050-EVKB|15.43 s|Remote Example Tests|
+|darwin-amd64|Hello_LPCXpresso55S69|15.10 s|Remote Example Tests|
+|darwin-amd64|Blinky_FRDM-K32L3A6|13.49 s|Remote Example Tests|
+|darwin-amd64|keil-studio-get-started|5.79 s|Remote Example Tests|
+|darwin-amd64|Hello_AVH|47.37 s|Remote Example Tests|
+
+## Failed Tests
+
+|Tag|Test|Message|:clock1030: Duration|Suite|
+|:---:|:---:|:---:|:---:|:---:|
+|windows-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|10.03 s|Local Example Tests|
+|windows-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|5.12 s|Local Example Tests|
+|windows-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.51 s|Local Example Tests|
+|windows-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|22.25 s|Remote Example Tests|
+|windows-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|27.58 s|Remote Example Tests|
+|linux-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.25 s|Local Example Tests|
+|linux-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|1.42 s|Local Example Tests|
+|linux-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|1.00 s|Local Example Tests|
+|linux-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|10.52 s|Remote Example Tests|
+|linux-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|11.91 s|Remote Example Tests|
+|darwin-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|8.99 s|Local Example Tests|
+|darwin-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|3.58 s|Local Example Tests|
+|darwin-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|2.35 s|Local Example Tests|
+|darwin-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|14.92 s|Remote Example Tests|
+|darwin-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|21.39 s|Remote Example Tests|
diff --git a/test/remote_example_tests.csv b/test/remote_example_tests.csv
@@ -0,0 +1,10 @@
+*** Test Cases ***;${github_url};${expect};[Tags];[Documentation]
+Hello_B-U585I-IOT02A;https://github.com/Arm-Examples/Hello_B-U585I-IOT02A;${0};;
+Hello_FRDM-K32L3A6;https://github.com/Arm-Examples/Hello_FRDM-K32L3A6;${0};;
+Hello_IMXRT1050-EVKB;https://github.com/Arm-Examples/Hello_IMXRT1050-EVKB;${0};;
+Hello_LPCXpresso55S69;https://github.com/Arm-Examples/Hello_LPCXpresso55S69;${0};;
+Blinky_FRDM-K32L3A6;https://github.com/Arm-Examples/Blinky_FRDM-K32L3A6;${0};;
+Blinky_NUCLEO-G0B1RE;https://github.com/Arm-Examples/Blinky_NUCLEO-G0B1RE;${0};;
+Blinky_NUCLEO-F756ZG;https://github.com/Arm-Examples/Blinky_NUCLEO-F756ZG;${0};;
+keil-studio-get-started;https://github.com/Arm-Examples/keil-studio-get-started;${0};;
+Hello_AVH;https://github.com/Arm-Examples/Hello_AVH;${0};;
diff --git a/test/remote_example_tests.robot b/test/remote_example_tests.robot
@@ -5,39 +5,28 @@ Suite Teardown          Global Teardown
 Library                 lib${/}utils.py 
 Library                 String
 Library                 Collections
-Library                 lib${/}elf_compare.py 
+Library                 lib${/}elf_compare.py
+Library                 DataDriver
 Resource                resources${/}global.resource
 Resource                resources${/}utils.resource
 Test Template           Test github examples
 
-
-*** Variables ***
-${csolution-examples}       https://github.com/Open-CMSIS-Pack/csolution-examples
-${Hello_B-U585I-IOT02A}     https://github.com/Arm-Examples/Hello_B-U585I-IOT02A
-${Hello_FRDM-K32L3A6}       https://github.com/Arm-Examples/Hello_FRDM-K32L3A6
-
+# *** Variables ***
 
 *** Test Cases ***
-# Test Csolution example
-#     ${csolution-examples}    ${Pass}
-
-# Test Hello_1 example
-#     ${Hello_B-U585I-IOT02A}    ${Pass}
-
-# Test Hello_2 example
-#     ${Hello_FRDM-K32L3A6}    ${Pass}
+Test remote example ${github_url} and expect ${expect}    Default    UserData
 
 *** Keywords ***
 Test github examples
-    [Arguments]    ${github_url}        ${expect}
-    ${dest_dir}=    Evaluate    "${github_url}".split('/')[-1]
-    ${dest_dir}=    Set Variable    ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
-    Checkout GitHub Repository    ${github_url}    ${dest_dir}       
+    [Arguments]     ${github_url}        ${expect}
+    ${dest_dir}=    Get Destination Path    ${github_url}
+    # ${dest_dir}=    Evaluate             "${github_url}".split('/')[-1]
+    # ${dest_dir}=    Set Variable         ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
+    Checkout GitHub Repository           ${github_url}    ${dest_dir}       
     ${files}    Glob Files In Directory    ${dest_dir}    *.csolution.*    ${True}
     FOR    ${file}    IN    @{files}
-        ${example_name}=    Get Parent Directory Name    ${file}
-        Run Keyword If    '${example_name}' != 'CubeMX'    
-        ...    Run Csolution Project    ${file}    ${expect}
+        # ${example_name}=    Get Parent Directory Name    ${file}
+        Run Csolution Project    ${file}    ${expect}
     END
 
 Run Csolution Project

diff --git a/test/requirements.txt b/test/requirements.txt
@@ -1,2 +1,6 @@
 # Requirements needed
 robotframework >= 7.0
+robotframework-datadriver >= 1.11.2
+markdown >= 3.6
+pandas >= 2.2.2
+lxml >= 5.0.0
diff --git a/test/resources/utils.resource b/test/resources/utils.resource
@@ -72,7 +72,7 @@ Run Project with cbuildgen
     [Documentation]     Run the csolution project with cbuildgen
     [Arguments]         ${input_file}    ${expect}        ${args}=@{EMPTY}
     ${parent_path}=     Get Parent Directory Path         ${input_file}
-    ${ex_args}=         Append Additional Arguments       ${args}    --output    ${parent_path}${/}${Out_Dir}
+    ${ex_args}=         Append Additional Arguments       ${args}    --output    ${parent_path}${/}${Out_Dir}    --cbuildgen
     ${ret_code}=        Run cbuild       ${input_file}    ${ex_args}
     Should Be Equal     ${ret_code}      ${expect}        msg=Unexpected status returned by cbuildgen execution
 
@@ -98,3 +98,10 @@ Get Output File Path
     [Documentation]             Find path to output files
     ${output_path}=             Get Variable Value    ${OUTPUT_FILE}
     RETURN                      ${output_path}
+
+Get Destination Path
+    [Documentation]                 Get destination directory path from url
+    [Arguments]                     ${github_url}
+    ${dest_dir}=    Evaluate        "${github_url}".split('/')[-1]
+    ${dest_dir}=    Set Variable    ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
+    RETURN                          ${dest_dir}