From c7d1f50e681aa3b8a6d9b603e7c9d7a984f0536c Mon Sep 17 00:00:00 2001 From: Sourabh Mehta Date: Wed, 10 Jul 2024 14:19:13 +0200 Subject: [PATCH] Reference check machanism --- .github/workflows/e2e_test.yaml | 5 +- .github/workflows/nightly.yml | 14 +++- test/data/build-asm/solution.csolution.yml | 1 - test/data/build-c/solution.csolution.yml | 1 - test/data/build-cpp/solution.csolution.yml | 1 - .../include-define/solution.csolution.yml | 1 - .../language-scope/solution.csolution.yml | 1 - .../solution.csolution.yml | 1 - test/data/pre-include/solution.csolution.yml | 1 - test/data/whitespace/solution.csolution.yml | 1 - test/lib/execution_summary.py | 9 ++- test/lib/reference_compare.py | 56 +++++++++++++ test/reference.md | 81 +++++++++++++++++++ test/remote_example_tests.csv | 10 +++ test/remote_example_tests.robot | 33 +++----- test/requirements.txt | 4 + test/resources/utils.resource | 9 ++- 17 files changed, 194 insertions(+), 35 deletions(-) create mode 100644 test/lib/reference_compare.py create mode 100644 test/reference.md create mode 100644 test/remote_example_tests.csv diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index dc68b966..2b9791fd 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -72,7 +72,10 @@ jobs: - name: Run Test shell: bash run: | - python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test + robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \ + --settag ${{ matrix.target }}-${{ matrix.arch }} \ + --name ${{ matrix.target }}-${{ matrix.arch }} \ + ./test - name: Archieve test results if: always() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fdbb2d48..11ad0e20 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -229,9 +229,14 @@ jobs: - name: Run Test shell: bash + continue-on-error: true run: | export PATH="${{steps.toolbox_path.outputs.path}}:$PATH" - python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test + robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \ + --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \ + --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \ + --name ${{ matrix.target }}-${{ matrix.arch }} \ + ./test - name: Archieve test results if: always() @@ -245,6 +250,7 @@ jobs: needs: [run-tests] runs-on: ubuntu-latest permissions: write-all + steps: - uses: actions/checkout@v4 @@ -267,6 +273,7 @@ jobs: - name: Consolidate robot test results working-directory: artifacts + continue-on-error: true run: | python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \ ./reports-windows-amd64/output.xml \ @@ -276,7 +283,10 @@ jobs: - name: Generate Summary report if: always() run: | - python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md + python ./test/lib/execution_summary.py artifacts \ + -o artifacts/collective_robot_results/output.xml \ + -r ./test/reference.md \ + -m summary_report.md - name: Print E2E Report if: always() diff --git a/test/data/build-asm/solution.csolution.yml b/test/data/build-asm/solution.csolution.yml index 2c8cfb32..754f63d4 100644 --- a/test/data/build-asm/solution.csolution.yml +++ b/test/data/build-asm/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/build-c/solution.csolution.yml b/test/data/build-c/solution.csolution.yml index 03925585..1090b1e7 100644 --- a/test/data/build-c/solution.csolution.yml +++ b/test/data/build-c/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/build-cpp/solution.csolution.yml b/test/data/build-cpp/solution.csolution.yml index b95b8dfc..f5d5826a 100644 --- a/test/data/build-cpp/solution.csolution.yml +++ b/test/data/build-cpp/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/include-define/solution.csolution.yml b/test/data/include-define/solution.csolution.yml index aa534637..748369db 100644 --- a/test/data/include-define/solution.csolution.yml +++ b/test/data/include-define/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/language-scope/solution.csolution.yml b/test/data/language-scope/solution.csolution.yml index 74492e45..172fc3c2 100644 --- a/test/data/language-scope/solution.csolution.yml +++ b/test/data/language-scope/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/linker-pre-processing/solution.csolution.yml b/test/data/linker-pre-processing/solution.csolution.yml index b2a7aa5b..f88e8e54 100644 --- a/test/data/linker-pre-processing/solution.csolution.yml +++ b/test/data/linker-pre-processing/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/pre-include/solution.csolution.yml b/test/data/pre-include/solution.csolution.yml index f078e696..7ddb6ccc 100644 --- a/test/data/pre-include/solution.csolution.yml +++ b/test/data/pre-include/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/data/whitespace/solution.csolution.yml b/test/data/whitespace/solution.csolution.yml index 03925585..1090b1e7 100644 --- a/test/data/whitespace/solution.csolution.yml +++ b/test/data/whitespace/solution.csolution.yml @@ -1,5 +1,4 @@ solution: - created-for: CMSIS-Toolbox@2.2.1 cdefault: packs: diff --git a/test/lib/execution_summary.py b/test/lib/execution_summary.py index d4a7f77e..4b9e99a1 100644 --- a/test/lib/execution_summary.py +++ b/test/lib/execution_summary.py @@ -13,6 +13,7 @@ from robot.api import ExecutionResult, ResultVisitor from robot.result.model import TestCase from robot.result.executionresult import Result +from reference_compare import * class ResultVisitorEx(ResultVisitor): def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str): @@ -113,20 +114,26 @@ def __write_test_section(self, file, test_dict, section_header, table_header): def main(): parser = argparse.ArgumentParser(description='Consolidate test summary report') parser.add_argument('test_env_files_path', type=str, help='Path to the test environment files') + parser.add_argument('-r', '--reference_file', type=str, help='Path to reference file') parser.add_argument('-o', '--output_file', type=str, nargs='?', default='output.xml', help='Path to output xml file') parser.add_argument('-m', '--markdown_file', type=str, nargs='?', default='summary_report.md', help='Path to consolidated summary markdown file') + args = parser.parse_args() test_env_files_path = args.test_env_files_path output_file = args.output_file markdown_file = args.markdown_file + reference_file = args.reference_file result = ExecutionResult(output_file) result.visit(ResultVisitorEx(test_env_files_path, output_file, markdown_file)) + print("MD: {}, REF: {}".format(markdown_file, reference_file)) + return compare_summary(markdown_file, reference_file) + if __name__ == '__main__': try: - main() + sys.exit(main()) except Exception as e: print(f'An error occurred: {e}', file=sys.stderr) sys.exit(1) diff --git a/test/lib/reference_compare.py b/test/lib/reference_compare.py new file mode 100644 index 00000000..5f931fb0 --- /dev/null +++ b/test/lib/reference_compare.py @@ -0,0 +1,56 @@ +import markdown +import pandas as pd +from io import StringIO + +class SummaryResult: + def __init__(self, passed, failed, skipped, total): + self.passed = passed + self.failed = failed + self.skipped = skipped + self.total = total + +class MarkdownReader: + def __init__(self, input_file_path:str): + self.input_file=input_file_path + self.html_content = "" + + def read(self): + # Read the Markdown file + with open(self.input_file, 'r') as file: + md_content = file.read() + + # Convert the Markdown to HTML + self.html_content = markdown.markdown(md_content) + + def extract_summary_table(self): + # Extract the tables from the HTML + tables = pd.read_html(StringIO(self.html_content)) + # The second table is the summary table + summary_table = tables[1] + # Extract the summary results + passed = int(summary_table.iloc[0, 0]) + failed = int(summary_table.iloc[0, 1]) + skipped = int(summary_table.iloc[0, 2]) + total = int(summary_table.iloc[0, 3]) + + return SummaryResult(passed, failed, skipped, total) + + +def compare_summary(markdownFile:str, referenceFile:str): + mdFileSummary = MarkdownReader(markdownFile).extract_summary_table() + refFileSummary = MarkdownReader(referenceFile).extract_summary_table() + # Compare the results + results_match = ( + mdFileSummary.passed == refFileSummary.passed and + mdFileSummary.failed == refFileSummary.failed and + mdFileSummary.skipped == refFileSummary.skipped and + mdFileSummary.total == refFileSummary.total + ) + + return results_match + +# print("Summary results match expected values:", results_match) +# print(f"Passed: {passed}, Expected: {expected_passed}") +# print(f"Failed: {failed}, Expected: {expected_failed}") +# print(f"Skipped: {skipped}, Expected: {expected_skipped}") +# print(f"Total: {total}, Expected: {expected_total}") diff --git a/test/reference.md b/test/reference.md new file mode 100644 index 00000000..3e3f4d6b --- /dev/null +++ b/test/reference.md @@ -0,0 +1,81 @@ +# Robot Framework Report + +## Test Environment + +|Name|Version| +|:---|:------| +|cbuild|2.4.0-43-gd715d2f| +|cpackget|2.1.3-1-g0aaeca9| +|csolution|2.4.0+p63-g2fdb3c85| +|cbuild2cmake|0.9.1-26-g1120292| +|cbuildgen|2.4.0+p62-g2fdb3c85| + +## Summary + +|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total| +|:----:|:----:|:-----:|:---:| +|39|15|0|53| + +## Passed Tests + +|Tag|Test|:clock1030: Duration|Suite| +|:---:|:---:|:---:|:---:| +|windows-amd64|Validate build-c Example|6.01 s|Local Example Tests| +|windows-amd64|Validate build-cpp Example|9.21 s|Local Example Tests| +|windows-amd64|Validate linker-pre-processing Example|6.39 s|Local Example Tests| +|windows-amd64|Validate pre-include Example|6.10 s|Local Example Tests| +|windows-amd64|Validate whitespace Example|6.04 s|Local Example Tests| +|windows-amd64|Validate trustzone Example|48.31 s|Local Example Tests| +|windows-amd64|Hello_B-U585I-IOT02A|44.34 s|Remote Example Tests| +|windows-amd64|Hello_FRDM-K32L3A6|27.11 s|Remote Example Tests| +|windows-amd64|Hello_IMXRT1050-EVKB|33.85 s|Remote Example Tests| +|windows-amd64|Hello_LPCXpresso55S69|29.09 s|Remote Example Tests| +|windows-amd64|Blinky_FRDM-K32L3A6|24.95 s|Remote Example Tests| +|windows-amd64|keil-studio-get-started|7.20 s|Remote Example Tests| +|windows-amd64|Hello_AVH|72.43 s|Remote Example Tests| +|linux-amd64|Validate build-c Example|1.59 s|Local Example Tests| +|linux-amd64|Validate build-cpp Example|2.45 s|Local Example Tests| +|linux-amd64|Validate linker-pre-processing Example|1.62 s|Local Example Tests| +|linux-amd64|Validate pre-include Example|1.69 s|Local Example Tests| +|linux-amd64|Validate whitespace Example|1.61 s|Local Example Tests| +|linux-amd64|Validate trustzone Example|7.59 s|Local Example Tests| +|linux-amd64|Hello_B-U585I-IOT02A|19.91 s|Remote Example Tests| +|linux-amd64|Hello_FRDM-K32L3A6|8.79 s|Remote Example Tests| +|linux-amd64|Hello_IMXRT1050-EVKB|11.89 s|Remote Example Tests| +|linux-amd64|Hello_LPCXpresso55S69|15.33 s|Remote Example Tests| +|linux-amd64|Blinky_FRDM-K32L3A6|9.20 s|Remote Example Tests| +|linux-amd64|keil-studio-get-started|3.63 s|Remote Example Tests| +|linux-amd64|Hello_AVH|28.00 s|Remote Example Tests| +|darwin-amd64|Validate build-c Example|4.44 s|Local Example Tests| +|darwin-amd64|Validate build-cpp Example|6.57 s|Local Example Tests| +|darwin-amd64|Validate linker-pre-processing Example|4.56 s|Local Example Tests| +|darwin-amd64|Validate pre-include Example|4.72 s|Local Example Tests| +|darwin-amd64|Validate whitespace Example|4.82 s|Local Example Tests| +|darwin-amd64|Validate trustzone Example|19.98 s|Local Example Tests| +|darwin-amd64|Hello_B-U585I-IOT02A|32.40 s|Remote Example Tests| +|darwin-amd64|Hello_FRDM-K32L3A6|14.36 s|Remote Example Tests| +|darwin-amd64|Hello_IMXRT1050-EVKB|15.43 s|Remote Example Tests| +|darwin-amd64|Hello_LPCXpresso55S69|15.10 s|Remote Example Tests| +|darwin-amd64|Blinky_FRDM-K32L3A6|13.49 s|Remote Example Tests| +|darwin-amd64|keil-studio-get-started|5.79 s|Remote Example Tests| +|darwin-amd64|Hello_AVH|47.37 s|Remote Example Tests| + +## Failed Tests + +|Tag|Test|Message|:clock1030: Duration|Suite| +|:---:|:---:|:---:|:---:|:---:| +|windows-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|10.03 s|Local Example Tests| +|windows-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|5.12 s|Local Example Tests| +|windows-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.51 s|Local Example Tests| +|windows-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|22.25 s|Remote Example Tests| +|windows-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|27.58 s|Remote Example Tests| +|linux-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.25 s|Local Example Tests| +|linux-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|1.42 s|Local Example Tests| +|linux-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|1.00 s|Local Example Tests| +|linux-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|10.52 s|Remote Example Tests| +|linux-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|11.91 s|Remote Example Tests| +|darwin-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|8.99 s|Local Example Tests| +|darwin-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|3.58 s|Local Example Tests| +|darwin-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|2.35 s|Local Example Tests| +|darwin-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|14.92 s|Remote Example Tests| +|darwin-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|21.39 s|Remote Example Tests| diff --git a/test/remote_example_tests.csv b/test/remote_example_tests.csv new file mode 100644 index 00000000..a6cb8d48 --- /dev/null +++ b/test/remote_example_tests.csv @@ -0,0 +1,10 @@ +*** Test Cases ***;${github_url};${expect};[Tags];[Documentation] +Hello_B-U585I-IOT02A;https://github.com/Arm-Examples/Hello_B-U585I-IOT02A;${0};; +Hello_FRDM-K32L3A6;https://github.com/Arm-Examples/Hello_FRDM-K32L3A6;${0};; +Hello_IMXRT1050-EVKB;https://github.com/Arm-Examples/Hello_IMXRT1050-EVKB;${0};; +Hello_LPCXpresso55S69;https://github.com/Arm-Examples/Hello_LPCXpresso55S69;${0};; +Blinky_FRDM-K32L3A6;https://github.com/Arm-Examples/Blinky_FRDM-K32L3A6;${0};; +Blinky_NUCLEO-G0B1RE;https://github.com/Arm-Examples/Blinky_NUCLEO-G0B1RE;${0};; +Blinky_NUCLEO-F756ZG;https://github.com/Arm-Examples/Blinky_NUCLEO-F756ZG;${0};; +keil-studio-get-started;https://github.com/Arm-Examples/keil-studio-get-started;${0};; +Hello_AVH;https://github.com/Arm-Examples/Hello_AVH;${0};; \ No newline at end of file diff --git a/test/remote_example_tests.robot b/test/remote_example_tests.robot index 20093fa6..8cd0fea6 100644 --- a/test/remote_example_tests.robot +++ b/test/remote_example_tests.robot @@ -5,39 +5,28 @@ Suite Teardown Global Teardown Library lib${/}utils.py Library String Library Collections -Library lib${/}elf_compare.py +Library lib${/}elf_compare.py +Library DataDriver Resource resources${/}global.resource Resource resources${/}utils.resource Test Template Test github examples - -*** Variables *** -${csolution-examples} https://github.com/Open-CMSIS-Pack/csolution-examples -${Hello_B-U585I-IOT02A} https://github.com/Arm-Examples/Hello_B-U585I-IOT02A -${Hello_FRDM-K32L3A6} https://github.com/Arm-Examples/Hello_FRDM-K32L3A6 - +# *** Variables *** *** Test Cases *** -# Test Csolution example -# ${csolution-examples} ${Pass} - -# Test Hello_1 example -# ${Hello_B-U585I-IOT02A} ${Pass} - -# Test Hello_2 example -# ${Hello_FRDM-K32L3A6} ${Pass} +Test remote example ${github_url} and expect ${expect} Default UserData *** Keywords *** Test github examples - [Arguments] ${github_url} ${expect} - ${dest_dir}= Evaluate "${github_url}".split('/')[-1] - ${dest_dir}= Set Variable ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir} - Checkout GitHub Repository ${github_url} ${dest_dir} + [Arguments] ${github_url} ${expect} + ${dest_dir}= Get Destination Path ${github_url} + # ${dest_dir}= Evaluate "${github_url}".split('/')[-1] + # ${dest_dir}= Set Variable ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir} + Checkout GitHub Repository ${github_url} ${dest_dir} ${files} Glob Files In Directory ${dest_dir} *.csolution.* ${True} FOR ${file} IN @{files} - ${example_name}= Get Parent Directory Name ${file} - Run Keyword If '${example_name}' != 'CubeMX' - ... Run Csolution Project ${file} ${expect} + # ${example_name}= Get Parent Directory Name ${file} + Run Csolution Project ${file} ${expect} END Run Csolution Project diff --git a/test/requirements.txt b/test/requirements.txt index e7f6d4ac..44c4084b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,6 @@ # Requirements needed robotframework >= 7.0 +robotframework-datadriver >= 1.11.2 +markdown >= 3.6 +pandas >= 2.2.2 +lxml >= 5.0.0 diff --git a/test/resources/utils.resource b/test/resources/utils.resource index c01fbe43..cbc8306f 100644 --- a/test/resources/utils.resource +++ b/test/resources/utils.resource @@ -72,7 +72,7 @@ Run Project with cbuildgen [Documentation] Run the csolution project with cbuildgen [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} ${parent_path}= Get Parent Directory Path ${input_file} - ${ex_args}= Append Additional Arguments ${args} --output ${parent_path}${/}${Out_Dir} + ${ex_args}= Append Additional Arguments ${args} --output ${parent_path}${/}${Out_Dir} --cbuildgen ${ret_code}= Run cbuild ${input_file} ${ex_args} Should Be Equal ${ret_code} ${expect} msg=Unexpected status returned by cbuildgen execution @@ -98,3 +98,10 @@ Get Output File Path [Documentation] Find path to output files ${output_path}= Get Variable Value ${OUTPUT_FILE} RETURN ${output_path} + +Get Destination Path + [Documentation] Get destination directory path from url + [Arguments] ${github_url} + ${dest_dir}= Evaluate "${github_url}".split('/')[-1] + ${dest_dir}= Set Variable ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir} + RETURN ${dest_dir}