diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 2b9791f..4374dbd 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -71,9 +71,11 @@ jobs: - name: Run Test shell: bash + continue-on-error: true run: | robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \ - --settag ${{ matrix.target }}-${{ matrix.arch }} \ + --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \ + --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \ --name ${{ matrix.target }}-${{ matrix.arch }} \ ./test @@ -111,6 +113,7 @@ jobs: - name: Consolidate robot test results working-directory: artifacts + continue-on-error: true run: | python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \ ./reports-windows-amd64/output.xml \ @@ -120,7 +123,10 @@ jobs: - name: Generate Summary report if: always() run: | - python ./test/lib/execution_summary.py artifacts -o artifacts/collective_robot_results/output.xml -m summary_report.md + python ./test/lib/execution_summary.py artifacts \ + -r ./test/reference.md \ + -o artifacts/collective_robot_results/output.xml \ + -m summary_report.md - name: Print E2E Report if: always() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a229452..74feb97 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -90,8 +90,8 @@ jobs: with: go-version-file: cpackget/go.mod cache-dependency-path: | - cpackget/go.mod - cpackget/go.sum + **/go.mod + **/go.sum - name: Build cpackget executable working-directory: cpackget @@ -112,8 +112,8 @@ jobs: with: go-version-file: cbuild2cmake/go.mod cache-dependency-path: | - cbuild2cmake/go.mod - cbuild2cmake/go.sum + **/go.mod + **/go.sum - name: Build cbuild2cmake executable working-directory: cbuild2cmake @@ -236,7 +236,7 @@ jobs: --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \ --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \ --name ${{ matrix.target }}-${{ matrix.arch }} \ - ./test/pack_example_tests.robot + ./test - name: Archieve test results if: always() diff --git a/test/lib/elf_compare.py b/test/lib/elf_compare.py index 67c77b8..d18e4d1 100644 --- a/test/lib/elf_compare.py +++ b/test/lib/elf_compare.py @@ -2,11 +2,12 @@ import subprocess from robot.api import logger from robot.utils import asserts +from pathlib import Path def compare_elf_information(input_file, cbuildgen_out_dir, cbuild2cmake_out_dir): - logger.debug('Input file: %s' % input_file) - logger.debug('Cbuildgen out dir: %s' % cbuildgen_out_dir) - logger.debug('Cbuild2cmake out dir: %s' % cbuild2cmake_out_dir) + logger.info('Input file: %s' % input_file) + logger.info('Cbuildgen out dir: %s' % cbuildgen_out_dir) + logger.info('Cbuild2cmake out dir: %s' % cbuild2cmake_out_dir) return CompareELF(input_file, cbuildgen_out_dir, cbuild2cmake_out_dir).compare_elf_files() class Context: @@ -34,7 +35,7 @@ class Utils: def run_command(exe_path, args): try: command = exe_path + ' ' + ' '.join(args) - logger.info(f"Running Command: {' '.join(command)}") + logger.info('Running Command: %s' % command) processOut = subprocess.run(command, check=True, shell=True, universal_newlines=True, capture_output=True, timeout=300) return True, processOut.stdout except subprocess.CalledProcessError as e: @@ -66,6 +67,18 @@ def compare_elf_files(self): if path != '': cbuildgen_elf_file = os.path.join(self.cbuildgen_out_dir, path) cbuild2cmake_elf_file = os.path.join(self.cbuild2cmake_out_dir, path) + cbuildgenPath = Path(cbuildgen_elf_file) + cbuild2cmakePath = Path(cbuild2cmake_elf_file) + if cbuildgenPath.exists(): + logger.info('Path Exist: %s' % cbuildgenPath) + else: + logger.info('Path doesnot Exist: %s' % cbuildgenPath) + + if cbuild2cmakePath.exists(): + logger.info('Path Exist: %s' % cbuild2cmakePath) + else: + logger.info('Path doesnot Exist: %s' % cbuild2cmakePath) + res1, stdout1 = self.get_elf_info(cbuildgen_elf_file) res2, stdout2 = self.get_elf_info(cbuild2cmake_elf_file) logger.info('Object Info %s' % stdout1) diff --git a/test/lib/reference_compare.py b/test/lib/reference_compare.py index 6ec03c1..40e1a75 100644 --- a/test/lib/reference_compare.py +++ b/test/lib/reference_compare.py @@ -51,9 +51,36 @@ def extract_summary(self): print(f"No summary found in file: {self.input_file}", file=sys.stderr) return None + def extract_passed_tests(self): + self.read() + + passed_tests_pattern = re.compile( + r"## Passed Tests\n\n\|Tag\|Test\|:clock1030: Duration\|Suite\|\n\|:---:\|:---:\|:---:\|:---:\|\n((\|.+\|\n)+)" + ) + + # Find the failed tests section + passed_tests_section = passed_tests_pattern.search(self.content) + + # Extract the passed tests table if it exists + if not passed_tests_section: + return [] + + passed_tests_table = passed_tests_section.group(1) + + # Regular expression to extract tag and test name from each row + # test_pattern = re.compile(r"\|(.+?)\|(.+?)\|.+?\|.+?\|.+?\|") + test_pattern = re.compile(r"(\|.+?\|.+?\|).+?\|.+?\|") + + # Find all matches + passed_tests = test_pattern.findall(passed_tests_table) + return passed_tests + def compare_summaries(markdown_file: str, reference_file: str): - md_summary = MarkdownReader(markdown_file).extract_summary() - ref_summary = MarkdownReader(reference_file).extract_summary() + md_file = MarkdownReader(markdown_file) + ref_file = MarkdownReader(reference_file) + + md_summary = md_file.extract_summary() + ref_summary = ref_file.extract_summary() if md_summary is None or ref_summary is None: print("Comparison could not be performed due to missing summary data.", file=sys.stderr) @@ -63,8 +90,20 @@ def compare_summaries(markdown_file: str, reference_file: str): print(f"error: Test results do not match the reference\n" f" Expected: Passed: {ref_summary.passed}, Failed: {ref_summary.failed}, Skipped: {ref_summary.skipped}, Total: {ref_summary.total}\n" f" Actual: Passed: {md_summary.passed}, Failed: {md_summary.failed}, Skipped: {md_summary.skipped}, Total: {md_summary.total}") - return 1 # failure + md_passed_tests = md_file.extract_passed_tests() + ref_passed_tests = ref_file.extract_passed_tests() + + for testInfo in md_passed_tests: + if testInfo in ref_passed_tests: + ref_passed_tests.remove(testInfo) + + if len(ref_passed_tests) > 0: + print(f"error: Regression detected. The following tests were expected to execute and pass:") + for reg_test in ref_passed_tests: + print(f" {reg_test}") + return 1 # failure + return 0 # success # def main(): @@ -74,9 +113,9 @@ def compare_summaries(markdown_file: str, reference_file: str): # args = parser.parse_args() # if compare_summaries(args.markdown_file, args.reference_file): -# print("Summaries are equal.") -# else: # print("Summaries are not equal.") +# else: +# print("Summaries are equal.") # if __name__ == '__main__': # try: diff --git a/test/lib/utils.py b/test/lib/utils.py index 8432c52..7798f36 100644 --- a/test/lib/utils.py +++ b/test/lib/utils.py @@ -1,6 +1,5 @@ import glob from pathlib import Path -# import stat import subprocess import re # import os @@ -14,7 +13,9 @@ def get_parent_directory_name(file_path:str): return parent_dir.name def get_parent_directory_path(file_path:str): - return Path(file_path).parent.absolute() + absPath = Path(file_path).parent.absolute() + path = str(absPath).replace('\\', '/') + return path def write_test_environment(test_env_file:str): toolList = ["cbuild", "cpackget", "csolution", "cbuild2cmake", "cbuildgen"] diff --git a/test/local_example_tests.robot b/test/local_example_tests.robot index 3a048a9..dcbf095 100644 --- a/test/local_example_tests.robot +++ b/test/local_example_tests.robot @@ -6,12 +6,12 @@ Resource resources${/}global.resource Resource resources${/}utils.resource Library String Library Collections -Library lib${/}elf_compare.py Library lib${/}utils.py -Test Template Run CSolution Project +Library lib${/}elf_compare.py +Test Template Build Local CSolution Example *** Variables *** -# The directory name of the example to be built +# The directory name of the examples to be built ${build-asm} build-asm ${build-c} build-c ${build-cpp} build-cpp @@ -55,13 +55,7 @@ Validate trustzone Example ${TEST_DATA_DIR}${/}${trustzone}${/}solution.csolution.yml ${Pass} *** Keywords *** -Run Csolution Project - [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} - Run Project With cbuildgen ${input_file} ${expect} ${args} - Run Project with cbuild2cmake ${input_file} ${expect} ${args} - ${parent_path}= Get Parent Directory Path ${input_file} - ${result}= Run Keyword And Return - ... Compare Elf Information ${input_file} - ... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir} - ... ${parent_path}${/}${Default_Out_Dir} - Should Be True ${result} +Build Local CSolution Example + [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} + ${result}= Build CSolution Example ${input_file} ${expect} ${args} + Should Be True ${result} diff --git a/test/pack_example_tests.csv b/test/pack_example_tests.csv index 96a440c..074bd2e 100644 --- a/test/pack_example_tests.csv +++ b/test/pack_example_tests.csv @@ -1,4 +1,5 @@ *** Test Cases ***;${pack_id};${expect};[Tags];[Documentation] +NXP::FRDM-K32L3A6_BSP.17.0.0;NXP::FRDM-K32L3A6_BSP@17.0.0;${0};; NXP::LPCXpresso55S69_BSP.18.0.0;NXP::LPCXpresso55S69_BSP@18.0.0;${0};; NXP::LPCXpresso55S69_USB_Examples.1.0.0;NXP::LPCXpresso55S69_USB_Examples@1.0.0;${0};; NXP::FRDM-K32L3A6_BSP.18.0.0;NXP::FRDM-K32L3A6_BSP@18.0.0;${0};; diff --git a/test/pack_example_tests.robot b/test/pack_example_tests.robot index ea78969..fa3c449 100644 --- a/test/pack_example_tests.robot +++ b/test/pack_example_tests.robot @@ -1,14 +1,14 @@ *** Settings *** -Documentation Tests to compile & execute remote csolution examples +Documentation Tests to build pack csolution examples Suite Setup Global Setup Suite Teardown Global Teardown -Library lib${/}utils.py Library BuiltIn -Library String Library Collections -Library lib${/}elf_compare.py Library DataDriver Library OperatingSystem +Library String +Library lib${/}elf_compare.py +Library lib${/}utils.py Resource resources${/}global.resource Resource resources${/}utils.resource Test Template Test packs examples @@ -23,7 +23,7 @@ Test pack example ${pack_id} and expect ${expect} Default UserData *** Keywords *** Test packs examples [Arguments] ${pack_id} ${expect} - ${pack_root_dir}= Join And Normalize Path ${TEST_DATA_DIR} ${Local_Pack_Root_Dir} + ${pack_root_dir}= Join Paths ${TEST_DATA_DIR} ${Local_Pack_Root_Dir} Create Directory ${pack_root_dir} Cpackget Init ${pack_root_dir} Cpackget Install Pack ${pack_id} ${pack_root_dir} @@ -36,27 +36,22 @@ Test packs examples Run Csolution Project [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} - ${contexts}= Get Contexts From Project ${input_file} ${expect} ${args} - ${filcontexts}= Convert And Filter Contexts ${contexts} - Log Many ${args} - Log Many ${filcontexts} - # Append To List ${args} ${filcontexts} - @{MERGED_LIST} Create List @{args} @{filcontexts} - Log Many ${MERGED_LIST} - Run Project With cbuildgen ${input_file} ${expect} ${MERGED_LIST} - Run Project with cbuild2cmake ${input_file} ${expect} ${args} - ${parent_path}= Get Parent Directory Path ${input_file} - ${result}= Run Keyword And Return - ... Compare Elf Information ${input_file} - ... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir} - ... ${parent_path}${/}${Default_Out_Dir} - Should Be True ${result} + ${contexts}= Get Contexts From Project ${input_file} ${expect} ${args} + ${filcontexts}= Convert And Filter Contexts ${contexts} + @{MERGED_LIST} Create List @{args} @{filcontexts} + Build Example With cbuildgen ${input_file} ${expect} ${MERGED_LIST} + Build Example with cbuild2cmake ${input_file} ${expect} ${MERGED_LIST} + ${parent_path}= Get Parent Directory Path ${input_file} + ${result}= Run Keyword And Return + ... Compare Elf Information ${input_file} + ... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir} + ... ${parent_path}${/}${Default_Out_Dir} + Should Be True ${result} Convert And Filter Contexts [Documentation] Example test to convert a string to an array, filter it, and join it [Arguments] ${all_contexts} @{lines}= Split String ${all_contexts} \n - Log ${lines} # Filter out items containing '+iar' @{filtered_lines}= Create List @@ -65,17 +60,6 @@ Convert And Filter Contexts ${result}= Run Keyword And Return Status Should Contain ${line} iar Run Keyword If ${result}==False Append To List ${filtered_lines} ${line} END - Log Many ${filtered_lines} - - # # Prefix remaining items with '-c ' and join them - # ${context_string}= Set Variable ${EMPTY} - # FOR ${line} IN @{filtered_lines} - # Append To List - # ${context_string}= Evaluate "-c ${line} ${context_string}" - # END - - # Log Many ${context_string} - # RETURN ${context_string} # Prefix remaining items with '-c ' and join them @{context_list_args}= Create List @@ -83,5 +67,4 @@ Convert And Filter Contexts Append To List ${context_list_args} -c ${line} END - Log Many ${context_list_args} RETURN ${context_list_args} diff --git a/test/reference.md b/test/reference.md index 3e3f4d6..c1176bc 100644 --- a/test/reference.md +++ b/test/reference.md @@ -1,81 +1,86 @@ # Robot Framework Report -## Test Environment - -|Name|Version| -|:---|:------| -|cbuild|2.4.0-43-gd715d2f| -|cpackget|2.1.3-1-g0aaeca9| -|csolution|2.4.0+p63-g2fdb3c85| -|cbuild2cmake|0.9.1-26-g1120292| -|cbuildgen|2.4.0+p62-g2fdb3c85| - ## Summary |:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total| |:----:|:----:|:-----:|:---:| -|39|15|0|53| +|39|30|0|69| ## Passed Tests |Tag|Test|:clock1030: Duration|Suite| |:---:|:---:|:---:|:---:| -|windows-amd64|Validate build-c Example|6.01 s|Local Example Tests| -|windows-amd64|Validate build-cpp Example|9.21 s|Local Example Tests| -|windows-amd64|Validate linker-pre-processing Example|6.39 s|Local Example Tests| -|windows-amd64|Validate pre-include Example|6.10 s|Local Example Tests| -|windows-amd64|Validate whitespace Example|6.04 s|Local Example Tests| -|windows-amd64|Validate trustzone Example|48.31 s|Local Example Tests| -|windows-amd64|Hello_B-U585I-IOT02A|44.34 s|Remote Example Tests| -|windows-amd64|Hello_FRDM-K32L3A6|27.11 s|Remote Example Tests| -|windows-amd64|Hello_IMXRT1050-EVKB|33.85 s|Remote Example Tests| -|windows-amd64|Hello_LPCXpresso55S69|29.09 s|Remote Example Tests| -|windows-amd64|Blinky_FRDM-K32L3A6|24.95 s|Remote Example Tests| -|windows-amd64|keil-studio-get-started|7.20 s|Remote Example Tests| -|windows-amd64|Hello_AVH|72.43 s|Remote Example Tests| -|linux-amd64|Validate build-c Example|1.59 s|Local Example Tests| -|linux-amd64|Validate build-cpp Example|2.45 s|Local Example Tests| -|linux-amd64|Validate linker-pre-processing Example|1.62 s|Local Example Tests| -|linux-amd64|Validate pre-include Example|1.69 s|Local Example Tests| -|linux-amd64|Validate whitespace Example|1.61 s|Local Example Tests| -|linux-amd64|Validate trustzone Example|7.59 s|Local Example Tests| -|linux-amd64|Hello_B-U585I-IOT02A|19.91 s|Remote Example Tests| -|linux-amd64|Hello_FRDM-K32L3A6|8.79 s|Remote Example Tests| -|linux-amd64|Hello_IMXRT1050-EVKB|11.89 s|Remote Example Tests| -|linux-amd64|Hello_LPCXpresso55S69|15.33 s|Remote Example Tests| -|linux-amd64|Blinky_FRDM-K32L3A6|9.20 s|Remote Example Tests| -|linux-amd64|keil-studio-get-started|3.63 s|Remote Example Tests| -|linux-amd64|Hello_AVH|28.00 s|Remote Example Tests| -|darwin-amd64|Validate build-c Example|4.44 s|Local Example Tests| -|darwin-amd64|Validate build-cpp Example|6.57 s|Local Example Tests| -|darwin-amd64|Validate linker-pre-processing Example|4.56 s|Local Example Tests| -|darwin-amd64|Validate pre-include Example|4.72 s|Local Example Tests| -|darwin-amd64|Validate whitespace Example|4.82 s|Local Example Tests| -|darwin-amd64|Validate trustzone Example|19.98 s|Local Example Tests| -|darwin-amd64|Hello_B-U585I-IOT02A|32.40 s|Remote Example Tests| -|darwin-amd64|Hello_FRDM-K32L3A6|14.36 s|Remote Example Tests| -|darwin-amd64|Hello_IMXRT1050-EVKB|15.43 s|Remote Example Tests| -|darwin-amd64|Hello_LPCXpresso55S69|15.10 s|Remote Example Tests| -|darwin-amd64|Blinky_FRDM-K32L3A6|13.49 s|Remote Example Tests| -|darwin-amd64|keil-studio-get-started|5.79 s|Remote Example Tests| -|darwin-amd64|Hello_AVH|47.37 s|Remote Example Tests| +|windows-amd64|Validate build-c Example|6.16 s|Local Example Tests| +|windows-amd64|Validate build-cpp Example|9.34 s|Local Example Tests| +|windows-amd64|Validate linker-pre-processing Example|6.21 s|Local Example Tests| +|windows-amd64|Validate pre-include Example|6.53 s|Local Example Tests| +|windows-amd64|Validate whitespace Example|6.19 s|Local Example Tests| +|windows-amd64|Validate trustzone Example|43.75 s|Local Example Tests| +|windows-amd64|Hello_B-U585I-IOT02A|44.89 s|Remote Example Tests| +|windows-amd64|Hello_FRDM-K32L3A6|26.63 s|Remote Example Tests| +|windows-amd64|Hello_IMXRT1050-EVKB|40.38 s|Remote Example Tests| +|windows-amd64|Hello_LPCXpresso55S69|34.32 s|Remote Example Tests| +|windows-amd64|Blinky_FRDM-K32L3A6|31.87 s|Remote Example Tests| +|windows-amd64|keil-studio-get-started|7.74 s|Remote Example Tests| +|windows-amd64|Hello_AVH|66.01 s|Remote Example Tests| +|linux-amd64|Validate build-c Example|1.56 s|Local Example Tests| +|linux-amd64|Validate build-cpp Example|2.36 s|Local Example Tests| +|linux-amd64|Validate linker-pre-processing Example|1.58 s|Local Example Tests| +|linux-amd64|Validate pre-include Example|1.67 s|Local Example Tests| +|linux-amd64|Validate whitespace Example|1.56 s|Local Example Tests| +|linux-amd64|Validate trustzone Example|7.31 s|Local Example Tests| +|linux-amd64|Hello_B-U585I-IOT02A|18.62 s|Remote Example Tests| +|linux-amd64|Hello_FRDM-K32L3A6|7.96 s|Remote Example Tests| +|linux-amd64|Hello_IMXRT1050-EVKB|10.29 s|Remote Example Tests| +|linux-amd64|Hello_LPCXpresso55S69|9.72 s|Remote Example Tests| +|linux-amd64|Blinky_FRDM-K32L3A6|9.19 s|Remote Example Tests| +|linux-amd64|keil-studio-get-started|3.48 s|Remote Example Tests| +|linux-amd64|Hello_AVH|21.13 s|Remote Example Tests| +|darwin-amd64|Validate build-c Example|6.21 s|Local Example Tests| +|darwin-amd64|Validate build-cpp Example|7.63 s|Local Example Tests| +|darwin-amd64|Validate linker-pre-processing Example|5.42 s|Local Example Tests| +|darwin-amd64|Validate pre-include Example|5.73 s|Local Example Tests| +|darwin-amd64|Validate whitespace Example|6.03 s|Local Example Tests| +|darwin-amd64|Validate trustzone Example|25.30 s|Local Example Tests| +|darwin-amd64|Hello_B-U585I-IOT02A|37.63 s|Remote Example Tests| +|darwin-amd64|Hello_FRDM-K32L3A6|16.78 s|Remote Example Tests| +|darwin-amd64|Hello_IMXRT1050-EVKB|17.91 s|Remote Example Tests| +|darwin-amd64|Hello_LPCXpresso55S69|16.27 s|Remote Example Tests| +|darwin-amd64|Blinky_FRDM-K32L3A6|19.12 s|Remote Example Tests| +|darwin-amd64|keil-studio-get-started|11.36 s|Remote Example Tests| +|darwin-amd64|Hello_AVH|57.27 s|Remote Example Tests| ## Failed Tests |Tag|Test|Message|:clock1030: Duration|Suite| |:---:|:---:|:---:|:---:|:---:| -|windows-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|10.03 s|Local Example Tests| -|windows-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|5.12 s|Local Example Tests| -|windows-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.51 s|Local Example Tests| -|windows-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|22.25 s|Remote Example Tests| -|windows-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|27.58 s|Remote Example Tests| -|linux-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|3.25 s|Local Example Tests| -|linux-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|1.42 s|Local Example Tests| -|linux-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|1.00 s|Local Example Tests| -|linux-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|10.52 s|Remote Example Tests| -|linux-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|11.91 s|Remote Example Tests| -|darwin-amd64|Validate build-asm Example|Unexpected status returned by cbuildgen execution: 1 != 0|8.99 s|Local Example Tests| -|darwin-amd64|Validate include-define Example|Unexpected status returned by cbuild2cmake execution: 1 != 0|3.58 s|Local Example Tests| -|darwin-amd64|Validate language-scope Example|Unexpected status returned by cbuildgen execution: 1 != 0|2.35 s|Local Example Tests| -|darwin-amd64|Blinky_NUCLEO-G0B1RE|Unexpected status returned by cbuild2cmake execution: 1 != 0|14.92 s|Remote Example Tests| -|darwin-amd64|Blinky_NUCLEO-F756ZG|Unexpected status returned by cbuild2cmake execution: 1 != 0|21.39 s|Remote Example Tests| +|windows-amd64|Validate build-asm Example|build status doesn't match: PASS != FAIL|20.27 s|Local Example Tests| +|windows-amd64|Validate include-define Example|build status doesn't match: FAIL != PASS|5.27 s|Local Example Tests| +|windows-amd64|Validate language-scope Example|build status doesn't match: PASS != FAIL|8.75 s|Local Example Tests| +|windows-amd64|NXP::FRDM-K32L3A6_BSP.17.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|23.57 s|Pack Example Tests| +|windows-amd64|NXP::LPCXpresso55S69_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|12.32 s|Pack Example Tests| +|windows-amd64|NXP::LPCXpresso55S69_USB_Examples.1.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|30.55 s|Pack Example Tests| +|windows-amd64|NXP::FRDM-K32L3A6_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|8.08 s|Pack Example Tests| +|windows-amd64|NXP::MIMXRT1060-EVKB_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|16.00 s|Pack Example Tests| +|windows-amd64|Blinky_NUCLEO-G0B1RE|build status doesn't match: FAIL != PASS|42.22 s|Remote Example Tests| +|windows-amd64|Blinky_NUCLEO-F756ZG|build status doesn't match: FAIL != PASS|44.33 s|Remote Example Tests| +|linux-amd64|Validate build-asm Example|build status doesn't match: PASS != FAIL|4.49 s|Local Example Tests| +|linux-amd64|Validate include-define Example|build status doesn't match: FAIL != PASS|1.36 s|Local Example Tests| +|linux-amd64|Validate language-scope Example|build status doesn't match: PASS != FAIL|2.21 s|Local Example Tests| +|linux-amd64|NXP::FRDM-K32L3A6_BSP.17.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|12.43 s|Pack Example Tests| +|linux-amd64|NXP::LPCXpresso55S69_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|7.63 s|Pack Example Tests| +|linux-amd64|NXP::LPCXpresso55S69_USB_Examples.1.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|15.19 s|Pack Example Tests| +|linux-amd64|NXP::FRDM-K32L3A6_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|5.31 s|Pack Example Tests| +|linux-amd64|NXP::MIMXRT1060-EVKB_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|9.13 s|Pack Example Tests| +|linux-amd64|Blinky_NUCLEO-G0B1RE|build status doesn't match: FAIL != PASS|10.70 s|Remote Example Tests| +|linux-amd64|Blinky_NUCLEO-F756ZG|build status doesn't match: FAIL != PASS|12.31 s|Remote Example Tests| +|darwin-amd64|Validate build-asm Example|build status doesn't match: PASS != FAIL|11.88 s|Local Example Tests| +|darwin-amd64|Validate include-define Example|build status doesn't match: FAIL != PASS|4.25 s|Local Example Tests| +|darwin-amd64|Validate language-scope Example|build status doesn't match: PASS != FAIL|7.10 s|Local Example Tests| +|darwin-amd64|NXP::FRDM-K32L3A6_BSP.17.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|31.85 s|Pack Example Tests| +|darwin-amd64|NXP::LPCXpresso55S69_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|19.25 s|Pack Example Tests| +|darwin-amd64|NXP::LPCXpresso55S69_USB_Examples.1.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|44.59 s|Pack Example Tests| +|darwin-amd64|NXP::FRDM-K32L3A6_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|13.90 s|Pack Example Tests| +|darwin-amd64|NXP::MIMXRT1060-EVKB_BSP.18.0.0|Unexpected status returned by cbuildgen execution: 1 != 0|25.17 s|Pack Example Tests| +|darwin-amd64|Blinky_NUCLEO-G0B1RE|build status doesn't match: FAIL != PASS|27.46 s|Remote Example Tests| +|darwin-amd64|Blinky_NUCLEO-F756ZG|build status doesn't match: FAIL != PASS|40.96 s|Remote Example Tests| \ No newline at end of file diff --git a/test/remote_example_tests.robot b/test/remote_example_tests.robot index e94b63c..1874ae0 100644 --- a/test/remote_example_tests.robot +++ b/test/remote_example_tests.robot @@ -2,11 +2,10 @@ Documentation Tests to compile & execute remote csolution examples Suite Setup Global Setup Suite Teardown Global Teardown -Library lib${/}utils.py -Library String -Library Collections -Library lib${/}elf_compare.py Library DataDriver +Library Collections +Library String +Library lib${/}utils.py Resource resources${/}global.resource Resource resources${/}utils.resource Test Template Test github examples @@ -17,24 +16,15 @@ Test remote example ${github_url} and expect ${expect} Default UserData *** Keywords *** Test github examples - [Arguments] ${github_url} ${expect} + [Arguments] ${github_url} ${expect} ${dest_dir}= Get Destination Path ${github_url} - # ${dest_dir}= Evaluate "${github_url}".split('/')[-1] - # ${dest_dir}= Set Variable ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir} - Checkout GitHub Repository ${github_url} ${dest_dir} - ${files} Glob Files In Directory ${dest_dir} *.csolution.* ${True} + Checkout GitHub Repository ${github_url} ${dest_dir} + ${files} Glob Files In Directory ${dest_dir} *.csolution.* ${True} FOR ${file} IN @{files} - # ${example_name}= Get Parent Directory Name ${file} - Run Csolution Project ${file} ${expect} + Build Remote CSolution Example ${file} ${expect} END -Run Csolution Project - [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} - Run Project With cbuildgen ${input_file} ${expect} ${args} - Run Project with cbuild2cmake ${input_file} ${expect} ${args} - ${parent_path}= Get Parent Directory Path ${input_file} - ${result}= Run Keyword And Return - ... Compare Elf Information ${input_file} - ... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir} - ... ${parent_path}${/}${Default_Out_Dir} - Should Be True ${result} +Build Remote CSolution Example + [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} + ${result}= Build CSolution Example ${input_file} ${expect} ${args} + Should Be True ${result} diff --git a/test/resources/utils.resource b/test/resources/utils.resource index 3c6eecf..9e01b3c 100644 --- a/test/resources/utils.resource +++ b/test/resources/utils.resource @@ -2,88 +2,59 @@ Documentation A collection of commonly used keywords across multiple test suites Library BuiltIn Library Collections -Library Process -Library String Library OperatingSystem +Library Process +Library String +Library ..${/}lib${/}elf_compare.py Library ..${/}lib${/}utils.py Resource global.resource *** Variables *** ${TEST_ENV_FILE} test_env.md -${RESULT} ${EMPTY} *** Keywords *** Global Setup - ${parent_dir}= Join And Normalize Path ${CURDIR} .. - ${src_dir}= Join And Normalize Path ${parent_dir} ${Data} + # Set the global variable for test data directory + ${parent_dir}= Join Paths ${CURDIR} .. + ${src_dir}= Join Paths ${parent_dir} ${Data} ${dest_dir}= Get Test Data Directory - Set Global Variable ${TEST_DATA_DIR} ${dest_dir} - Copy Directory ${src_dir} ${dest_dir} - ${output_file_path} Get Output File Path + Set Global Variable ${TEST_DATA_DIR} ${dest_dir} + # Copy test data under build tree + Copy Directory ${src_dir} ${dest_dir} + # Generate test_env.md file under output directory + ${output_file_path}= Get Output File Path ${parent_path}= Get Parent Directory Path ${output_file_path} - ${test_env_file}= Join And Normalize Path ${parent_path} ${TEST_ENV_FILE} + ${test_env_file}= Join Paths ${parent_path} ${TEST_ENV_FILE} Write Test Environment ${test_env_file} Global Teardown + # Remove test data after test execution Remove Directory with Content ${TEST_DATA_DIR} Get Test Data Directory [Documentation] Retrieve the directory path for test input data - ${parent_dir}= Join And Normalize Path ${CURDIR} .. - ${test_data_dir}= Join And Normalize Path ${parent_dir} ${Build} + ${parent_dir}= Join Paths ${CURDIR} .. + ${test_data_dir}= Join Paths ${parent_dir} ${Build} RETURN ${test_data_dir} -Run Program1 - [Documentation] Run specified executable with arguments - [Arguments] ${exe_path} ${command} - ${result} Run Process ${exe_path} ${command} - ... shell=True stdout=${CURDIR}/stdout.txt - ${ret_code}= Set Variable If ${result.rc} == ${0} ${result.rc} ${1} - Log Output String ${result.stdout} - RETURN ${ret_code} - Run Program [Documentation] Run specified executable with arguments - [Arguments] ${exe_path} ${command} ${input_File} @{args} - ${result} Run Process ${exe_path} ${command} ${input_File} @{args} - ... shell=True stdout=${CURDIR}/stdout.txt - ${ret_code}= Set Variable If ${result.rc} == ${0} ${result.rc} ${1} - Log Output String ${result.stdout} - RETURN - -Run Program2 - [Documentation] Run specified executable with arguments - [Arguments] ${exe_path} ${input_File} @{args} - Log Many ${exe_path} - Log Many ${input_file} - Log Many ${args} - ${result} Run Process ${exe_path} ${input_File} @{args} - ... shell=True stdout=${CURDIR}/stdout.txt - ${ret_code}= Set Variable If ${result.rc} == ${0} ${result.rc} ${1} + [Arguments] ${exe_path} ${input_file} @{args} ${command}=${EMPTY} + ${is_command_empty}= Evaluate '${command}' == '${EMPTY}' + ${result}= Run Keyword If ${is_command_empty} Run Process ${exe_path} ${input_file} @{args} shell=True stdout=${CURDIR}/stdout.txt + ... ELSE Run Process ${exe_path} ${command} ${input_file} @{args} shell=True stdout=${CURDIR}/stdout.txt + ${ret_code}= Set Variable If ${result.rc} == 0 ${result.rc} ${1} Log Output String ${result.stdout} - RETURN - -# Run cbuild -# [Documentation] Run cbuild with specified arguments -# [Arguments] ${cmdline_args}=@{EMPTY} -# Append To List ${cmdline_args} -p -r --update-rte -# Log Many ${cmdline_args} -# ${str_args}= Convert List To Single String ${cmdline_args} -# Log Many ${str_args} -# ${ret_code}= Run Program cbuild ${str_args} -# RETURN ${ret_code} + RETURN ${ret_code} Run cbuild - [Documentation] Run cbuild with specified arguments + [Documentation] Execute cbuild command with specified arguments [Arguments] ${input_file} ${command} ${args}=@{EMPTY} Append To List ${args} -p -r --update-rte - Log Many ${command} - Log Many ${input_file} - Log Many ${args} - ${ret_code}= Run Keyword If '${command}' == '${EMPTY}' Run Program2 cbuild ${input_file} @{args} - ... ELSE Run Program cbuild ${command} ${input_file} @{args} - # ${ret_code}= Run Program cbuild ${command} ${input_file} @{args} - RETURN ${ret_code} + ${is_command_empty}= Evaluate '${command}' == '${EMPTY}' + ${ret_code}= Run Keyword If ${is_command_empty} Run Program cbuild ${input_file} @{args} + ... ELSE Run Program cbuild ${input_file} @{args} ${command} + RETURN ${ret_code} Change Directory Permissions [Documentation] Change directory permissions @@ -107,22 +78,22 @@ Checkout GitHub Repository Log ${result.stderr} Should Be Equal As Integers ${result.rc} ${0} -Run Project with cbuildgen - [Documentation] Run the csolution project with cbuildgen +Build Example With cbuildgen + [Documentation] Build csolution example with cbuildgen [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} ${parent_path}= Get Parent Directory Path ${input_file} - ${output_dir}= Normalize Path ${parent_path}${/}${Out_Dir} - Append To List ${args} --output ${output_dir} --cbuildgen - # ${args}= Insert At Front ${args} ${input_file} - ${ret_code}= Run cbuild ${input_file} ${EMPTY} ${args} + ${output_dir}= Normalize Path ${parent_path}${/}${Out_Dir} + ${args_Ex}= Copy List ${args} + Append To List ${args_Ex} --output ${output_dir} --cbuildgen + ${ret_code}= Run cbuild ${input_file} ${EMPTY} ${args_Ex} Should Be Equal ${ret_code} ${expect} msg=Unexpected status returned by cbuildgen execution -Run Project With cbuild2cmake - [Documentation] Run the csolution project with cbuild2cmake +Build Example With cbuild2cmake + [Documentation] Build the csolution example with cbuild2cmake [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} Append To List ${args} --cbuild2cmake - ${args}= Insert At Front ${args} ${input_file} - ${ret_code}= Run cbuild ${input_file} ${empty} ${args} + # ${args}= Insert At Front ${args} ${input_file} + ${ret_code}= Run cbuild ${input_file} ${EMPTY} ${args} Should Be Equal ${ret_code} ${expect} msg=Unexpected status returned by cbuild2cmake execution Append Additional Arguments @@ -157,35 +128,55 @@ Cpackget Install Pack [Documentation] Install pack to pack root directory [Arguments] ${pack_id} ${pack_root_dir} ${ret_code}= Run Program cpackget add ${pack_id} - ... -R ${pack_root_dir} - ... --agree-embedded-license --force-reinstall + ... -R ${pack_root_dir} + ... --agree-embedded-license --force-reinstall RETURN ${ret_code} Get Contexts From Project - [Documentation] Run the csolution project with cbuildgen + [Documentation] Get list of contexts from the csolution project [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} ${parent_path}= Get Parent Directory Path ${input_file} ${result} Run Process csolution list contexts ${input_File} - ... -q shell=True stdout=${CURDIR}/stdout.txt + ... -q shell=True stdout=${CURDIR}/stdout.txt ${ret_code}= Set Variable If ${result.rc} == ${0} ${result.rc} ${1} Should Be Equal ${ret_code} ${expect} msg=Unexpected status returned by list context command RETURN ${result.stdout} -Convert List To Single String - [Arguments] ${LIST} - ${RESULT}= Set Variable ${EMPTY} - FOR ${element} IN @{LIST} - ${RESULT}= Evaluate ${element} ${RESULT} - END - RETURN ${RESULT} - -Join And Normalize Path - [Arguments] ${prefixPath} ${suffixPath} - ${resultPath}= Join Path ${prefixPath} ${suffixPath} +Join Paths + [Documentation] Join Paths + [Arguments] ${prefixPath} ${suffixPath} + ${resultPath}= Join Path ${prefixPath} ${suffixPath} ${resultPath}= Normalize Path ${resultPath} RETURN ${resultPath} Normalize Path [Arguments] ${Path} ${resultPath}= Replace String ${Path} \\ / - RETURN ${resultPath} \ No newline at end of file + RETURN ${resultPath} + +Build CSolution Example + [Arguments] ${input_file} ${expect} ${args}=@{EMPTY} + ${res_cbuildgen}= Run Keyword And Ignore Error Build Example With cbuild2cmake ${input_file} ${expect} ${args} + ${res_cbuild2cmake}= Run Keyword And Ignore Error Build Example With cbuildgen ${input_file} ${expect} ${args} + + # Check the result of the first run + ${success}= Set Variable ${res_cbuildgen[0]} + ${message}= Set Variable ${res_cbuildgen[1]} + Run Keyword If '${success}' == 'PASS' Log cbuildgen ran successfully + ... ELSE Log cbuildgen failed with message: ${message} + + # Check the result of the second run + ${success}= Set Variable ${res_cbuild2cmake[0]} + ${message}= Set Variable ${res_cbuild2cmake[1]} + Run Keyword If '${success}' == 'PASS' Log cbuild2cmake ran successfully + ... ELSE Log cbuild2cmake failed with message: ${message} + + Should Be Equal ${res_cbuildgen[0]} ${res_cbuild2cmake[0]} build status doesn't match + # Build Example With cbuild2cmake ${input_file} ${expect} ${args} + # Build Example With cbuildgen ${input_file} ${expect} ${args} + ${parent_path}= Get Parent Directory Path ${input_file} + ${result}= Run Keyword And Return + ... Compare Elf Information ${input_file} + ... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir} + ... ${parent_path}${/}${Default_Out_Dir} + RETURN ${result}