diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index eefe548b..4a26d66a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -85,6 +85,14 @@ jobs: path: cpackget fetch-depth: 0 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: cpackget/go.mod + cache-dependency-path: | + cpackget/go.mod + cpackget/go.sum + - name: Build cpackget executable working-directory: cpackget shell: bash @@ -99,6 +107,14 @@ jobs: path: cbuild2cmake fetch-depth: 0 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: cbuild2cmake/go.mod + cache-dependency-path: | + cbuild2cmake/go.mod + cbuild2cmake/go.sum + - name: Build cbuild2cmake executable working-directory: cbuild2cmake shell: bash @@ -112,7 +128,7 @@ jobs: gh run download -D projmgr-${{ matrix.target }}-${{ matrix.arch }} ${{ steps.get_run_id.outputs.NIGHTLY_RUN_ID }} -n projmgr-${{ matrix.target }}-${{ matrix.arch }} -R Open-CMSIS-Pack/devtools gh run download -D cbuildgen-${{ matrix.target }}-${{ matrix.arch }} ${{ steps.get_run_id.outputs.NIGHTLY_RUN_ID }} -n cbuildgen-${{ matrix.target }}-${{ matrix.arch }} -R Open-CMSIS-Pack/devtools - - name: Checkout toolbox repo + - name: Checkout cmsis-toolbox repo uses: actions/checkout@v4 with: repository: Open-CMSIS-PACK/devtools @@ -217,15 +233,12 @@ jobs: if: always() shell: bash run: | - ls -l ${{steps.toolbox_path.outputs.path}} export PATH="${{steps.toolbox_path.outputs.path}}:$PATH" - echo "${{steps.toolbox_path.outputs.path}}" csolution -V cpackget -V cbuild -V cbuild2cmake -V cbuildgen -V - which csolution - name: Run Test shell: bash @@ -234,7 +247,7 @@ jobs: which cbuild ls -l ${{steps.toolbox_path.outputs.path}} ${{steps.toolbox_path.outputs.path}}/cbuild -V - python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test + python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test - name: Archieve test results if: always() @@ -273,12 +286,13 @@ jobs: run: | python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \ ./reports-windows-amd64/output.xml \ - ./reports-linux-amd64/output.xml + ./reports-linux-amd64/output.xml \ + ./reports-darwin-amd64/output.xml - name: Generate Summary report if: always() run: | - python ./test/lib/execution_summary.py artifacts/collective_robot_results/output.xml + python ./test/lib/execution_summary.py artifacts artifacts/collective_robot_results/output.xml summary_report.md - name: Print E2E Report if: always() diff --git a/test/lib/execution_summary.py b/test/lib/execution_summary.py index c57aaec6..4681b2ee 100644 --- a/test/lib/execution_summary.py +++ b/test/lib/execution_summary.py @@ -6,19 +6,22 @@ import os import re -import shutil -import subprocess +import argparse import sys +import glob +from pathlib import Path from robot.api import ExecutionResult, ResultVisitor from robot.result.model import TestCase from robot.result.executionresult import Result class ResultVisitorEx(ResultVisitor): - def __init__(self, markdown_file='summary_report.md'): + def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str): self.failed_tests = {} self.passed_tests = {} self.skipped_tests = {} + self.test_env_files_path=test_env_files_path self.markdown_file = markdown_file + self.output_path = output_path # Remove existing markdown file if it exists if os.path.exists(markdown_file): @@ -34,9 +37,50 @@ def visit_test(self, test: TestCase): test_status[tags] = [] test_status[tags].append(status) + def read_file_content(self, file_path): + with open(file_path, 'r') as file: + return file.read() + + def extract_platform(self, file_name): + match = re.search(r'test-env-(.*)\.md', file_name) + return match.group(1) if match else None + + def get_test_env(self, f): + test_env_files = glob.glob(self.test_env_files_path + '/**/test-env-*.md', recursive=False) + # return if no test_env file found + if not test_env_files: + print("No test-env-*.md files found") + return + + # Read the content of the first file + first_file_content = self.read_file_content(test_env_files[0]) + + has_same_env = True + f.write("## Test Environment\n\n") + test_env_content = "" + + # Iterate through the rest of the files and compare content + for file_path in test_env_files[1:]: + file_name = Path(file_path).name + platform = self.extract_platform(file_name) + if platform: + test_env_content += f"\n## {platform}\n\n" + + current_file_content = self.read_file_content(file_path) + test_env_content += current_file_content + + if current_file_content != first_file_content: + has_same_env &= False + + if has_same_env: + test_env_content = first_file_content + + f.write(test_env_content + "\n") + def end_result(self, result: Result): with open(self.markdown_file, "w") as f: f.write("# Robot Framework Report\n\n") + self.get_test_env(f) f.write("## Summary\n\n") f.write("|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|\n") f.write("|:----:|:----:|:-----:|:---:|\n") @@ -65,12 +109,24 @@ def __write_test_section(self, file, test_dict, section_header, table_header): file.write(f"|{key}|{name}|{msg}|{duration}|{suite}|\n") elif section_header.startswith("Skip"): file.write(f"|{key}|{name}|{suite}|\n") - -if __name__ == '__main__': - # Get output and markdown file paths from command line arguments - output_file = sys.argv[1] if len(sys.argv) > 1 else "output.xml" - markdown_file = sys.argv[2] if len(sys.argv) > 2 else "summary_report.md" - - # Parse the Robot Framework output file and generate the summary report + +def main(): + parser = argparse.ArgumentParser(description='Consolidate test summary report') + parser.add_argument('test_env_files_path', type=str, help='Path to the test environment files') + parser.add_argument('output_file', type=str, help='Path to output xml file') + parser.add_argument('markdown_file', type=str, help='Path to consolidated summary markdown file') + args = parser.parse_args() + + test_env_files_path = args.test_env_files_path + output_file = args.output_file + markdown_file = args.markdown_file + result = ExecutionResult(output_file) - result.visit(ResultVisitorEx(markdown_file)) + result.visit(ResultVisitorEx(test_env_files_path, output_file, markdown_file)) + +if __name__ == '__main__': + try: + main() + except Exception as e: + print(f'An error occurred: {e}', file=sys.stderr) + sys.exit(1) diff --git a/test/lib/utils.py b/test/lib/utils.py index 1a449d19..2cffa0d5 100644 --- a/test/lib/utils.py +++ b/test/lib/utils.py @@ -1,5 +1,8 @@ import glob from pathlib import Path +import shutil +import subprocess +import re def glob_files_in_directory(directory:str, pattern: str, recursive: bool): yaml_files = glob.glob(directory + '/**/' + pattern, recursive=recursive) @@ -11,3 +14,26 @@ def get_parent_directory_name(file_path:str): def get_parent_directory_path(file_path:str): return Path(file_path).parent.absolute() + +def write_test_environment(test_env_file:str): + toolList = ["cbuild", "cpackget", "csolution", "cbuild2cmake", "cbuildgen"] + + # Create markdown content + markdown_content = "|Name|Version|\n" + markdown_content += "|:---|:------|\n" + + for index in range(0, len(toolList)): + version = "unknown" + tool = toolList[index] + versionCmd = tool + ' -V' + output = subprocess.run(versionCmd, shell=True, universal_newlines=True, capture_output=True) + if output.stdout != "": + result = re.search(r"(\d+\.\d+\.\d+.*) \(C\)", output.stdout) + if result: + if len(result.groups()) > 0: + version = result.group(1) + markdown_content += "|" + tool + "|" + version + "|\n" + + # Write tool version info + with open(test_env_file, "w") as file: + file.write(markdown_content) diff --git a/test/resources/utils.resource b/test/resources/utils.resource index 8a89daa6..bffb57ee 100644 --- a/test/resources/utils.resource +++ b/test/resources/utils.resource @@ -1,19 +1,26 @@ *** Settings *** Documentation A collection of commonly used keywords across multiple test suites +Library BuiltIn Library Collections Library Process Library OperatingSystem Library ..${/}lib${/}utils.py Resource global.resource +*** Variables *** +${TEST_ENV_FILE} test_env.md *** Keywords *** Global Setup - ${parent_dir}= Join Path ${CURDIR} .. - ${src_dir}= Join Path ${parent_dir} ${Data} - ${dest_dir}= Get Test Data Directory - Set Global Variable ${TEST_DATA_DIR} ${dest_dir} - Copy Directory ${src_dir} ${dest_dir} + ${parent_dir}= Join Path ${CURDIR} .. + ${src_dir}= Join Path ${parent_dir} ${Data} + ${dest_dir}= Get Test Data Directory + Set Global Variable ${TEST_DATA_DIR} ${dest_dir} + Copy Directory ${src_dir} ${dest_dir} + ${output_file_path} Get Output File Path + ${parent_path}= Get Parent Directory Path ${output_file_path} + ${test_env_file}= Join Path ${parent_path} ${TEST_ENV_FILE} + Write Test Environment ${test_env_file} Global Teardown Remove Directory with Content ${TEST_DATA_DIR} @@ -30,7 +37,7 @@ Run Program ${result} Run Process ${exe_path} ${input_File} @{args} ... shell=True stdout=${CURDIR}/stdout.txt ${ret_code}= Set Variable If ${result.rc} == ${0} ${result.rc} ${1} - Log Many StdOut: ${result.stdout} + Log Output String ${result.stdout} RETURN ${ret_code} Run cbuild @@ -77,7 +84,17 @@ Run Project With cbuild2cmake Should Be Equal ${ret_code} ${expect} msg=Unexpected status returned by cbuild2cmake execution Append Additional Arguments - [Documentation] Append values to the list + [Documentation] Append values to the list [Arguments] ${list} @{values} ${args}= Combine Lists ${list} ${values} RETURN ${args} + +Log Output String + [Documentation] Log STDOUT message + [Arguments] ${message} + Log Many StdOut: ${message} + +Get Output File Path + [Documentation] Find path to output files + ${output_path}= Get Variable Value ${OUTPUT_FILE} + RETURN ${output_path} \ No newline at end of file