test version

.github/workflows/nightly.yml

@@ -217,15 +217,12 @@ jobs:
         if: always()
         shell: bash
         run: |
-          ls -l ${{steps.toolbox_path.outputs.path}}
           export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
-          echo "${{steps.toolbox_path.outputs.path}}"
           csolution -V
           cpackget -V
           cbuild -V
           cbuild2cmake -V
           cbuildgen -V
-          which csolution
 
       - name: Run Test
         shell: bash
@@ -234,7 +231,7 @@ jobs:
           which cbuild
           ls -l ${{steps.toolbox_path.outputs.path}}
           ${{steps.toolbox_path.outputs.path}}/cbuild -V
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
 
       - name: Archieve test results
         if: always()
@@ -273,12 +270,13 @@ jobs:
       run: |
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-windows-amd64/output.xml \
-          ./reports-linux-amd64/output.xml
+          ./reports-linux-amd64/output.xml \
+          ./reports-darwin-amd64/output.xml
 
     - name: Generate Summary report
       if: always()
       run: |
-        python ./test/lib/execution_summary.py artifacts/collective_robot_results/output.xml
+        python ./test/lib/execution_summary.py artifacts artifacts/collective_robot_results/output.xml summary_report.md
 
     - name: Print E2E Report
       if: always()

test/lib/execution_summary.py

@@ -6,19 +6,24 @@
 
 import os
 import re
+import argparse
 import shutil
 import subprocess
 import sys
+import glob
+from pathlib import Path
 from robot.api import ExecutionResult, ResultVisitor
 from robot.result.model import TestCase
 from robot.result.executionresult import Result 
 
 class ResultVisitorEx(ResultVisitor):
-    def __init__(self, markdown_file='summary_report.md'):
+    def __init__(self, test_env_files_path:str, output_path:str, markdown_file:str):
         self.failed_tests = {}
         self.passed_tests = {}
         self.skipped_tests = {}
+        self.test_env_files_path=test_env_files_path
         self.markdown_file = markdown_file
+        self.output_path = output_path
 
         # Remove existing markdown file if it exists
         if os.path.exists(markdown_file):
@@ -34,9 +39,54 @@ def visit_test(self, test: TestCase):
             test_status[tags] = []
         test_status[tags].append(status)
 
+    def read_file_content(self, file_path):
+        with open(file_path, 'r') as file:
+            return file.read()
+
+    def extract_platform(self, file_name):
+        match = re.search(r'test-env-(.*)\.md', file_name)
+        return match.group(1) if match else None
+
+    def get_test_env(self, f):
+        test_env_files = glob.glob(self.test_env_files_path + '/test-env-*.md', recursive=False)
+        print("-----------------")
+        # return if no test_env file found
+        if not test_env_files:
+            print("NO files found")
+            return
+
+        # Read the content of the first file
+        first_file_content = self.read_file_content(test_env_files[0])
+
+        same_env = True
+        f.write("## Test Environment\n\n")
+        test_env_content = ""
+
+        # Iterate through the rest of the files and compare content
+        for file_path in test_env_files[1:]:
+            print(file_path)
+            file_name = Path(file_path).name
+            platform = self.extract_platform(file_name)
+            if platform:
+                test_env_content += f"\n## {platform}\n\n"
+            else:
+                print("match not found")
+
+            current_file_content = self.read_file_content(file_path)
+            test_env_content += current_file_content
+
+            if current_file_content != first_file_content:
+                same_env &= False
+
+        if same_env:
+            test_env_content = first_file_content
+
+        f.write(test_env_content + "\n")
+
     def end_result(self, result: Result):
         with open(self.markdown_file, "w") as f:
             f.write("# Robot Framework Report\n\n")
+            self.get_test_env(f)
             f.write("## Summary\n\n")
             f.write("|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|\n")
             f.write("|:----:|:----:|:-----:|:---:|\n")
@@ -65,12 +115,41 @@ def __write_test_section(self, file, test_dict, section_header, table_header):
                         file.write(f"|{key}|{name}|{msg}|{duration}|{suite}|\n")
                     elif section_header.startswith("Skip"):
                         file.write(f"|{key}|{name}|{suite}|\n")
-
-if __name__ == '__main__':
-    # Get output and markdown file paths from command line arguments
-    output_file = sys.argv[1] if len(sys.argv) > 1 else "output.xml"
-    markdown_file = sys.argv[2] if len(sys.argv) > 2 else "summary_report.md"
-
-    # Parse the Robot Framework output file and generate the summary report
+
+def main():
+    parser = argparse.ArgumentParser(description='Consolidate test summary report.')
+    parser.add_argument('test_env_files_path', type=str, help='The path to the test environment files')
+    parser.add_argument('output_file', type=str, help='File path to output xml file')
+    parser.add_argument('markdown_file', type=str, default='summary_report.md', help='File path to output summary markdown file')
+    args = parser.parse_args()
+
+    test_env_files_path = args.test_env_files_path
+    output_file = args.output_file
+    markdown_file = args.markdown_file
+
     result = ExecutionResult(output_file)
-    result.visit(ResultVisitorEx(markdown_file))
+    result.visit(ResultVisitorEx(test_env_files_path, output_path, markdown_file))
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as e:
+        print(f'An error occurred: {e}', file=sys.stderr)
+        sys.exit(1)
+
+# if __name__ == '__main__':
+#     # Get output and markdown file paths from command line arguments
+#     if len(sys.argv) > 1:
+#         test_env_files_path = sys.argv[1]
+#     else:
+
+#         exit(1)
+
+#     output_file = sys.argv[1] if len(sys.argv) > 1 else "output.xml"
+#     markdown_file = sys.argv[2] if len(sys.argv) > 2 else "summary_report.md"
+#     test_env_file_path = sys.argv[3] if len(sys.argv) > 3 else ""
+#     output_path = Path(output_file).parent.absolute()
+
+#     # Parse the Robot Framework output file and generate the summary report
+#     result = ExecutionResult(output_file)
+#     result.visit(ResultVisitorEx(output_path.as_posix(), markdown_file))

test/lib/utils.py

@@ -1,5 +1,8 @@
 import glob
 from pathlib import Path
+import shutil
+import subprocess
+import re
 
 def glob_files_in_directory(directory:str, pattern: str, recursive: bool):
     yaml_files = glob.glob(directory + '/**/' + pattern, recursive=recursive)
@@ -11,3 +14,26 @@ def get_parent_directory_name(file_path:str):
 
 def get_parent_directory_path(file_path:str):
     return Path(file_path).parent.absolute()
+
+def write_test_environment(test_env_file:str):
+    toolList = ["cbuild", "cpackget", "csolution", "cbuild2cmake", "cbuildgen"]
+
+    # Create markdown content
+    markdown_content = "|Name|Version|\n"
+    markdown_content += "|:---|:------|\n"
+
+    for index in range(0, len(toolList)):
+        version = "unknown"
+        tool = toolList[index]
+        versionCmd = tool + ' -V'
+        output = subprocess.run(versionCmd, shell=True, universal_newlines=True, capture_output=True)
+        if output.stdout != "":
+            result = re.search(r"(\d+\.\d+\.\d+.*) \(C\)", output.stdout)
+            if result:
+                if len(result.groups()) > 0:
+                    version = result.group(1)
+        markdown_content += "|" + tool + "|" + version + "|\n"
+
+    # Write tool version info
+    with open(test_env_file, "w") as file:
+        file.write(markdown_content)

test/resources/utils.resource

@@ -1,19 +1,26 @@
 *** Settings ***
 Documentation    A collection of commonly used keywords across multiple test suites
+Library          BuiltIn
 Library          Collections
 Library          Process
 Library          OperatingSystem
 Library          ..${/}lib${/}utils.py
 Resource         global.resource
 
+*** Variables ***
+${TEST_ENV_FILE}    test_env.md
 
 *** Keywords ***
 Global Setup
-    ${parent_dir}=                  Join Path           ${CURDIR}        ..
-    ${src_dir}=                     Join Path           ${parent_dir}    ${Data}
-    ${dest_dir}=                    Get Test Data Directory
-    Set Global Variable             ${TEST_DATA_DIR}    ${dest_dir}
-    Copy Directory                  ${src_dir}          ${dest_dir}
+    ${parent_dir}=                    Join Path           ${CURDIR}        ..
+    ${src_dir}=                       Join Path           ${parent_dir}    ${Data}
+    ${dest_dir}=                      Get Test Data Directory
+    Set Global Variable               ${TEST_DATA_DIR}    ${dest_dir}
+    Copy Directory                    ${src_dir}          ${dest_dir}
+    ${output_file_path}               Get Output File Path
+    ${parent_path}=                   Get Parent Directory Path    ${output_file_path}
+    ${test_env_file}=    Join Path    ${parent_path}    ${TEST_ENV_FILE}
+    Write Test Environment            ${test_env_file} 
 
 Global Teardown
     Remove Directory with Content    ${TEST_DATA_DIR}
@@ -30,7 +37,7 @@ Run Program
     ${result}    Run Process    ${exe_path}      ${input_File}    @{args}
     ...    shell=True    stdout=${CURDIR}/stdout.txt
     ${ret_code}=                Set Variable If    ${result.rc} == ${0}    ${result.rc}    ${1}
-    Log Many                    StdOut: ${result.stdout}
+    Log Output String           ${result.stdout}
     RETURN                      ${ret_code}
 
 Run cbuild
@@ -77,7 +84,17 @@ Run Project With cbuild2cmake
     Should Be Equal     ${ret_code}       ${expect}        msg=Unexpected status returned by cbuild2cmake execution
 
 Append Additional Arguments
-    [Documentation]     Append values to the list
+    [Documentation]              Append values to the list
     [Arguments]                  ${list}    @{values}
     ${args}=    Combine Lists    ${list}    ${values}
     RETURN                       ${args}
+
+Log Output String
+    [Documentation]             Log STDOUT message
+    [Arguments]                 ${message}
+    Log Many                    StdOut: ${message}
+
+Get Output File Path
+    [Documentation]             Find path to output files
+    ${output_path}=             Get Variable Value    ${OUTPUT_FILE}
+    RETURN                      ${output_path}