Skip to content

Commit

Permalink
less tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@soumeh01
soumeh01 committed Jul 25, 2024
1 parent 3725ad7 commit e5d7384
Show file tree
Hide file tree
Showing 11 changed files with 243 additions and 225 deletions.
1 change: 1 addition & 0 deletions .github/workflows/e2e_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ jobs:
- name: Run Test
shell: bash
continue-on-error: true
run: |
robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
--settag ${{ matrix.target }}-${{ matrix.arch }} \
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ jobs:
--variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
--consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
--name ${{ matrix.target }}-${{ matrix.arch }} \
./test/pack_example_tests.robot
./test
- name: Archieve test results
if: always()
Expand Down
21 changes: 17 additions & 4 deletions test/lib/elf_compare.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@
import subprocess
from robot.api import logger
from robot.utils import asserts
from pathlib import Path

def compare_elf_information(input_file, cbuildgen_out_dir, cbuild2cmake_out_dir):
logger.debug('Input file: %s' % input_file)
logger.debug('Cbuildgen out dir: %s' % cbuildgen_out_dir)
logger.debug('Cbuild2cmake out dir: %s' % cbuild2cmake_out_dir)
logger.info('Input file: %s' % input_file)
logger.info('Cbuildgen out dir: %s' % cbuildgen_out_dir)
logger.info('Cbuild2cmake out dir: %s' % cbuild2cmake_out_dir)
return CompareELF(input_file, cbuildgen_out_dir, cbuild2cmake_out_dir).compare_elf_files()

class Context:
Expand Down Expand Up @@ -34,7 +35,7 @@ class Utils:
def run_command(exe_path, args):
try:
command = exe_path + ' ' + ' '.join(args)
logger.info(f"Running Command: {' '.join(command)}")
logger.info('Running Command: %s' % command)
processOut = subprocess.run(command, check=True, shell=True, universal_newlines=True, capture_output=True, timeout=300)
return True, processOut.stdout
except subprocess.CalledProcessError as e:
Expand Down Expand Up @@ -66,6 +67,18 @@ def compare_elf_files(self):
if path != '':
cbuildgen_elf_file = os.path.join(self.cbuildgen_out_dir, path)
cbuild2cmake_elf_file = os.path.join(self.cbuild2cmake_out_dir, path)
cbuildgenPath = Path(cbuildgen_elf_file)
cbuild2cmakePath = Path(cbuild2cmake_elf_file)
if cbuildgenPath.exists():
logger.info('Path Exist: %s' % cbuildgenPath)
else:
logger.info('Path doesnot Exist: %s' % cbuildgenPath)

if cbuild2cmakePath.exists():
logger.info('Path Exist: %s' % cbuild2cmakePath)
else:
logger.info('Path doesnot Exist: %s' % cbuild2cmakePath)

res1, stdout1 = self.get_elf_info(cbuildgen_elf_file)
res2, stdout2 = self.get_elf_info(cbuild2cmake_elf_file)
logger.info('Object Info %s' % stdout1)
Expand Down
49 changes: 44 additions & 5 deletions test/lib/reference_compare.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,36 @@ def extract_summary(self):
print(f"No summary found in file: {self.input_file}", file=sys.stderr)
return None

def extract_passed_tests(self):
self.read()

passed_tests_pattern = re.compile(
r"## Passed Tests\n\n\|Tag\|Test\|:clock1030: Duration\|Suite\|\n\|:---:\|:---:\|:---:\|:---:\|\n((\|.+\|\n)+)"
)

# Find the failed tests section
passed_tests_section = passed_tests_pattern.search(self.content)

# Extract the passed tests table if it exists
if not passed_tests_section:
return []

passed_tests_table = passed_tests_section.group(1)

# Regular expression to extract tag and test name from each row
# test_pattern = re.compile(r"\|(.+?)\|(.+?)\|.+?\|.+?\|.+?\|")
test_pattern = re.compile(r"(\|.+?\|.+?\|).+?\|.+?\|")

# Find all matches
passed_tests = test_pattern.findall(passed_tests_table)
return passed_tests

def compare_summaries(markdown_file: str, reference_file: str):
md_summary = MarkdownReader(markdown_file).extract_summary()
ref_summary = MarkdownReader(reference_file).extract_summary()
md_file = MarkdownReader(markdown_file)
ref_file = MarkdownReader(reference_file)

md_summary = md_file.extract_summary()
ref_summary = ref_file.extract_summary()

if md_summary is None or ref_summary is None:
print("Comparison could not be performed due to missing summary data.", file=sys.stderr)
Expand All @@ -63,8 +90,20 @@ def compare_summaries(markdown_file: str, reference_file: str):
print(f"error: Test results do not match the reference\n"
f" Expected: Passed: {ref_summary.passed}, Failed: {ref_summary.failed}, Skipped: {ref_summary.skipped}, Total: {ref_summary.total}\n"
f" Actual: Passed: {md_summary.passed}, Failed: {md_summary.failed}, Skipped: {md_summary.skipped}, Total: {md_summary.total}")
return 1 # failure

md_passed_tests = md_file.extract_passed_tests()
ref_passed_tests = ref_file.extract_passed_tests()

for testInfo in md_passed_tests:
if testInfo in ref_passed_tests:
ref_passed_tests.remove(testInfo)

if len(ref_passed_tests) > 0:
print(f"error: regression found\n")
for reg_test in ref_passed_tests:
print(f" Expected: Passed: {reg_test}\n")
return 1 # failure

return 0 # success

# def main():
Expand All @@ -74,9 +113,9 @@ def compare_summaries(markdown_file: str, reference_file: str):
# args = parser.parse_args()

# if compare_summaries(args.markdown_file, args.reference_file):
# print("Summaries are equal.")
# else:
# print("Summaries are not equal.")
# else:
# print("Summaries are equal.")

# if __name__ == '__main__':
# try:
Expand Down
5 changes: 3 additions & 2 deletions test/lib/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import glob
from pathlib import Path
# import stat
import subprocess
import re
# import os
Expand All @@ -14,7 +13,9 @@ def get_parent_directory_name(file_path:str):
return parent_dir.name

def get_parent_directory_path(file_path:str):
return Path(file_path).parent.absolute()
absPath = Path(file_path).parent.absolute()
path = str(absPath).replace('\\', '/')
return path

def write_test_environment(test_env_file:str):
toolList = ["cbuild", "cpackget", "csolution", "cbuild2cmake", "cbuildgen"]
Expand Down
20 changes: 7 additions & 13 deletions test/local_example_tests.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ Resource resources${/}global.resource
Resource resources${/}utils.resource
Library String
Library Collections
Library lib${/}elf_compare.py
Library lib${/}utils.py
Test Template Run CSolution Project
Library lib${/}elf_compare.py
Test Template Build Local CSolution Example
*** Variables ***
# The directory name of the example to be built
# The directory name of the examples to be built
${build-asm} build-asm
${build-c} build-c
${build-cpp} build-cpp
Expand Down Expand Up @@ -55,13 +55,7 @@ Validate trustzone Example
${TEST_DATA_DIR}${/}${trustzone}${/}solution.csolution.yml ${Pass}
*** Keywords ***
Run Csolution Project
[Arguments] ${input_file} ${expect} ${args}=@{EMPTY}
Run Project With cbuildgen ${input_file} ${expect} ${args}
Run Project with cbuild2cmake ${input_file} ${expect} ${args}
${parent_path}= Get Parent Directory Path ${input_file}
${result}= Run Keyword And Return
... Compare Elf Information ${input_file}
... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir}
... ${parent_path}${/}${Default_Out_Dir}
Should Be True ${result}
Build Local CSolution Example
[Arguments] ${input_file} ${expect} ${args}=@{EMPTY}
${result}= Build CSolution Example ${input_file} ${expect} ${args}
Should Be True ${result}
1 change: 1 addition & 0 deletions test/pack_example_tests.csv
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
*** Test Cases ***;${pack_id};${expect};[Tags];[Documentation]
NXP::FRDM-K32L3A6_BSP.17.0.0;NXP::FRDM-K32L3A6_BSP@17.0.0;${0};;
NXP::LPCXpresso55S69_BSP.18.0.0;NXP::LPCXpresso55S69_BSP@18.0.0;${0};;
NXP::LPCXpresso55S69_USB_Examples.1.0.0;NXP::LPCXpresso55S69_USB_Examples@1.0.0;${0};;
NXP::FRDM-K32L3A6_BSP.18.0.0;NXP::FRDM-K32L3A6_BSP@18.0.0;${0};;
Expand Down
49 changes: 16 additions & 33 deletions test/pack_example_tests.robot
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
*** Settings ***
Documentation Tests to compile & execute remote csolution examples
Documentation Tests to build pack csolution examples
Suite Setup Global Setup
Suite Teardown Global Teardown
Library lib${/}utils.py
Library BuiltIn
Library String
Library Collections
Library lib${/}elf_compare.py
Library DataDriver
Library OperatingSystem
Library String
Library lib${/}elf_compare.py
Library lib${/}utils.py
Resource resources${/}global.resource
Resource resources${/}utils.resource
Test Template Test packs examples
Expand All @@ -23,7 +23,7 @@ Test pack example ${pack_id} and expect ${expect} Default UserData
*** Keywords ***
Test packs examples
[Arguments] ${pack_id} ${expect}
${pack_root_dir}= Join And Normalize Path ${TEST_DATA_DIR} ${Local_Pack_Root_Dir}
${pack_root_dir}= Join Paths ${TEST_DATA_DIR} ${Local_Pack_Root_Dir}
Create Directory ${pack_root_dir}
Cpackget Init ${pack_root_dir}
Cpackget Install Pack ${pack_id} ${pack_root_dir}
Expand All @@ -36,27 +36,22 @@ Test packs examples
Run Csolution Project
[Arguments] ${input_file} ${expect} ${args}=@{EMPTY}
${contexts}= Get Contexts From Project ${input_file} ${expect} ${args}
${filcontexts}= Convert And Filter Contexts ${contexts}
Log Many ${args}
Log Many ${filcontexts}
# Append To List ${args} ${filcontexts}
@{MERGED_LIST} Create List @{args} @{filcontexts}
Log Many ${MERGED_LIST}
Run Project With cbuildgen ${input_file} ${expect} ${MERGED_LIST}
Run Project with cbuild2cmake ${input_file} ${expect} ${args}
${parent_path}= Get Parent Directory Path ${input_file}
${result}= Run Keyword And Return
... Compare Elf Information ${input_file}
... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir}
... ${parent_path}${/}${Default_Out_Dir}
Should Be True ${result}
${contexts}= Get Contexts From Project ${input_file} ${expect} ${args}
${filcontexts}= Convert And Filter Contexts ${contexts}
@{MERGED_LIST} Create List @{args} @{filcontexts}
Build Example With cbuildgen ${input_file} ${expect} ${MERGED_LIST}
Build Example with cbuild2cmake ${input_file} ${expect} ${MERGED_LIST}
${parent_path}= Get Parent Directory Path ${input_file}
${result}= Run Keyword And Return
... Compare Elf Information ${input_file}
... ${parent_path}${/}${Out_Dir}${/}${Default_Out_Dir}
... ${parent_path}${/}${Default_Out_Dir}
Should Be True ${result}
Convert And Filter Contexts
[Documentation] Example test to convert a string to an array, filter it, and join it
[Arguments] ${all_contexts}
@{lines}= Split String ${all_contexts} \n
Log ${lines}
# Filter out items containing '+iar'
@{filtered_lines}= Create List
Expand All @@ -65,23 +60,11 @@ Convert And Filter Contexts
${result}= Run Keyword And Return Status Should Contain ${line} iar
Run Keyword If ${result}==False Append To List ${filtered_lines} ${line}
END
Log Many ${filtered_lines}
# # Prefix remaining items with '-c ' and join them
# ${context_string}= Set Variable ${EMPTY}
# FOR ${line} IN @{filtered_lines}
# Append To List
# ${context_string}= Evaluate "-c ${line} ${context_string}"
# END
# Log Many ${context_string}
# RETURN ${context_string}
# Prefix remaining items with '-c ' and join them
@{context_list_args}= Create List
FOR ${line} IN @{filtered_lines}
Append To List ${context_list_args} -c ${line}
END
Log Many ${context_list_args}
RETURN ${context_list_args}
Loading

0 comments on commit e5d7384

Please sign in to comment.