diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile index c4a6ea2..74dbf2c 100644 --- a/.gitpod.Dockerfile +++ b/.gitpod.Dockerfile @@ -3,9 +3,9 @@ FROM gitpod/workspace-full # Install custom tools, runtime, etc. RUN sudo apt-get update && sudo apt-get install -y gfortran && sudo apt-get clean && sudo rm -rf /var/cache/apt/* && sudo rm -rf /var/lib/apt/lists/* && sudo rm -rf /tmp/* -WORKDIR /opt/test-runner/testlib -RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/CMakeLists.txt -RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/TesterMain.f90 +WORKDIR /workspace/fortran-test-runner/testlib +RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/CMakeLists.txt +RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/TesterMain.f90 -WORKDIR /opt/test-runner -RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/config/CMakeLists.txt +WORKDIR /workspace/fortran-test-runner +RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/config/CMakeLists.txt diff --git a/.gitpod.yml b/.gitpod.yml index e4ce8f7..1cfc01f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -7,5 +7,5 @@ ports: onOpen: open-preview # List the start up tasks. You can start them in parallel in multiple terminals. See https://www.gitpod.io/docs/config-start-tasks/ -tasks: - - command: echo 'Create build directory and run build and test' && mkdir build && cd build && cmake .. && make && ctest -V && cd .. +#tasks: +# - command: echo 'Create build directory and run build and test' && mkdir -p build && cd build && cmake .. && make && ctest -V && cd .. diff --git a/Dockerfile b/Dockerfile index 29a0ba9..043fc3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,11 @@ FROM alpine:3.11 RUN apk add --no-cache coreutils curl jq gfortran libc-dev cmake make python3 git WORKDIR /opt/test-runner/testlib -RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/CMakeLists.txt -RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/TesterMain.f90 +RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/CMakeLists.txt +RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/TesterMain.f90 WORKDIR /opt/test-runner -RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/config/CMakeLists.txt +RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/config/CMakeLists.txt COPY . . diff --git a/README.md b/README.md index 9add2a0..3425d7a 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,14 @@ Since the test runners are deployed as Docker images the [related specification] To run tests: 1. Open project's root in terminal -2. Run `./test.sh` +2. Run `./bin/run-tests.sh` This will compile and run tests for all exercises in the `tests` folder, it will fail if the output file `result.json` is different from the expected one for the exercise. If you want to run specific tests: 1. Open project's root in terminal -2. Run `./test.sh ` +2. Run `./bin/run.sh ` In both cases you can show more informations, like Docker output, by using the flag `-v` or `--verbose`. diff --git a/bin/diff_expected_results.sh b/bin/diff_expected_results.sh new file mode 100755 index 0000000..52a8638 --- /dev/null +++ b/bin/diff_expected_results.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +for exp_res in $(find . -name expected_results.json); do + res=$(echo $exp_res | sed 's/expected_//') + cmd="diff $res $exp_res" + echo $cmd + $cmd + echo +done diff --git a/bin/run-tests.sh b/bin/run-tests.sh index 8b354fc..756eeb4 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -31,6 +31,16 @@ for test_dir in tests/*; do # -e "s~${test_dir_path}~/solution~g" \ # "${results_file_path}" + # Remove line numbers because they are different in Docker build vs local build + # also remove quotation marks (’‘ vs '') because they may be different on docker vs local linux + # But only on build error ("Makefile") + grep -q "Makefile" "${results_file_path}" + if [ $? -eq 0 ] ; then + message_norm=$(jq '.message' ${results_file_path} | sed 's/[0-9]*//g' | sed -e 's/["'\''’‘]//g' | sed 's/\\n//g' ) + jq --arg message_norm "$message_norm" '(.message |= $message_norm )' ${results_file_path} > ${results_file_path}_norm + cp ${results_file_path}_norm ${results_file_path} + rm ${results_file_path}_norm + fi echo "${test_dir_name}: comparing results.json to expected_results.json" diff "${results_file_path}" "${expected_results_file_path}" diff --git a/bin/run.sh b/bin/run.sh index 1e143f2..c234998 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -68,9 +68,7 @@ cmake --build . 2> "${compilation_errors_file_name}" 1>> "${compilation_stdout_f ret=$? if [ $ret -ne 0 ]; then - # remove line numbers because they are diffrent in Docker build vs local build - # also remove quotation marks (’‘ vs '') because they may be different on docker vs local linux - message=$(cat "${compilation_errors_file_name}" | sed 's/[0-9]*//g' | sed -e 's/["'\''’‘]//g' | tr \\n \; | sed 's/;/\\\\n/g') + message=$(cat "${compilation_errors_file_name}") jq -n --arg m "${message}" '{version: 2, status: "error", tests: [], message: $m}' > "${build_results_file}" else # build successful, now run test diff --git a/bin/update_expected_results.sh b/bin/update_expected_results.sh new file mode 100755 index 0000000..0ddd4c3 --- /dev/null +++ b/bin/update_expected_results.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +for exp_res in $(find . -name expected_results.json); do + res=$(echo $exp_res | sed 's/expected_//') + cmd="cp $res $exp_res" + echo $cmd + $cmd +done diff --git a/tests/example-all-fail/expected_results.json b/tests/example-all-fail/expected_results.json index 753c274..bebdcae 100644 --- a/tests/example-all-fail/expected_results.json +++ b/tests/example-all-fail/expected_results.json @@ -1,74 +1,152 @@ { "tests": [ - { "name" : "Test 1: stating something", - "status": "fail" } - , - { "name" : "Test 2: shouting", - "status": "pass" } - , - { "name" : "Test 3: shouting gibberish", - "status": "pass" } - , - { "name" : "Test 4: asking a question", - "status": "pass" } - , - { "name" : "Test 5: asking a numeric question", - "status": "pass" } - , - { "name" : "Test 6: asking gibberish", - "status": "pass" } - , - { "name" : "Test 7: talking forcefully", - "status": "fail" } - , - { "name" : "Test 8: using acronyms in regular speech", - "status": "fail" } - , - { "name" : "Test 9: forceful question", - "status": "pass" } - , - { "name" : "Test 10: shouting numbers", - "status": "pass" } - , - { "name" : "Test 11: no letters", - "status": "fail" } - , - { "name" : "Test 12: question with no letters", - "status": "pass" } - , - { "name" : "Test 13: shouting with special characters", - "status": "pass" } - , - { "name" : "Test 14: shouting with no exclamation mark", - "status": "pass" } - , - { "name" : "Test 15: statement containing question mark", - "status": "fail" } - , - { "name" : "Test 16: non-letters with question", - "status": "pass" } - , - { "name" : "Test 17: prattling on", - "status": "pass" } - , - { "name" : "Test 18: silence", - "status": "pass" } - , - { "name" : "Test 19: prolonged silence", - "status": "pass" } - , - { "name" : "Test 20: multiple line question", - "status": "fail" } - , - { "name" : "Test 21: starting with whitespace", - "status": "fail" } - , - { "name" : "Test 22: ending with whitespace", - "status": "pass" } - , - { "name" : "Test 23: non-question ending with whitespace", - "status": "fail" } + { + "name" : "Test 1: stating something", + "test_code": "Test 1: stating something", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 2: shouting", + "test_code": "Test 2: shouting", + "status" : "pass", + } + , + { + "name" : "Test 3: shouting gibberish", + "test_code": "Test 3: shouting gibberish", + "status" : "pass", + } + , + { + "name" : "Test 4: asking a question", + "test_code": "Test 4: asking a question", + "status" : "pass", + } + , + { + "name" : "Test 5: asking a numeric question", + "test_code": "Test 5: asking a numeric question", + "status" : "pass", + } + , + { + "name" : "Test 6: asking gibberish", + "test_code": "Test 6: asking gibberish", + "status" : "pass", + } + , + { + "name" : "Test 7: talking forcefully", + "test_code": "Test 7: talking forcefully", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 8: using acronyms in regular speech", + "test_code": "Test 8: using acronyms in regular speech", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 9: forceful question", + "test_code": "Test 9: forceful question", + "status" : "pass", + } + , + { + "name" : "Test 10: shouting numbers", + "test_code": "Test 10: shouting numbers", + "status" : "pass", + } + , + { + "name" : "Test 11: no letters", + "test_code": "Test 11: no letters", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 12: question with no letters", + "test_code": "Test 12: question with no letters", + "status" : "pass", + } + , + { + "name" : "Test 13: shouting with special characters", + "test_code": "Test 13: shouting with special characters", + "status" : "pass", + } + , + { + "name" : "Test 14: shouting with no exclamation mark", + "test_code": "Test 14: shouting with no exclamation mark", + "status" : "pass", + } + , + { + "name" : "Test 15: statement containing question mark", + "test_code": "Test 15: statement containing question mark", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 16: non-letters with question", + "test_code": "Test 16: non-letters with question", + "status" : "pass", + } + , + { + "name" : "Test 17: prattling on", + "test_code": "Test 17: prattling on", + "status" : "pass", + } + , + { + "name" : "Test 18: silence", + "test_code": "Test 18: silence", + "status" : "pass", + } + , + { + "name" : "Test 19: prolonged silence", + "test_code": "Test 19: prolonged silence", + "status" : "pass", + } + , + { + "name" : "Test 20: multiple line question", + "test_code": "Test 20: multiple line question", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 21: starting with whitespace", + "test_code": "Test 21: starting with whitespace", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } + , + { + "name" : "Test 22: ending with whitespace", + "test_code": "Test 22: ending with whitespace", + "status" : "pass", + } + , + { + "name" : "Test 23: non-question ending with whitespace", + "test_code": "Test 23: non-question ending with whitespace", + "status" : "fail", + "message" : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >" + } ], "version": 2, - "status": "fail" + "status" : "fail" + "message": "Test summary: 8 of 23 tests failed", } diff --git a/tests/example-empty-file/expected_results.json b/tests/example-empty-file/expected_results.json index 0ef7263..94fa9d9 100644 --- a/tests/example-empty-file/expected_results.json +++ b/tests/example-empty-file/expected_results.json @@ -2,5 +2,5 @@ "version": 2, "status": "error", "tests": [], - "message": "/tmp/example-empty-file/example_empty_file_test.f:::\\\\n\\\\n | use bob\\\\n | \\\\nFatal Error: Cannot open module file bob.mod for reading at (): No such file or directory\\\\ncompilation terminated.\\\\nmake[]: *** [CMakeFiles/example-empty-file.dir/build.make:: CMakeFiles/example-empty-file.dir/example_empty_file_test.f.o] Error \\\\nmake[]: *** [CMakeFiles/Makefile:: CMakeFiles/example-empty-file.dir/all] Error \\\\nmake: *** [Makefile:: all] Error \\\\n" + "message": "/tmp/example-empty-file/example_empty_file_test.f::: | use bob | Fatal Error: Cannot open module file bob.mod for reading at (): No such file or directorycompilation terminated.make[]: *** [CMakeFiles/example-empty-file.dir/build.make:: CMakeFiles/example-empty-file.dir/example_empty_file_test.f.o] Error make[]: *** [CMakeFiles/Makefile:: CMakeFiles/example-empty-file.dir/all] Error make: *** [Makefile:: all] Error " } diff --git a/tests/example-success/expected_results.json b/tests/example-success/expected_results.json index 71835c8..a2526cb 100644 --- a/tests/example-success/expected_results.json +++ b/tests/example-success/expected_results.json @@ -1,74 +1,143 @@ { "tests": [ - { "name" : "Test 1: stating something", - "status": "pass" } - , - { "name" : "Test 2: shouting", - "status": "pass" } - , - { "name" : "Test 3: shouting gibberish", - "status": "pass" } - , - { "name" : "Test 4: asking a question", - "status": "pass" } - , - { "name" : "Test 5: asking a numeric question", - "status": "pass" } - , - { "name" : "Test 6: asking gibberish", - "status": "pass" } - , - { "name" : "Test 7: talking forcefully", - "status": "pass" } - , - { "name" : "Test 8: using acronyms in regular speech", - "status": "pass" } - , - { "name" : "Test 9: forceful question", - "status": "pass" } - , - { "name" : "Test 10: shouting numbers", - "status": "pass" } - , - { "name" : "Test 11: no letters", - "status": "pass" } - , - { "name" : "Test 12: question with no letters", - "status": "pass" } - , - { "name" : "Test 13: shouting with special characters", - "status": "pass" } - , - { "name" : "Test 14: shouting with no exclamation mark", - "status": "pass" } - , - { "name" : "Test 15: statement containing question mark", - "status": "pass" } - , - { "name" : "Test 16: non-letters with question", - "status": "pass" } - , - { "name" : "Test 17: prattling on", - "status": "pass" } - , - { "name" : "Test 18: silence", - "status": "pass" } - , - { "name" : "Test 19: prolonged silence", - "status": "pass" } - , - { "name" : "Test 20: multiple line question", - "status": "pass" } - , - { "name" : "Test 21: starting with whitespace", - "status": "pass" } - , - { "name" : "Test 22: ending with whitespace", - "status": "pass" } - , - { "name" : "Test 23: non-question ending with whitespace", - "status": "pass" } + { + "name" : "Test 1: stating something", + "test_code": "Test 1: stating something", + "status" : "pass", + } + , + { + "name" : "Test 2: shouting", + "test_code": "Test 2: shouting", + "status" : "pass", + } + , + { + "name" : "Test 3: shouting gibberish", + "test_code": "Test 3: shouting gibberish", + "status" : "pass", + } + , + { + "name" : "Test 4: asking a question", + "test_code": "Test 4: asking a question", + "status" : "pass", + } + , + { + "name" : "Test 5: asking a numeric question", + "test_code": "Test 5: asking a numeric question", + "status" : "pass", + } + , + { + "name" : "Test 6: asking gibberish", + "test_code": "Test 6: asking gibberish", + "status" : "pass", + } + , + { + "name" : "Test 7: talking forcefully", + "test_code": "Test 7: talking forcefully", + "status" : "pass", + } + , + { + "name" : "Test 8: using acronyms in regular speech", + "test_code": "Test 8: using acronyms in regular speech", + "status" : "pass", + } + , + { + "name" : "Test 9: forceful question", + "test_code": "Test 9: forceful question", + "status" : "pass", + } + , + { + "name" : "Test 10: shouting numbers", + "test_code": "Test 10: shouting numbers", + "status" : "pass", + } + , + { + "name" : "Test 11: no letters", + "test_code": "Test 11: no letters", + "status" : "pass", + } + , + { + "name" : "Test 12: question with no letters", + "test_code": "Test 12: question with no letters", + "status" : "pass", + } + , + { + "name" : "Test 13: shouting with special characters", + "test_code": "Test 13: shouting with special characters", + "status" : "pass", + } + , + { + "name" : "Test 14: shouting with no exclamation mark", + "test_code": "Test 14: shouting with no exclamation mark", + "status" : "pass", + } + , + { + "name" : "Test 15: statement containing question mark", + "test_code": "Test 15: statement containing question mark", + "status" : "pass", + } + , + { + "name" : "Test 16: non-letters with question", + "test_code": "Test 16: non-letters with question", + "status" : "pass", + } + , + { + "name" : "Test 17: prattling on", + "test_code": "Test 17: prattling on", + "status" : "pass", + } + , + { + "name" : "Test 18: silence", + "test_code": "Test 18: silence", + "status" : "pass", + } + , + { + "name" : "Test 19: prolonged silence", + "test_code": "Test 19: prolonged silence", + "status" : "pass", + } + , + { + "name" : "Test 20: multiple line question", + "test_code": "Test 20: multiple line question", + "status" : "pass", + } + , + { + "name" : "Test 21: starting with whitespace", + "test_code": "Test 21: starting with whitespace", + "status" : "pass", + } + , + { + "name" : "Test 22: ending with whitespace", + "test_code": "Test 22: ending with whitespace", + "status" : "pass", + } + , + { + "name" : "Test 23: non-question ending with whitespace", + "test_code": "Test 23: non-question ending with whitespace", + "status" : "pass", + } ], "version": 2, - "status": "pass" + "status" : "pass" } diff --git a/tests/example-syntax-error/expected_results.json b/tests/example-syntax-error/expected_results.json index 26d3846..9d6695a 100644 --- a/tests/example-syntax-error/expected_results.json +++ b/tests/example-syntax-error/expected_results.json @@ -2,5 +2,5 @@ "version": 2, "status": "error", "tests": [], - "message": "/tmp/example-syntax-error/example_syntax_error.f:::\\\\n\\\\n | function hey(statement_THIS_IS_A_SYNTAX_ERROR)\\\\n | \\\\nError: Symbol statement_this_is_a_syntax_error at () has no IMPLICIT type\\\\n/tmp/example-syntax-error/example_syntax_error.f:::\\\\n\\\\n | character(len=*), intent(in) :: statement\\\\n | \\\\nError: Symbol at () is not a DUMMY variable\\\\nmake[]: *** [CMakeFiles/example-syntax-error.dir/build.make:: CMakeFiles/example-syntax-error.dir/example_syntax_error.f.o] Error \\\\nmake[]: *** [CMakeFiles/Makefile:: CMakeFiles/example-syntax-error.dir/all] Error \\\\nmake: *** [Makefile:: all] Error \\\\n" + "message": "/tmp/example-syntax-error/example_syntax_error.f::: | function hey(statement_THIS_IS_A_SYNTAX_ERROR) | Error: Symbol statement_this_is_a_syntax_error at () has no IMPLICIT type/tmp/example-syntax-error/example_syntax_error.f::: | character(len=*), intent(in) :: statement | Error: Symbol at () is not a DUMMY variablemake[]: *** [CMakeFiles/example-syntax-error.dir/build.make:: CMakeFiles/example-syntax-error.dir/example_syntax_error.f.o] Error make[]: *** [CMakeFiles/Makefile:: CMakeFiles/example-syntax-error.dir/all] Error make: *** [Makefile:: all] Error " }