Update handling of result files

.gitpod.Dockerfile

@@ -3,9 +3,9 @@ FROM gitpod/workspace-full
 # Install custom tools, runtime, etc.
 RUN sudo apt-get update && sudo apt-get install -y gfortran && sudo apt-get clean && sudo rm -rf /var/cache/apt/* && sudo rm -rf /var/lib/apt/lists/* && sudo rm -rf /tmp/*
 
-WORKDIR /opt/test-runner/testlib
-RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/CMakeLists.txt
-RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/TesterMain.f90
+WORKDIR /workspace/fortran-test-runner/testlib
+RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/CMakeLists.txt
+RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/TesterMain.f90
 
-WORKDIR /opt/test-runner
-RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/config/CMakeLists.txt
+WORKDIR /workspace/fortran-test-runner
+RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/config/CMakeLists.txt

.gitpod.yml

@@ -7,5 +7,5 @@ ports:
     onOpen: open-preview
 
 # List the start up tasks. You can start them in parallel in multiple terminals. See https://www.gitpod.io/docs/config-start-tasks/
-tasks:
-  - command: echo 'Create build directory and run build and test' && mkdir build && cd build && cmake .. && make && ctest -V && cd ..
+#tasks:
+#  - command: echo 'Create build directory and run build and test' && mkdir -p build && cd build && cmake .. && make && ctest -V && cd ..

Dockerfile

@@ -3,11 +3,11 @@ FROM alpine:3.11
 RUN apk add --no-cache coreutils curl jq gfortran libc-dev cmake make python3 git
 
 WORKDIR /opt/test-runner/testlib
-RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/CMakeLists.txt
-RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/testlib/TesterMain.f90
+RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/CMakeLists.txt
+RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/testlib/TesterMain.f90
 
 WORKDIR /opt/test-runner
-RUN curl -R -O https://raw.githubusercontent.com/pclausen/fortran/main/config/CMakeLists.txt
+RUN curl -R -O https://raw.githubusercontent.com/exercism/fortran/main/config/CMakeLists.txt
 
 
 COPY . .

README.md

@@ -13,14 +13,14 @@ Since the test runners are deployed as Docker images the [related specification]
 To run tests:
 
 1. Open project's root in terminal
-2. Run `./test.sh`
+2. Run `./bin/run-tests.sh`
 
 This will compile and run tests for all exercises in the `tests` folder, it will fail if the output file `result.json` is different from the expected one for the exercise.
 
 If you want to run specific tests:
 
 1. Open project's root in terminal
-2. Run `./test.sh <EXERCISE_SLUG_1> <EXERCISE_SLUG_2>`
+2. Run `./bin/run.sh <EXERCISE_SLUG_1> <EXERCISE_SLUG_2>`
 
 In both cases you can show more informations, like Docker output, by using the flag `-v` or `--verbose`.
 

bin/diff_expected_results.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+for exp_res in $(find . -name expected_results.json); do
+    res=$(echo $exp_res | sed 's/expected_//')
+    cmd="diff $res $exp_res"
+    echo $cmd
+    $cmd
+    echo 
+done

bin/run-tests.sh

@@ -31,6 +31,16 @@ for test_dir in tests/*; do
     #   -e "s~${test_dir_path}~/solution~g" \
     #   "${results_file_path}"
 
+    # Remove line numbers because they are different in Docker build vs local build
+    # also remove quotation marks (’‘ vs '') because they may be different on docker vs local linux
+    # But only on build error ("Makefile")
+    grep -q "Makefile" "${results_file_path}"
+    if [ $? -eq 0  ] ; then
+        message_norm=$(jq '.message' ${results_file_path} | sed 's/[0-9]*//g' | sed -e 's/["'\''’‘]//g' | sed 's/\\n//g' )
+        jq --arg message_norm "$message_norm"  '(.message |= $message_norm )' ${results_file_path} > ${results_file_path}_norm
+        cp ${results_file_path}_norm ${results_file_path} 
+        rm ${results_file_path}_norm
+    fi
     echo "${test_dir_name}: comparing results.json to expected_results.json"
     diff "${results_file_path}" "${expected_results_file_path}"
 

bin/run.sh

@@ -68,9 +68,7 @@ cmake --build . 2> "${compilation_errors_file_name}" 1>> "${compilation_stdout_f
 ret=$?
 
 if [ $ret -ne 0 ]; then
-    # remove line numbers because they are diffrent in Docker build vs local build
-    # also remove quotation marks (’‘ vs '') because they may be different on docker vs local linux
-    message=$(cat "${compilation_errors_file_name}" | sed 's/[0-9]*//g' | sed -e 's/["'\''’‘]//g' | tr \\n \; | sed 's/;/\\\\n/g')
+    message=$(cat "${compilation_errors_file_name}") 
     jq -n --arg m "${message}" '{version: 2, status: "error", tests: [], message: $m}' > "${build_results_file}"
 else
     # build successful, now run test

bin/update_expected_results.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+for exp_res in $(find . -name expected_results.json); do
+    res=$(echo $exp_res | sed 's/expected_//')
+    cmd="cp $res $exp_res"
+    echo $cmd
+    $cmd
+done

tests/example-all-fail/expected_results.json

@@ -1,74 +1,152 @@
  {
    "tests": [
-     { "name"  : "Test 1: stating something",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 2: shouting",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 3: shouting gibberish",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 4: asking a question",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 5: asking a numeric question",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 6: asking gibberish",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 7: talking forcefully",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 8: using acronyms in regular speech",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 9: forceful question",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 10: shouting numbers",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 11: no letters",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 12: question with no letters",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 13: shouting with special characters",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 14: shouting with no exclamation mark",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 15: statement containing question mark",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 16: non-letters with question",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 17: prattling on",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 18: silence",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 19: prolonged silence",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 20: multiple line question",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 21: starting with whitespace",
-       "status": "fail" }
-     ,
-     { "name"  : "Test 22: ending with whitespace",
-       "status": "pass" }
-     ,
-     { "name"  : "Test 23: non-question ending with whitespace",
-       "status": "fail" }
+     {
+       "name"     : "Test 1: stating something",
+       "test_code": "Test 1: stating something",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 2: shouting",
+       "test_code": "Test 2: shouting",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 3: shouting gibberish",
+       "test_code": "Test 3: shouting gibberish",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 4: asking a question",
+       "test_code": "Test 4: asking a question",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 5: asking a numeric question",
+       "test_code": "Test 5: asking a numeric question",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 6: asking gibberish",
+       "test_code": "Test 6: asking gibberish",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 7: talking forcefully",
+       "test_code": "Test 7: talking forcefully",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 8: using acronyms in regular speech",
+       "test_code": "Test 8: using acronyms in regular speech",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 9: forceful question",
+       "test_code": "Test 9: forceful question",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 10: shouting numbers",
+       "test_code": "Test 10: shouting numbers",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 11: no letters",
+       "test_code": "Test 11: no letters",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 12: question with no letters",
+       "test_code": "Test 12: question with no letters",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 13: shouting with special characters",
+       "test_code": "Test 13: shouting with special characters",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 14: shouting with no exclamation mark",
+       "test_code": "Test 14: shouting with no exclamation mark",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 15: statement containing question mark",
+       "test_code": "Test 15: statement containing question mark",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 16: non-letters with question",
+       "test_code": "Test 16: non-letters with question",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 17: prattling on",
+       "test_code": "Test 17: prattling on",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 18: silence",
+       "test_code": "Test 18: silence",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 19: prolonged silence",
+       "test_code": "Test 19: prolonged silence",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 20: multiple line question",
+       "test_code": "Test 20: multiple line question",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 21: starting with whitespace",
+       "test_code": "Test 21: starting with whitespace",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
+     ,
+     {
+       "name"     : "Test 22: ending with whitespace",
+       "test_code": "Test 22: ending with whitespace",
+       "status"   : "pass",
+     }
+     ,
+     {
+       "name"     : "Test 23: non-question ending with whitespace",
+       "test_code": "Test 23: non-question ending with whitespace",
+       "status"   : "fail",
+       "message"  : "Expected < Whatever. > but got < Whatever. THIS IS A FAILING TEST >"
+     }
    ],
    "version": 2,
-   "status": "fail"
+   "status" : "fail"
+   "message": "Test summary: 8 of 23 tests failed",
  }

tests/example-empty-file/expected_results.json

@@ -2,5 +2,5 @@
   "version": 2,
   "status": "error",
   "tests": [],
-  "message": "/tmp/example-empty-file/example_empty_file_test.f:::\\\\n\\\\n    |   use bob\\\\n      |      \\\\nFatal Error: Cannot open module file bob.mod for reading at (): No such file or directory\\\\ncompilation terminated.\\\\nmake[]: *** [CMakeFiles/example-empty-file.dir/build.make:: CMakeFiles/example-empty-file.dir/example_empty_file_test.f.o] Error \\\\nmake[]: *** [CMakeFiles/Makefile:: CMakeFiles/example-empty-file.dir/all] Error \\\\nmake: *** [Makefile:: all] Error \\\\n"
+  "message": "/tmp/example-empty-file/example_empty_file_test.f:::    |   use bob      |      Fatal Error: Cannot open module file bob.mod for reading at (): No such file or directorycompilation terminated.make[]: *** [CMakeFiles/example-empty-file.dir/build.make:: CMakeFiles/example-empty-file.dir/example_empty_file_test.f.o] Error make[]: *** [CMakeFiles/Makefile:: CMakeFiles/example-empty-file.dir/all] Error make: *** [Makefile:: all] Error "
 }