Fix TL when output is large (#149)

* Fix TL when program generates large output

* Add tests

* Fix tests

* Change lim package

* Fix ovl package

* Fix lim package

* Fix lim package again

* Change wait in lim2.cpp

* Change limits in lim package

* Change wait time in lim2

* Change wait time in lim2

src/sinol_make/commands/run/__init__.py

@@ -403,16 +403,18 @@ def check_output(self, name, input_file, output_file_path, output, answer_file_p
                 output_file.write("\n".join(output) + "\n")
             return self.check_output_checker(name, input_file, output_file_path, answer_file_path)
 
-    def execute_oiejq(self, command, name, result_file_path, input_file_path, output_file_path, answer_file_path,
+    def execute_oiejq(self, name, timetool_path, executable, result_file_path, input_file_path, output_file_path, answer_file_path,
                       time_limit, memory_limit, hard_time_limit):
+        command = f'"{timetool_path}" "{executable}"'
         env = os.environ.copy()
         env["MEM_LIMIT"] = f'{memory_limit}K'
         env["MEASURE_MEM"] = "1"
 
         timeout = False
-        with open(input_file_path, "r") as input_file:
-            process = subprocess.Popen(command, shell=True, stdin=input_file, stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE, env=env, preexec_fn=os.setsid)
+        with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file, \
+                open(result_file_path, "w") as result_file:
+            process = subprocess.Popen(command, shell=True, stdin=input_file, stdout=output_file,
+                                       stderr=result_file, env=env, preexec_fn=os.setsid)
 
             def sigint_handler(signum, frame):
                 try:
@@ -423,7 +425,7 @@ def sigint_handler(signum, frame):
             signal.signal(signal.SIGINT, sigint_handler)
 
             try:
-                output, lines = process.communicate(timeout=hard_time_limit)
+                process.wait(timeout=time_limit)
             except subprocess.TimeoutExpired:
                 timeout = True
                 try:
@@ -432,11 +434,15 @@ def sigint_handler(signum, frame):
                     pass
                 process.communicate()
 
+        with open(result_file_path, "r") as result_file:
+            lines = result_file.read()
+        with open(output_file_path, "r") as output_file:
+            output = output_file.read()
         result = ExecutionResult()
 
         if not timeout:
-            lines = lines.decode('utf-8').splitlines()
-            output = output.decode('utf-8').splitlines()
+            lines = lines.splitlines()
+            output = output.splitlines()
 
             for line in lines:
                 line = line.strip()
@@ -476,14 +482,20 @@ def sigint_handler(signum, frame):
         return result
 
 
-    def execute_time(self, command, name, result_file_path, input_file_path, output_file_path, answer_file_path,
+    def execute_time(self, name, executable, result_file_path, input_file_path, output_file_path, answer_file_path,
                       time_limit, memory_limit, hard_time_limit):
-
-        executable = package_util.get_executable(name)
+        if sys.platform == 'darwin':
+            time_name = 'gtime'
+        elif sys.platform == 'linux':
+            time_name = 'time'
+        elif sys.platform == 'win32' or sys.platform == 'cygwin':
+            raise Exception("Measuring time with GNU time on Windows is not supported.")
+
+        command = [f'{time_name}', '-f', '%U\\n%M\\n%x', '-o', result_file_path, executable]
         timeout = False
         mem_limit_exceeded = False
-        with open(input_file_path, "r") as input_file:
-            process = subprocess.Popen(command, stdin=input_file, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+        with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file:
+            process = subprocess.Popen(command, stdin=input_file, stdout=output_file, stderr=subprocess.DEVNULL,
                                        preexec_fn=os.setsid)
 
             def sigint_handler(signum, frame):
@@ -520,12 +532,13 @@ def sigint_handler(signum, frame):
                         pass
                     timeout = True
                     break
-            output, _ = process.communicate()
 
+        with open(output_file_path, "r") as output_file:
+            output = output_file.read()
         result = ExecutionResult()
         program_exit_code = None
         if not timeout:
-            output = output.decode("utf-8").splitlines()
+            output = output.splitlines()
             with open(result_file_path, "r") as result_file:
                 lines = result_file.readlines()
             if len(lines) == 3:
@@ -589,22 +602,10 @@ def run_solution(self, data_for_execution: ExecutionData):
         hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0)
 
         if self.timetool_name == 'oiejq':
-            command = f'"{timetool_path}" "{executable}"'
-
-            return self.execute_oiejq(command, name, result_file, test, output_file, self.get_output_file(test),
+            return self.execute_oiejq(name, timetool_path, executable, result_file, test, output_file, self.get_output_file(test),
                                       time_limit, memory_limit, hard_time_limit_in_s)
         elif self.timetool_name == 'time':
-            if sys.platform == 'darwin':
-                timeout_name = 'gtimeout'
-                time_name = 'gtime'
-            elif sys.platform == 'linux':
-                timeout_name = 'timeout'
-                time_name = 'time'
-            elif sys.platform == 'win32' or sys.platform == 'cygwin':
-                raise Exception("Measuring time with GNU time on Windows is not supported.")
-
-            command = [f'{time_name}', '-f', '%U\\n%M\\n%x', '-o', result_file, executable]
-            return self.execute_time(command, name, result_file, test, output_file, self.get_output_file(test),
+            return self.execute_time(name, executable, result_file, test, output_file, self.get_output_file(test),
                                      time_limit, memory_limit, hard_time_limit_in_s)
 
     def run_solutions(self, compiled_commands, names, solutions):
@@ -1179,11 +1180,11 @@ def run(self, args):
         self.check_errors(all_results)
         try:
             validation_results = self.validate_expected_scores(results)
-        except:
+        except Exception:
             self.config = util.try_fix_config(self.config)
             try:
                 validation_results = self.validate_expected_scores(results)
-            except:
+            except Exception:
                 util.exit_with_error("Validating expected scores failed. "
                                      "This probably means that `sinol_expected_scores` is broken. "
                                      "Delete it and run `sinol-make run --apply-suggestions` again.")

tests/commands/run/test_integration.py

@@ -14,7 +14,8 @@
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
                                             get_checker_package_path(), get_library_package_path(),
                                             get_library_string_args_package_path(), get_limits_package_path(),
-                                            get_override_limits_package_path(), get_icpc_package_path()],
+                                            get_override_limits_package_path(), get_icpc_package_path(),
+                                            get_large_output_package_path()],
                          indirect=True)
 def test_simple(create_package, time_tool):
     """
@@ -70,7 +71,8 @@ def test_wrong_solution(create_package, time_tool):
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
                                             get_checker_package_path(), get_library_package_path(),
                                             get_library_string_args_package_path(), get_limits_package_path(),
-                                            get_override_limits_package_path(), get_icpc_package_path()],
+                                            get_override_limits_package_path(), get_icpc_package_path(),
+                                            get_large_output_package_path()],
                          indirect=True)
 def test_no_expected_scores(capsys, create_package, time_tool):
     """
@@ -106,7 +108,8 @@ def test_no_expected_scores(capsys, create_package, time_tool):
 @pytest.mark.parametrize("create_package", [get_simple_package_path(), get_verify_status_package_path(),
                                             get_checker_package_path(), get_library_package_path(),
                                             get_library_string_args_package_path(), get_limits_package_path(),
-                                            get_override_limits_package_path(), get_icpc_package_path()],
+                                            get_override_limits_package_path(), get_icpc_package_path(),
+                                            get_large_output_package_path()],
                          indirect=True)
 def test_apply_suggestions(create_package, time_tool):
     """
@@ -452,8 +455,8 @@ def test_mem_limit_kill(create_package, time_tool):
     end_time = time.time()
 
     assert e.value.code == 1
-    assert end_time - start_time < 5  # The solution runs for 20 seconds, but it immediately exceeds memory limit,
-                                      # so it should be killed.
+    assert end_time - start_time < 10  # The solution runs for 20 seconds, but it immediately exceeds memory limit,
+                                       # so it should be killed.
 
 
 @pytest.mark.parametrize("create_package", [get_undocumented_options_package_path()], indirect=True)

tests/packages/large_output/config.yml

@@ -0,0 +1,13 @@
+title: Package with large outputs
+sinol_task_id: lou
+memory_limit: 16000
+time_limit: 2000
+sinol_expected_scores:
+  lou.cpp:
+    expected:
+      0: {points: 0, status: OK}
+    points: 0
+  lou1.cpp:
+    expected:
+      0: {points: 0, status: WA}
+    points: 0

tests/packages/large_output/in/lou0.in

@@ -0,0 +1 @@
+1

tests/packages/large_output/prog/lou.cpp

@@ -0,0 +1,11 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int bytes_to_write = (1 << 16) + 1;
+    for (int i = 0; i < bytes_to_write; i++) {
+        cout << "x";
+    }
+    return 0;
+}

tests/packages/large_output/prog/lou1.cpp

@@ -0,0 +1,11 @@
+#include <bits/stdc++.h>
+
+using namespace std;
+
+int main() {
+    int bytes_to_write = (1 << 20);
+    for (int i = 0; i < bytes_to_write; i++) {
+        cout << "x";
+    }
+    return 0;
+}

tests/packages/lim/config.yml

@@ -4,7 +4,7 @@ memory_limits:
   2: 60000
 time_limit: 1000
 time_limits:
-  1: 3000
+  1: 5000
 scores:
   1: 50
   2: 50

tests/packages/lim/prog/lim2.cpp

@@ -2,17 +2,27 @@
 #include <chrono>
 
 using namespace std;
-using namespace std::chrono_literals;
+using namespace std::chrono;
+
+int wait(int milisecs) {
+    auto start = high_resolution_clock::now();
+    int i = 0;
+    while (duration_cast<milliseconds>(high_resolution_clock::now() - start).count() < milisecs)
+        i++;
+    return i;
+}
 
 int main() {
     int a, b;
     cin >> a >> b;
 
     if (a == 2 && b == 1) {
-        this_thread::sleep_for(6s);
+        int i = wait(7000);
+        a += i - i;
     }
     else {
-        this_thread::sleep_for(2s);
+        int i = wait(2000);
+        a += i - i;
     }
 
     cout << a + b << endl;

tests/packages/ovl/prog/ovl.cpp

@@ -2,12 +2,21 @@
 #include <chrono>
 
 using namespace std;
-using namespace std::chrono_literals;
+using namespace std::chrono;
+
+int wait(int secs) {
+    auto start = high_resolution_clock::now();
+    int i = 0;
+    while (duration_cast<seconds>(high_resolution_clock::now() - start).count() < secs)
+        i++;
+    return i;
+}
 
 int main() {
-    this_thread::sleep_for(2s);
+    int i = wait(2);
 
     int a, b;
     cin >> a >> b;
-    cout << a + b;
+    b += i;
+    cout << a + b - i;
 }

tests/packages/vso/prog/vso4.cpp

@@ -8,17 +8,26 @@
 #include <bits/stdc++.h>
 
 using namespace std;
+using namespace std::chrono;
 
 int rnd() {
     return rand() % 100;
 }
 
+int wait(int secs) {
+    auto start = high_resolution_clock::now();
+    int i = 0;
+    while (duration_cast<seconds>(high_resolution_clock::now() - start).count() < secs)
+        i++;
+    return i;
+}
+
 int main() {
     int a, b;
     cin >> a >> b;
     if (a == 1 && b == 1) {
-        this_thread::sleep_for(chrono::seconds(3));
-        cout << a + b;
+        int i = wait(3);
+        cout << a + b + i - i;
     }
     else if (a == 1 && b == 2) {
         vector<int*> v;

tests/packages/vso/prog/vso7.cpp

@@ -8,11 +8,20 @@
 #include <bits/stdc++.h>
 
 using namespace std;
+using namespace std::chrono;
 
 int rnd() {
     return rand() % 100;
 }
 
+int wait(int secs) {
+    auto start = high_resolution_clock::now();
+    int i = 0;
+    while (duration_cast<seconds>(high_resolution_clock::now() - start).count() < secs)
+        i++;
+    return i;
+}
+
 int main() {
     int a, b;
     cin >> a >> b;
@@ -23,8 +32,8 @@ int main() {
         cout << a + b;
     }
     else if (a == 1 && b == 3) {
-        this_thread::sleep_for(chrono::seconds(3));
-        cout << a + b;
+        int i = wait(3);
+        cout << a + b + i - i;
     }
     else if (a == 1 && b == 4) {
         int c = 0;

tests/util.py

@@ -122,11 +122,20 @@ def get_icpc_package_path():
     return os.path.join(os.path.dirname(__file__), "packages", "icpc")
 
 
+def get_large_output_package_path():
+    """
+    Get path to package with large output (/tests/packages/large_output)
+    """
+    return os.path.join(os.path.dirname(__file__), "packages", "large_output")
+
 def create_ins(package_path, task_id):
     """
     Create .in files for package.
     """
-    ingen = package_util.get_files_matching_pattern(task_id, f'{task_id}ingen.*')[0]
+    all_ingens = package_util.get_files_matching_pattern(task_id, f'{task_id}ingen.*')
+    if len(all_ingens) == 0:
+        return
+    ingen = all_ingens[0]
     ingen_executable = paths.get_executables_path("ingen.e")
     os.makedirs(paths.get_executables_path(), exist_ok=True)
     assert compile.compile(ingen, ingen_executable)