Fix interactive problems in python (#220)

* Fix interactive problems in python

* Cleanup

* Copy swig generated files

* Fix tests

* Valid lib package in python

* Add caching for extra execution files

* Fix typo

Author: MasloMaslane

src/sinol_make/commands/run/__init__.py

@@ -7,6 +7,7 @@
 import time
 import psutil
 import glob
+import shutil
 from io import StringIO
 from typing import Dict
 
@@ -438,7 +439,7 @@ def check_output(self, name, input_file, output_file_path, output, answer_file_p
             return self.check_output_checker(name, input_file, output_file_path, answer_file_path)
 
     def execute_oiejq(self, name, timetool_path, executable, result_file_path, input_file_path, output_file_path, answer_file_path,
-                      time_limit, memory_limit, hard_time_limit):
+                      time_limit, memory_limit, hard_time_limit, execution_dir):
         command = f'"{timetool_path}" "{executable}"'
         env = os.environ.copy()
         env["MEM_LIMIT"] = f'{memory_limit}K'
@@ -449,7 +450,7 @@ def execute_oiejq(self, name, timetool_path, executable, result_file_path, input
         with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file, \
                 open(result_file_path, "w") as result_file:
             process = subprocess.Popen(command, shell=True, stdin=input_file, stdout=output_file,
-                                       stderr=result_file, env=env, preexec_fn=os.setsid)
+                                       stderr=result_file, env=env, preexec_fn=os.setsid, cwd=execution_dir)
 
             def sigint_handler(signum, frame):
                 try:
@@ -518,7 +519,7 @@ def sigint_handler(signum, frame):
 
 
     def execute_time(self, name, executable, result_file_path, input_file_path, output_file_path, answer_file_path,
-                      time_limit, memory_limit, hard_time_limit):
+                      time_limit, memory_limit, hard_time_limit, execution_dir):
         if sys.platform == 'darwin':
             time_name = 'gtime'
         elif sys.platform == 'linux':
@@ -531,7 +532,7 @@ def execute_time(self, name, executable, result_file_path, input_file_path, outp
         mem_limit_exceeded = False
         with open(input_file_path, "r") as input_file, open(output_file_path, "w") as output_file:
             process = subprocess.Popen(command, stdin=input_file, stdout=output_file, stderr=subprocess.DEVNULL,
-                                       preexec_fn=os.setsid)
+                                       preexec_fn=os.setsid, cwd=execution_dir)
 
             def sigint_handler(signum, frame):
                 try:
@@ -596,7 +597,7 @@ def sigint_handler(signum, frame):
                 program_exit_code = int(lines[0].strip().split(" ")[-1])
             elif not mem_limit_exceeded:
                 result.Status = Status.RE
-                result.Error = "Unexpected output from time command: " + "\n".join(lines)
+                result.Error = "Unexpected output from time command: " + "".join(lines)
                 return result
 
         if program_exit_code is not None and program_exit_code != 0:
@@ -630,20 +631,20 @@ def run_solution(self, data_for_execution: ExecutionData):
         Run an execution and return the result as ExecutionResult object.
         """
 
-        (name, executable, test, time_limit, memory_limit, timetool_path) = data_for_execution
+        (name, executable, test, time_limit, memory_limit, timetool_path, execution_dir) = data_for_execution
         file_no_ext = paths.get_executions_path(name, package_util.extract_test_id(test, self.ID))
         output_file = file_no_ext + ".out"
         result_file = file_no_ext + ".res"
         hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0)
 
         if self.timetool_name == 'oiejq':
             return self.execute_oiejq(name, timetool_path, executable, result_file, test, output_file, self.get_output_file(test),
-                                      time_limit, memory_limit, hard_time_limit_in_s)
+                                      time_limit, memory_limit, hard_time_limit_in_s, execution_dir)
         elif self.timetool_name == 'time':
             return self.execute_time(name, executable, result_file, test, output_file, self.get_output_file(test),
-                                     time_limit, memory_limit, hard_time_limit_in_s)
+                                     time_limit, memory_limit, hard_time_limit_in_s, execution_dir)
 
-    def run_solutions(self, compiled_commands, names, solutions):
+    def run_solutions(self, compiled_commands, names, solutions, executables_dir):
         """
         Run solutions on tests and print the results as a table to stdout.
         """
@@ -653,6 +654,13 @@ def run_solutions(self, compiled_commands, names, solutions):
         all_results = collections.defaultdict(
             lambda: collections.defaultdict(lambda: collections.defaultdict(map)))
 
+        for lang, files in self.config.get('extra_execution_files', {}).items():
+            for file in files:
+                shutil.copy(os.path.join(os.getcwd(), "prog", file), executables_dir)
+        # Copy swig generated .so files
+        for file in glob.glob(os.path.join(os.getcwd(), "prog", f"_{self.ID}lib.so")):
+            shutil.copy(file, executables_dir)
+
         for (name, executable, result) in compiled_commands:
             lang = package_util.get_file_lang(name)
             solution_cache = cache.get_cache_file(os.path.join(os.getcwd(), "prog", name))
@@ -670,7 +678,7 @@ def run_solutions(self, compiled_commands, names, solutions):
                         all_results[name][self.get_group(test)][test] = test_result.result
                     else:
                         executions.append((name, executable, test, test_time_limit, test_memory_limit,
-                                           self.timetool_path))
+                                           self.timetool_path, os.path.dirname(executable)))
                         all_results[name][self.get_group(test)][test] = ExecutionResult(Status.PENDING)
                 os.makedirs(paths.get_executions_path(name), exist_ok=True)
             else:
@@ -743,7 +751,7 @@ def compile_and_run(self, solutions):
         executables = [paths.get_executables_path(package_util.get_executable(solution)) for solution in solutions]
         compiled_commands = zip(solutions, executables, compilation_results)
         names = solutions
-        return self.run_solutions(compiled_commands, names, solutions)
+        return self.run_solutions(compiled_commands, names, solutions, paths.get_executables_path())
 
     def convert_status_to_string(self, dictionary):
         """
@@ -1196,7 +1204,8 @@ def run(self, args):
         title = self.config["title"]
         print("Task: %s (tag: %s)" % (title, self.ID))
         self.cpus = args.cpus or util.default_cpu_count()
-        cache.save_to_cache_extra_compilation_files(self.config.get("extra_compilation_files", []), self.ID)
+        cache.process_extra_compilation_files(self.config.get("extra_compilation_files", []), self.ID)
+        cache.process_extra_execution_files(self.config.get("extra_execution_files", {}), self.ID)
         cache.remove_results_if_contest_type_changed(self.config.get("sinol_contest_type", "default"))
 
         checker = package_util.get_files_matching_pattern(self.ID, f'{self.ID}chk.*')

src/sinol_make/helpers/cache.py

@@ -70,14 +70,29 @@ def save_compiled(file_path: str, exe_path: str, is_checker: bool = False):
         remove_results_cache()
 
 
-def save_to_cache_extra_compilation_files(extra_compilation_files, task_id):
+def _check_file_changed(file_path, lang, task_id):
+    solutions_re = package_util.get_solutions_re(task_id)
+    md5sum = util.get_file_md5(file_path)
+    info = get_cache_file(file_path)
+
+    if info.md5sum != md5sum:
+        for solution in os.listdir(paths.get_cache_path('md5sums')):
+            # Remove only files in the same language and matching the solution regex
+            if package_util.get_file_lang(solution) == lang and \
+                    solutions_re.match(solution) is not None:
+                os.unlink(paths.get_cache_path('md5sums', solution))
+
+    info.md5sum = md5sum
+    info.save(file_path)
+
+
+def process_extra_compilation_files(extra_compilation_files, task_id):
     """
     Checks if extra compilation files have changed and saves them to cache.
     If they have, removes all cached solutions that use them.
     :param extra_compilation_files: List of extra compilation files
     :param task_id: Task id
     """
-    solutions_re = package_util.get_solutions_re(task_id)
     for file in extra_compilation_files:
         file_path = os.path.join(os.getcwd(), "prog", file)
         if not os.path.exists(file_path):
@@ -86,17 +101,22 @@ def save_to_cache_extra_compilation_files(extra_compilation_files, task_id):
         lang = package_util.get_file_lang(file)
         if lang == 'h':
             lang = 'cpp'
-        info = get_cache_file(file_path)
+        _check_file_changed(file_path, lang, task_id)
 
-        if info.md5sum != md5sum:
-            for solution in os.listdir(paths.get_cache_path('md5sums')):
-                # Remove only files in the same language and matching the solution regex
-                if package_util.get_file_lang(solution) == lang and \
-                        solutions_re.match(solution) is not None:
-                    os.unlink(paths.get_cache_path('md5sums', solution))
 
-        info.md5sum = md5sum
-        info.save(file_path)
+def process_extra_execution_files(extra_execution_files, task_id):
+    """
+    Checks if extra execution files have changed and saves them to cache.
+    If they have, removes all cached solutions that use them.
+    :param extra_execution_files: List of extra execution files
+    :param task_id: Task id
+    """
+    for lang, files in extra_execution_files.items():
+        for file in files:
+            file_path = os.path.join(os.getcwd(), "prog", file)
+            if not os.path.exists(file_path):
+                continue
+            _check_file_changed(file_path, lang, task_id)
 
 
 def remove_results_cache():

tests/commands/run/test_integration.py

@@ -676,14 +676,17 @@ def change_file(file, comment_character):
         with open(file, "w") as f:
             f.write(f"{comment_character} Changed source code.\n" + source)
 
-    def test(file_to_change, lang, comment_character):
+    def test(file_to_change, lang, comment_character, extra_compilation_files=True):
         # First run to cache test results.
         command.run(args)
 
         # Change file
         change_file(os.path.join(os.getcwd(), "prog", file_to_change), comment_character)
 
-        cache.save_to_cache_extra_compilation_files(command.config.get("extra_compilation_files", []), command.ID)
+        if extra_compilation_files:
+            cache.process_extra_compilation_files(command.config.get("extra_compilation_files", []), command.ID)
+        else:
+            cache.process_extra_execution_files(command.config.get("extra_execution_files", {}), command.ID)
         task_id = package_util.get_task_id()
         solutions = package_util.get_solutions(task_id, None)
         for solution in solutions:
@@ -695,7 +698,7 @@ def test(file_to_change, lang, comment_character):
 
     test("liblib.cpp", "cpp", "//")
     test("liblib.h", "cpp", "//")
-    test("liblib.py", "py", "#")
+    test("liblib.py", "py", "#", False)
 
 
 @pytest.mark.parametrize("create_package", [get_simple_package_path()], indirect=True)

tests/commands/run/test_unit.py

@@ -39,7 +39,8 @@ def test_execution(create_package, time_tool):
         config = yaml.load(config_file, Loader=yaml.FullLoader)
 
     os.makedirs(paths.get_executions_path(solution), exist_ok=True)
-    result = command.run_solution((solution, paths.get_executables_path(executable), test, config['time_limit'], config['memory_limit'], oiejq.get_oiejq_path()))
+    result = command.run_solution((solution, paths.get_executables_path(executable), test, config['time_limit'],
+                                   config['memory_limit'], oiejq.get_oiejq_path(), paths.get_executions_path()))
     assert result.Status == Status.OK
 
 

tests/helpers/test_cache.py

@@ -81,15 +81,15 @@ def test_cache():
         with open("prog/abclib.cpp", "w") as f:
             f.write("int main() { return 0; }")
 
-        cache.save_to_cache_extra_compilation_files(["abclib.cpp"], "abc")
+        cache.process_extra_compilation_files(["abclib.cpp"], "abc")
         assert cache.get_cache_file("/some/very/long/path/abc.cpp") == CacheFile()
         assert cache.get_cache_file("abclib.cpp") != CacheFile()
 
         cache_file.save("abc.cpp")
         cache_file.save("abc.py")
         with open("prog/abclib.cpp", "w") as f:
             f.write("/* Changed file */ int main() { return 0; }")
-        cache.save_to_cache_extra_compilation_files(["abclib.cpp"], "abc")
+        cache.process_extra_compilation_files(["abclib.cpp"], "abc")
         assert not os.path.exists(paths.get_cache_path("md5sums", "abc.cpp"))
         assert os.path.exists(paths.get_cache_path("md5sums", "abc.py"))
         assert cache.get_cache_file("abc.py") == cache_file

tests/packages/lib/config.yml

@@ -4,9 +4,11 @@ time_limit: 1000
 scores:
   1: 50
   2: 50
-extra_compilation_files: [liblib.cpp, liblib.h, liblib.py]
+extra_compilation_files: [liblib.cpp, liblib.h]
 extra_compilation_args:
   cpp: [liblib.cpp]
+extra_execution_files:
+  py: [liblib.py]
 sinol_expected_scores:
   lib.cpp:
     expected: