Rename cache directory to .cache (#106)

* Rename `cache` dir to `.cache`

* Remove moving cache to .cache on newer versions

* Add function for getting paths in cache dir

* Fix tests

* Refactor `paths` module

Author: MasloMaslane

.gitignore

@@ -6,6 +6,7 @@ build
 .idea
 __pycache__
 /tests/packages/**/cache
+/tests/packages/**/.cache
 src/sinol_make/data
 
 # pytest-cov

src/sinol_make/__init__.py

@@ -8,7 +8,7 @@
 
 from sinol_make import util, oiejq
 
-__version__ = "1.5.4"
+__version__ = "1.5.5"
 
 def configure_parsers():
     parser = argparse.ArgumentParser(

src/sinol_make/commands/export/__init__.py

@@ -6,7 +6,7 @@
 import yaml
 
 from sinol_make import util
-from sinol_make.helpers import package_util, parsers
+from sinol_make.helpers import package_util, parsers, paths
 from sinol_make.commands.gen import gen_util
 from sinol_make.interfaces.BaseCommand import BaseCommand
 
@@ -34,7 +34,7 @@ def get_generated_tests(self):
         if not gen_util.ingen_exists(self.task_id):
             return []
 
-        working_dir = os.path.join(os.getcwd(), 'cache', 'export', 'tests')
+        working_dir = paths.get_cache_path('export', 'tests')
         if os.path.exists(working_dir):
             shutil.rmtree(working_dir)
         os.makedirs(working_dir)
@@ -136,7 +136,7 @@ def run(self, args: argparse.Namespace):
         with open(os.path.join(os.getcwd(), 'config.yml'), 'r') as config_file:
             config = yaml.load(config_file, Loader=yaml.FullLoader)
 
-        export_package_path = os.path.join(os.getcwd(), 'cache', 'export', self.task_id)
+        export_package_path = paths.get_cache_path('export', self.task_id)
         if os.path.exists(export_package_path):
             shutil.rmtree(export_package_path)
         os.makedirs(export_package_path)

src/sinol_make/commands/inwer/__init__.py

@@ -9,7 +9,7 @@
 
 from sinol_make import util
 from sinol_make.commands.inwer.structs import TestResult, InwerExecution, VerificationResult, TableData
-from sinol_make.helpers import package_util, compile, printer
+from sinol_make.helpers import package_util, compile, printer, paths
 from sinol_make.helpers.parsers import add_compilation_arguments
 from sinol_make.interfaces.BaseCommand import BaseCommand
 from sinol_make.commands.inwer import inwer_util
@@ -52,7 +52,7 @@ def verify_test(execution: InwerExecution) -> VerificationResult:
         """
         Verifies a test and returns the result of inwer on this test.
         """
-        output_dir = os.path.join(os.getcwd(), 'cache', 'executions', execution.test_name)
+        output_dir = paths.get_executables_path(execution.test_name)
         os.makedirs(output_dir, exist_ok=True)
 
         command = [execution.inwer_exe_path]

src/sinol_make/commands/run/__init__.py

@@ -14,7 +14,7 @@
 from sinol_make.helpers.parsers import add_compilation_arguments
 from sinol_make.interfaces.BaseCommand import BaseCommand
 from sinol_make.interfaces.Errors import CompilationError, CheckerOutputException, UnknownContestType
-from sinol_make.helpers import compile, compiler, package_util, printer
+from sinol_make.helpers import compile, compiler, package_util, printer, paths
 from sinol_make.structs.status_structs import Status
 import sinol_make.util as util
 import yaml, os, collections, sys, re, math, dictdiffer
@@ -339,8 +339,8 @@ def get_groups(self, tests):
 
 
     def compile_solutions(self, solutions):
-        os.makedirs(self.COMPILATION_DIR, exist_ok=True)
-        os.makedirs(self.EXECUTABLES_DIR, exist_ok=True)
+        os.makedirs(paths.get_compilation_log_path(), exist_ok=True)
+        os.makedirs(paths.get_executables_path(), exist_ok=True)
         print("Compiling %d solutions..." % len(solutions))
         args = [(solution, True) for solution in solutions]
         with mp.Pool(self.cpus) as pool:
@@ -349,10 +349,9 @@ def compile_solutions(self, solutions):
 
 
     def compile(self, solution, use_extras = False):
-        compile_log_file = os.path.join(
-            self.COMPILATION_DIR, "%s.compile_log" % package_util.get_file_name(solution))
+        compile_log_file = paths.get_compilation_log_path("%s.compile_log" % package_util.get_file_name(solution))
         source_file = os.path.join(os.getcwd(), "prog", self.get_solution_from_exe(solution))
-        output = os.path.join(self.EXECUTABLES_DIR, package_util.get_executable(solution))
+        output = paths.get_executables_path(package_util.get_executable(solution))
 
         extra_compilation_args = []
         extra_compilation_files = []
@@ -581,7 +580,7 @@ def run_solution(self, data_for_execution: ExecutionData):
         """
 
         (name, executable, test, time_limit, memory_limit, timetool_path) = data_for_execution
-        file_no_ext = os.path.join(self.EXECUTIONS_DIR, name, package_util.extract_test_id(test))
+        file_no_ext = paths.get_executions_path(name, package_util.extract_test_id(test))
         output_file = file_no_ext + ".out"
         result_file = file_no_ext + ".res"
         hard_time_limit_in_s = math.ceil(2 * time_limit / 1000.0)
@@ -620,7 +619,7 @@ def run_solutions(self, compiled_commands, names, solutions):
                     executions.append((name, executable, test, package_util.get_time_limit(test, self.config, lang, self.args),
                                        package_util.get_memory_limit(test, self.config, lang, self.args), self.timetool_path))
                     all_results[name][self.get_group(test)][test] = ExecutionResult(Status.PENDING)
-                os.makedirs(os.path.join(self.EXECUTIONS_DIR, name), exist_ok=True)
+                os.makedirs(paths.get_executions_path(name), exist_ok=True)
             else:
                 for test in self.tests:
                     all_results[name][self.get_group(test)][test] = ExecutionResult(Status.CE)
@@ -685,9 +684,8 @@ def compile_and_run(self, solutions):
         for i in range(len(solutions)):
             if not compilation_results[i]:
                 self.failed_compilations.append(solutions[i])
-        os.makedirs(self.EXECUTIONS_DIR, exist_ok=True)
-        executables = [os.path.join(self.EXECUTABLES_DIR, package_util.get_executable(solution))
-                       for solution in solutions]
+        os.makedirs(paths.get_executions_path(), exist_ok=True)
+        executables = [paths.get_executables_path(package_util.get_executable(solution)) for solution in solutions]
         compiled_commands = zip(solutions, executables, compilation_results)
         names = solutions
         return self.run_solutions(compiled_commands, names, solutions)
@@ -966,12 +964,7 @@ def set_group_result(solution, group, result):
 
     def set_constants(self):
         self.ID = package_util.get_task_id()
-        self.TMP_DIR = os.path.join(os.getcwd(), "cache")
-        self.COMPILATION_DIR = os.path.join(self.TMP_DIR, "compilation")
-        self.EXECUTIONS_DIR = os.path.join(self.TMP_DIR, "executions")
-        self.EXECUTABLES_DIR = os.path.join(self.TMP_DIR, "executables")
         self.SOURCE_EXTENSIONS = ['.c', '.cpp', '.py', '.java']
-        self.PROGRAMS_IN_ROW = 8
         self.SOLUTIONS_RE = re.compile(r"^%s[bs]?[0-9]*\.(cpp|cc|java|py|pas)$" % self.ID)
 
 
@@ -1138,7 +1131,7 @@ def run(self, args):
             print(util.info("Checker found: %s" % os.path.basename(checker[0])))
             self.checker = checker[0]
             checker_basename = os.path.basename(self.checker)
-            self.checker_executable = os.path.join(self.EXECUTABLES_DIR, checker_basename + ".e")
+            self.checker_executable = paths.get_executables_path(checker_basename + ".e")
 
             checker_compilation = self.compile_solutions([self.checker])
             if not checker_compilation[0]:

src/sinol_make/helpers/compile.py

@@ -8,12 +8,13 @@
 
 import sinol_make.helpers.compiler as compiler
 from sinol_make import util
+from sinol_make.helpers import paths
 from sinol_make.interfaces.Errors import CompilationError
 from sinol_make.structs.compiler_structs import Compilers
 
 
 def create_compilation_cache():
-    os.makedirs(os.path.join(os.getcwd(), "cache", "md5sums"), exist_ok=True)
+    os.makedirs(paths.get_cache_path("md5sums"), exist_ok=True)
 
 
 def check_compiled(file_path: str):
@@ -25,7 +26,7 @@ def check_compiled(file_path: str):
     create_compilation_cache()
     md5sum = util.get_file_md5(file_path)
     try:
-        info_file_path = os.path.join(os.getcwd(), "cache", "md5sums", os.path.basename(file_path))
+        info_file_path = paths.get_cache_path("md5sums", os.path.basename(file_path))
         with open(info_file_path, 'r') as info_file:
             info = yaml.load(info_file, Loader=yaml.FullLoader)
             if info.get("md5sum", "") == md5sum:
@@ -39,13 +40,13 @@ def check_compiled(file_path: str):
 
 def save_compiled(file_path: str, exe_path: str):
     """
-    Save the compiled executable path to cache in `cache/md5sums/<basename of file_path>`,
+    Save the compiled executable path to cache in `.cache/md5sums/<basename of file_path>`,
     which contains the md5sum of the file and the path to the executable.
     :param file_path: Path to the file
     :param exe_path: Path to the compiled executable
     """
     create_compilation_cache()
-    info_file_path = os.path.join(os.getcwd(), "cache", "md5sums", os.path.basename(file_path))
+    info_file_path = paths.get_cache_path("md5sums", os.path.basename(file_path))
     info = {
         "md5sum": util.get_file_md5(file_path),
         "executable_path": exe_path
@@ -147,11 +148,8 @@ def compile_file(file_path: str, name: str, compilers: Compilers, weak_compilati
     :param weak_compilation_flags: Use weaker compilation flags
     :return: Tuple of (executable path or None if compilation failed, log path)
     """
-
-    executable_dir = os.path.join(os.getcwd(), 'cache', 'executables')
-    compile_log_dir = os.path.join(os.getcwd(), 'cache', 'compilation')
-    os.makedirs(executable_dir, exist_ok=True)
-    os.makedirs(compile_log_dir, exist_ok=True)
+    os.makedirs(paths.get_executables_path(), exist_ok=True)
+    os.makedirs(paths.get_compilation_log_path(), exist_ok=True)
 
     with open(os.path.join(os.getcwd(), "config.yml"), "r") as config_file:
         config = yaml.load(config_file, Loader=yaml.FullLoader)
@@ -164,8 +162,8 @@ def compile_file(file_path: str, name: str, compilers: Compilers, weak_compilati
         args = [args]
     extra_compilation_args = [os.path.join(os.getcwd(), "prog", file) for file in args]
 
-    output = os.path.join(executable_dir, name)
-    compile_log_path = os.path.join(compile_log_dir, os.path.splitext(name)[0] + '.compile_log')
+    output = paths.get_executables_path(name)
+    compile_log_path = paths.get_compilation_log_path(os.path.splitext(name)[0] + '.compile_log')
     with open(compile_log_path, 'w') as compile_log:
         try:
             if compile(file_path, output, compilers, compile_log, weak_compilation_flags, extra_compilation_args,

src/sinol_make/helpers/package_util.py

@@ -5,6 +5,7 @@
 from typing import List, Union, Dict, Any
 
 from sinol_make import util
+from sinol_make.helpers import paths
 
 
 def get_task_id() -> str:
@@ -71,7 +72,7 @@ def get_executable_path(solution: str) -> str:
     """
     Returns path to compiled executable for given solution.
     """
-    return os.path.join(os.getcwd(), 'cache', 'executables', get_executable(solution))
+    return paths.get_executables_path(get_executable(solution))
 
 
 def get_file_lang(file_path):

src/sinol_make/helpers/paths.py

@@ -0,0 +1,33 @@
+import os
+
+
+def get_cache_path(*paths):
+    """
+    Function to get a path in the cache directory. Works the same as os.path.join.
+    With no arguments, it returns the path to the cache directory.
+    """
+    return os.path.join(os.getcwd(), ".cache", *paths)
+
+
+def get_executables_path(*paths):
+    """
+    Function to get a path in executables directory. Works the same as os.path.join.
+    With no arguments, it returns the path to the executables directory.
+    """
+    return os.path.join(get_cache_path("executables"), *paths)
+
+
+def get_compilation_log_path(*paths):
+    """
+    Function to get a path in compilation log directory. Works the same as os.path.join.
+    With no arguments, it returns the path to the compilation log directory.
+    """
+    return os.path.join(get_cache_path("compilation"), *paths)
+
+
+def get_executions_path(*paths):
+    """
+    Function to get a path in executions directory. Works the same as os.path.join.
+    With no arguments, it returns the path to the executions directory.
+    """
+    return os.path.join(get_cache_path("executions"), *paths)

src/sinol_make/oiejq/__init__.py

@@ -63,9 +63,9 @@ def install_oiejq():
     if request.status_code != 200:
         raise Exception('Couldn\'t download oiejq (https://oij.edu.pl/zawodnik/srodowisko/oiejq.tar.gz returned status code: ' + str(request.status_code) + ')')
 
-    # oiejq is downloaded to a temporary directory and not to the `cache` dir,
+    # oiejq is downloaded to a temporary directory and not to the `.cache` dir,
     # as there is no guarantee that the current directory is the package directory.
-    # The `cache` dir is only used for files that are part of the package and those
+    # The `.cache` dir is only used for files that are part of the package and those
     # that the package creator might want to look into.
     with tempfile.TemporaryDirectory() as tmpdir:
         oiejq_path = os.path.join(tmpdir, 'oiejq.tar.gz')

src/sinol_make/util.py

@@ -1,10 +1,13 @@
 import glob, importlib, os, sys, requests, yaml
 import platform
 import tempfile
+import shutil
 import hashlib
 import threading
 from typing import Union
 
+import sinol_make
+
 
 def get_commands():
     """

tests/commands/run/test_unit.py

@@ -55,8 +55,8 @@ def test_execution(create_package, time_tool):
     with open(os.path.join(package_path, "config.yml"), "r") as config_file:
         config = yaml.load(config_file, Loader=yaml.FullLoader)
 
-    os.makedirs(os.path.join(command.EXECUTIONS_DIR, solution), exist_ok=True)
-    result = command.run_solution((solution, os.path.join(command.EXECUTABLES_DIR, executable), test, config['time_limit'], config['memory_limit'], oiejq.get_oiejq_path()))
+    os.makedirs(paths.get_executions_path(solution), exist_ok=True)
+    result = command.run_solution((solution, paths.get_executables_path(executable), test, config['time_limit'], config['memory_limit'], oiejq.get_oiejq_path()))
     assert result.Status == Status.OK
 
 
@@ -129,8 +129,8 @@ def test_print_expected_scores(capsys):
 
 
 def test_validate_expected_scores_success():
-    command = get_command()
     os.chdir(get_simple_package_path())
+    command = get_command()
     command.scores = command.config["scores"]
     command.tests = package_util.get_tests(None)
 
@@ -195,8 +195,8 @@ def test_validate_expected_scores_success():
 
 
 def test_validate_expected_scores_fail(capsys):
-    command = get_command()
     os.chdir(get_simple_package_path())
+    command = get_command()
     command.scores = {1: 20, 2: 20, 3: 20, 4: 20}
 
     # Test with missing points for group in config.

tests/conftest.py

@@ -5,15 +5,14 @@
 import pytest
 import multiprocessing as mp
 
-from sinol_make.helpers import compile
+from sinol_make.helpers import compile, paths
 
 
 def _compile(args):
     package, file_path = args
     os.chdir(package)
-    output = os.path.join(package, "cache", "executables", os.path.splitext(os.path.basename(file_path))[0] + ".e")
-    with open(os.path.join(package, "cache", "compilation",
-                           os.path.basename(file_path) + ".compile_log"), "w") as compile_log:
+    output = paths.get_executables_path(os.path.splitext(os.path.basename(file_path))[0] + ".e")
+    with open(paths.get_compilation_log_path(os.path.basename(file_path) + ".compile_log"), "w") as compile_log:
         compile.compile(file_path, output, compile_log=compile_log)
 
 
@@ -43,7 +42,7 @@ def pytest_configure(config):
                 continue
 
             for d in ["compilation", "executables"]:
-                os.makedirs(os.path.join(package, "cache", d), exist_ok=True)
+                os.makedirs(os.path.join(package, ".cache", d), exist_ok=True)
 
             for program in glob.glob(os.path.join(package, "prog", "*")):
                 if os.path.isfile(program) and os.path.splitext(program)[1] in [".c", ".cpp", ".py", ".java"]:

tests/packages/gen/prog/geningen.sh

@@ -12,7 +12,7 @@ function generate() {
     done
 }
 
-cache=$(dirname "$0")/../cache
+cache=$(dirname "$0")/../.cache
 mkdir -p "$cache"
 g++ gen_helper.cpp -o "$cache"/gen
 

tests/util.py

@@ -2,8 +2,7 @@
 import glob
 import subprocess
 
-from sinol_make.helpers import compile
-
+from sinol_make.helpers import compile, paths
 
 
 def get_simple_package_path():
@@ -93,11 +92,11 @@ def create_ins(package_path):
     Create .in files for package.
     """
     ingen = glob.glob(os.path.join(package_path, "prog", "*ingen.*"))[0]
-    ingen_executable = os.path.join(package_path, "cache", "executables", "ingen.e")
-    os.makedirs(os.path.join(package_path, "cache", "executables"), exist_ok=True)
+    ingen_executable = paths.get_executables_path("ingen.e")
+    os.makedirs(paths.get_executables_path(), exist_ok=True)
     assert compile.compile(ingen, ingen_executable)
     os.chdir(os.path.join(package_path, "in"))
-    os.system("../cache/executables/ingen.e")
+    os.system("../.cache/executables/ingen.e")
     os.chdir(package_path)
 
 
@@ -106,13 +105,13 @@ def create_outs(package_path):
     Create .out files for package.
     """
     solution = glob.glob(os.path.join(package_path, "prog", "???.*"))[0]
-    solution_executable = os.path.join(package_path, "cache", "executables", "solution.e")
-    os.makedirs(os.path.join(package_path, "cache", "executables"), exist_ok=True)
+    solution_executable = paths.get_executables_path("solution.e")
+    os.makedirs(paths.get_executables_path(), exist_ok=True)
     assert compile.compile(solution, solution_executable)
     os.chdir(os.path.join(package_path, "in"))
     for file in glob.glob("*.in"):
         with open(file, "r") as in_file, open(os.path.join("../out", file.replace(".in", ".out")), "w") as out_file:
-            subprocess.Popen([os.path.join(package_path, "cache", "executables", "solution.e")],
+            subprocess.Popen([os.path.join(package_path, ".cache", "executables", "solution.e")],
                              stdin=in_file, stdout=out_file).wait()
     os.chdir(package_path)