update requirements (#99)

Makefile

@@ -1,5 +1,5 @@
 lint:
-	python3 -m isort -rc -c src/ tests/ example/
+	python3 -m isort -c src/ tests/ example/
 	python3 -m black --check src/ tests/ example/
 	python3 -m flake8 src/ tests/ example/
 	python3 -m mypy src/ tests/ example/
@@ -9,7 +9,7 @@ diff:
 	python3 -m black --diff src/ tests/ example/
 
 format:
-	python3 -m isort -rc src/ tests/ example/
+	python3 -m isort src/ tests/ example/
 	python3 -m black src/ tests/ example/
 
 test:

dev-requirements.txt

@@ -1,23 +1,23 @@
 appdirs==1.4.4
-atomicwrites==1.4.0
-attrs==19.3.0
-black==19.10b0
+attrs==20.3.0
+black==20.8b1
 click==7.1.2
-coverage==5.1
-flake8==3.8.2
-isort==4.3.21
+coverage==5.5
+flake8==3.9.1
+iniconfig==1.1.1
+isort==5.8.0
 mccabe==0.6.1
-more-itertools==8.3.0
-mypy==0.780
+mypy==0.812
 mypy-extensions==0.4.3
-pathspec==0.8.0
+packaging==20.9
+pathspec==0.8.1
 pluggy==0.13.1
 py==1.10.0
-pycodestyle==2.6.0
-pyflakes==2.2.0
-pytest==4.4.2
-regex==2020.5.14
-six==1.15.0
-toml==0.10.1
-typed-ast==1.4.1
-typing-extensions==3.7.4.2
+pycodestyle==2.7.0
+pyflakes==2.3.1
+pyparsing==2.4.7
+pytest==6.2.3
+regex==2021.4.4
+toml==0.10.2
+typed-ast==1.4.3
+typing-extensions==3.7.4.3

requirements.txt

@@ -1,24 +1,16 @@
-atomicwrites==1.4.0
-attrs==19.3.0
-blinker==1.4
-certifi==2019.3.9
-chardet==3.0.4
+certifi==2020.12.5
+chardet==4.0.0
 click==7.1.2
-colorama==0.4.1
 enochecker-cli==0.6.0
 enochecker-core==0.8.1
-Flask==1.0.3
-idna==2.8
+Flask==1.1.2
+idna==2.10
 itsdangerous==1.1.0
 Jinja2==2.11.3
 jsons==1.4.2
 MarkupSafe==1.1.1
-psutil==5.6.6
-pymongo==3.8.0
-requests==2.21.0
-six==1.15.0
+pymongo==3.11.3
+requests==2.25.1
 typish==1.9.2
-urllib3==1.24.3
-uwsgidecorators==1.1.0
-websocket-client==0.56.0
-Werkzeug==0.15.5
+urllib3==1.26.4
+Werkzeug==1.0.1

src/enochecker/enochecker.py

@@ -185,7 +185,8 @@ def __init__(
 
         self.debug(
             "Initialized checker for task {} in {} seconds".format(
-                jsons.dumps(task), datetime.datetime.now() - self.time_started_at,
+                jsons.dumps(task),
+                datetime.datetime.now() - self.time_started_at,
             )
         )
 
@@ -773,7 +774,8 @@ def http(
 
 
 def run(
-    checker_cls: Type[BaseChecker], args: Optional[Sequence[str]] = None,
+    checker_cls: Type[BaseChecker],
+    args: Optional[Sequence[str]] = None,
 ) -> Optional[CheckerResult]:
     """
     Run a checker, either from cmdline or as uwsgi script.

src/enochecker/results.py

@@ -38,7 +38,9 @@ class EnoException(Exception, ABC):
     result: CheckerTaskResult = CheckerTaskResult.CHECKER_TASK_RESULT_INTERNAL_ERROR
 
     def __init__(
-        self, message: Optional[str], internal_message: Optional[str] = None,
+        self,
+        message: Optional[str],
+        internal_message: Optional[str] = None,
     ):
         self.message: Optional[str] = message
         self.internal_message: Optional[str] = internal_message

src/enochecker/storeddict.py

@@ -1,6 +1,5 @@
 """Backend for team_db based on a local filesystem directory."""
 
-import atexit
 import json
 import logging
 import os
@@ -11,11 +10,9 @@
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterator, Optional, Set
 
-from .utils import base64ify, debase64ify, ensure_valid_filename, start_daemon
+from .utils import base64ify, debase64ify, ensure_valid_filename
 
 logging.basicConfig(level=logging.DEBUG)
-dictlogger = logging.Logger(__name__)
-dictlogger.setLevel(logging.DEBUG)
 
 DB_DEFAULT_DIR = os.path.join(
     os.getcwd(), ".data"
@@ -86,7 +83,6 @@ def __init__(
         self,
         name: str = "default",
         base_path: str = DB_DEFAULT_DIR,
-        persist_secs: int = 3,
         ignore_locks: bool = False,
         logger: Optional[logging.Logger] = None,
         *args: Any,
@@ -99,61 +95,31 @@ def __init__(
 
         :param name: name of this store
         :param base_path: the base path
-        :param persist_secs: how often to persist dirty elements (0 means: never autostore. Call persist manually)
         :param ignore_locks: We usually write and read lock files before accessing the data.
                 This flag seaves them out.
         :param logger: The logger instance to log events to
         """
-        self._cache: Dict[str, Any] = {}
-        self._dirties: Set[str] = set()
+        self._cache: Dict[str, Any] = dict()
         self._locks: Set[str] = set()
         self._to_delete: Set[str] = set()
-        self._persist_thread: Optional[threading.Thread] = None
         self.name = name
         self.path: str = os.path.join(base_path, ensure_valid_filename(name))
-        self.persist_secs: int = persist_secs
         self.ignore_locks: bool = ignore_locks
 
         if logger:
             self.logger = logger
         else:
-            self.logger = dictlogger
+            self.logger = logging.Logger(__name__)
+            self.logger.setLevel(logging.DEBUG)
 
         self._local_lock = threading.RLock()
 
         makedirs(self.path, exist_ok=True)
-        atexit.register(self._cleanup)
-        self._stopping = False
 
         self.update(
             dict(*args, **kwargs)
         )  # In case we got initialized using a dict, make sure it's in sync.
 
-    @_locked
-    def _spawn_persist_thread(self) -> None:
-        """Spawn a thread persisting all changes, if necessary."""
-        if self.persist_secs > 0 and not self._persist_thread:
-
-            def persist_async() -> None:
-                time.sleep(self.persist_secs)
-                self.logger.debug("Persisting db {} from background.".format(self.name))
-                self.persist()
-
-            self._persist_thread = start_daemon(persist_async)
-
-    @_locked
-    def _cleanup(self) -> None:
-        """Clean up the db: persists and releases all locks currently held."""
-        self.logger.debug("StoredDict cleanup task running.")
-        self._stopping = True
-        self.persist()
-        for lock in self._locks:
-            self.release(lock)
-
-    def __del__(self) -> None:
-        """Delete a key."""
-        self._cleanup()
-
     def _dir(self, key: str) -> str:
         """
         Return the path where data for this key is stored.
@@ -256,8 +222,7 @@ def reload(self) -> None:
         Non persisted changes might be lost.
         Only reason would be if another process fiddles with our data concurrently.
         """
-        self._cache = {}
-        self._dirties = set()
+        self._cache = dict()
         self._to_delete = set()
 
     @_locked
@@ -277,7 +242,7 @@ def persist(self) -> None:
             self.logger.debug(f"Deleted {key} from db {self.name}")
         self._to_delete = set()
 
-        for key in self._dirties:
+        for key in self._cache.keys():
             locked = self.is_locked(key)
             if not locked:
                 self.lock(key)
@@ -288,7 +253,7 @@ def persist(self) -> None:
                 if not locked:
                     self.release(key)
             self.logger.debug(f"Set {key} in db {self.name}")
-        self._dirties = set()
+        self._cache = dict()
 
     @_locked
     def __getitem__(self, key: str) -> Any:
@@ -298,13 +263,6 @@ def __getitem__(self, key: str) -> Any:
         :param key: the key to look up
         :return: the value
         """
-        if key in self._to_delete:
-            raise KeyError("Key was marked for deletion: {}".format(key))
-        try:
-            return self._cache[key]
-        except KeyError:
-            self.logger.debug("Hitting disk for {}".format(key))
-
         locked = self.is_locked(key) or self.ignore_locks
         if not locked:
             self.lock(key)
@@ -317,40 +275,35 @@ def __getitem__(self, key: str) -> Any:
             if not locked:
                 self.release(key)
 
-        self._cache[key] = val
         return val
 
     @_locked
     def __setitem__(self, key: str, value: Any) -> None:
         """
-        Set an item. It'll be stored to disk on the next persist.
+        Set an item.
 
         :param key: Key to store
         :param value: Value to store
         """
-        if key in self._to_delete:
-            self._to_delete.remove(key)
         self._cache[key] = value
-        self._dirties.add(key)
+        self.persist()
 
     @_locked
     def __delitem__(self, key: str) -> None:
         """
-        Delete an item. It will be deleted from disk on the next .persist().
+        Delete an item.
 
         :param key: the key to delete
         """
         self._to_delete.add(key)
+        self.persist()
 
     def __iter__(self) -> Iterator[str]:
         """
         Return an iterator over the dict.
 
-        Implicitly persisting the data before reading.
-
         :return: An iterator containing all keys to a dict.
         """
-        self.persist()
         keys = [
             debase64ify(x[len(DB_PREFIX) : -len(DB_EXTENSION)], b"+-")
             for x in os.listdir(self.path)
@@ -362,11 +315,8 @@ def __len__(self) -> int:
         """
         Calculate the length of the dict.
 
-        Implicitly calls persist.
-
         :return: the the number of elements
         """
-        self.persist()
         keys = [
             x[len(DB_PREFIX) : -len(DB_EXTENSION)]
             for x in os.listdir(self.path)

tests/test_enochecker.py

@@ -5,8 +5,10 @@
 import time
 from logging import DEBUG
 
-import enochecker
 import pytest
+from enochecker_core import CheckerMethod, CheckerTaskMessage, CheckerTaskResult
+
+import enochecker
 from enochecker import (
     BaseChecker,
     BrokenServiceException,
@@ -18,7 +20,6 @@
     serve_once,
     snake_caseify,
 )
-from enochecker_core import CheckerMethod, CheckerTaskMessage, CheckerTaskResult
 
 STORAGE_DIR: str = "/tmp/enochecker_test"
 
@@ -70,6 +71,7 @@ def __init__(
                 round_length=60000,
                 task_chain_id=task_chain_id,
             ),
+            use_db_cache=False,
             **kwargs,
         )
         self.logger.setLevel(DEBUG)

tests/test_run_method.py

@@ -7,9 +7,10 @@
 
 import pytest
 import requests
+from enochecker_core import CheckerMethod, CheckerTaskMessage, CheckerTaskResult
+
 from enochecker import BaseChecker, BrokenServiceException, OfflineException
 from enochecker.results import CheckerResult
-from enochecker_core import CheckerMethod, CheckerTaskMessage, CheckerTaskResult
 
 
 @pytest.fixture()
@@ -21,7 +22,9 @@ class CheckerExampleImpl(BaseChecker):
         havoc_variants = 1
 
         def __init__(
-            self, method=CheckerMethod.CHECKER_METHOD_PUTFLAG, **kwargs,
+            self,
+            method=CheckerMethod.CHECKER_METHOD_PUTFLAG,
+            **kwargs,
         ):
             """
             An mocked implementation of a checker for testing purposes
@@ -42,7 +45,8 @@ def __init__(
                     timeout=30000,
                     round_length=60000,
                     task_chain_id="test",
-                )
+                ),
+                use_db_cache=False,
             )
             self.logger.setLevel(DEBUG)
 

tests/test_utils.py

@@ -2,6 +2,7 @@
 import sys
 
 import pytest
+
 from enochecker import BrokenServiceException
 from enochecker.utils import (
     assert_equals,