Mark rows excluded from ingestion

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "dash-dangerously-set-inner-html>=0.0.2",
     "tomli-w>=1.0.0",
     "waitress>=3.0.0",
+    "cachetools>=5.5.0",
 ]
 scripts = { InsightBoard = "InsightBoard:main" }
 dynamic = ["version"]

src/InsightBoard/__main__.py

@@ -1,3 +1,4 @@
+import logging
 import argparse
 from InsightBoard import main
 from InsightBoard.app import app
@@ -10,6 +11,7 @@
 
     if args.debug:
         # Dash launches a Flask development server
+        logging.basicConfig(level=logging.DEBUG)
         app.run(debug=True)
     else:
         # Otherwise, launch the production server

src/InsightBoard/database/database.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 from pyarrow import Table
 from datetime import datetime
-from functools import cache
+from cachetools import cached, TTLCache
 
 
 class DatabaseBackend(Enum):
@@ -71,6 +71,7 @@ def commit_tables(self, table_names: [str], datasets: [pd.DataFrame]):
         for table_name, df in zip(table_names, datasets, strict=True):
             self.commit_table(table_name, df)
 
+    @cached(TTLCache(maxsize=1, ttl=10))
     def get_primary_key(self, table_name: str):
         schema = self.get_table_schema(table_name)
         # Find field that has the 'PrimaryKey' set to 'True'
@@ -84,13 +85,14 @@ def get_primary_key(self, table_name: str):
             raise ValueError(f"Table '{table_name}' has more than one primary key.")
         return primary_key
 
+    @cached(TTLCache(maxsize=1, ttl=10))
     def get_primary_keys(self, table_name: str):
         primary_key = self.get_primary_key(table_name)
         if not primary_key:
             return []
         return self.read_table_column(table_name, primary_key).tolist()
 
-    @cache
+    @cached(TTLCache(maxsize=1, ttl=10))
     def get_table_schema(self, table_name: str):
         schema_filename = (
             Path(self.data_folder).parent / "schemas" / f"{table_name}.schema.json"