Fixes #64. Added xxhash and now hashes the full data structure.

pyproject.toml

@@ -23,7 +23,8 @@ dependencies = [
     "pandas",
     "scikit-learn",
     "tqdm",
-    "requests"
+    "requests",
+    "xxhash"
 ]
 requires-python = ">=3.10"
 

tmu/util/encoded_data_cache.py

@@ -1,6 +1,5 @@
-import hashlib
 from typing import Optional
-
+import xxhash
 import numpy as np
 
 class DataEncoderCache:
@@ -12,10 +11,7 @@ def __init__(self, seed: int):
 
     def compute_hash(self, arr: np.ndarray) -> str:
         """Compute a hash for a numpy array."""
-
-        sampled_indices = self.rng.choice(arr.size, min(15, arr.size), replace=False)
-        sampled_values = arr.flat[sampled_indices]
-        return hashlib.sha256(sampled_values.tobytes()).hexdigest()
+        return xxhash.xxh3_64_hexdigest(arr)
 
     def get_encoded_data(self, data: np.ndarray, encoder_func) -> np.ndarray:
         """Get encoded data for an array, using cache if available."""