Tokenizer fix (#234)

Co-authored-by: Arkadiusz Kwasigroch <arkadiusz.kwasigroch@hpi.de>
som-shahlab · Feb 21, 2025 · a956059 · a956059
1 parent 985b9b3
commit a956059
Showing 1 changed file with 8 additions and 8 deletions.
diff --git a/src/femr/models/tokenizer.py b/src/femr/models/tokenizer.py
@@ -190,14 +190,14 @@ def convert_statistics_to_msgpack(
                 if start_val == end_val:
                     continue
 
-            entry = {
-                "type": "numeric",
-                "code_string": code,
-                "val_start": start_val,
-                "val_end": end_val,
-                "weight": weight * math.log(weight) + (1 - weight) * math.log(1 - weight),
-            }
-            vocab.append(entry)
+                entry = {
+                    "type": "numeric",
+                    "code_string": code,
+                    "val_start": start_val,
+                    "val_end": end_val,
+                    "weight": weight * math.log(weight) + (1 - weight) * math.log(1 - weight),
+                }
+                vocab.append(entry)
     else:
         assert ontology
         for code, weight in statistics["code_counts"].items():