fix: misc minor tweaks

- Allow overriding CUMULUS_HUGGING_FACE_URL
- Show batch size with commas
- Change i2b2 argument name 'codebook' to 'display_codes' to avoid
  confusion with the de-id Codebook elsewhere in the ETL

compose.yaml

@@ -11,6 +11,7 @@ services:
       - AWS_SESSION_TOKEN
       - AWS_PROFILE
       - AWS_DEFAULT_PROFILE
+      - CUMULUS_HUGGING_FACE_URL
       - URL_CTAKES_REST=http://ctakes-covid:8080/ctakes-web-rest/service/analyze
       - URL_CNLP_NEGATION=http://cnlp-transformers:8000/negation/process
     volumes:

cumulus_etl/etl/cli.py

@@ -188,7 +188,7 @@ def print_config(args: argparse.Namespace, job_datetime: datetime.datetime, all_
     if args.errors_to:
         table.add_row("Errors path:", args.errors_to)
     table.add_row("Current time:", f"{common.timestamp_datetime(job_datetime)} UTC")
-    table.add_row("Batch size:", str(args.batch_size))
+    table.add_row("Batch size:", f"{args.batch_size:,}")
     table.add_row("Tasks:", ", ".join(sorted(t.name for t in all_tasks)))
     if args.comment:
         table.add_row("Comment:", args.comment)

cumulus_etl/loaders/i2b2/loader.py

@@ -63,7 +63,7 @@ def _load_all_with_extractors(
                 code_dict = json.load(code_json)
             self._loop(
                 conditions(),
-                partial(transform.to_fhir_condition, codebook=code_dict),
+                partial(transform.to_fhir_condition, display_codes=code_dict),
                 os.path.join(tmpdir.name, "Condition.ndjson"),
             )
 

cumulus_etl/loaders/i2b2/transform.py

@@ -175,7 +175,7 @@ def to_fhir_observation_vitals(obsfact: ObservationFact) -> dict:
     return observation
 
 
-def to_fhir_condition(obsfact: ObservationFact, codebook: dict) -> dict:
+def to_fhir_condition(obsfact: ObservationFact, display_codes: dict) -> dict:
     """
     :param obsfact: i2b2 observation fact containing ICD9, ICD10, or SNOMED
                     diagnosis
@@ -216,7 +216,7 @@ def to_fhir_condition(obsfact: ObservationFact, codebook: dict) -> dict:
         logging.warning("Condition: unknown system %s", i2b2_sys)
         i2b2_sys = "http://cumulus.smarthealthit.org/i2b2"
         i2b2_code = obsfact.concept_cd
-    condition["code"] = make_concept(i2b2_code, i2b2_sys, codebook=codebook)
+    condition["code"] = make_concept(i2b2_code, i2b2_sys, display_codes=display_codes)
 
     return condition
 
@@ -341,12 +341,12 @@ def get_observation_value(obsfact: ObservationFact) -> dict:
     return {"valueQuantity": quantity}
 
 
-def make_concept(code: str, system: str | None, display: str = None, codebook: dict = None) -> dict:
+def make_concept(code: str, system: str | None, display: str = None, display_codes: dict = None) -> dict:
     """Syntactic sugar to make a codeable concept"""
     coding = {"code": code, "system": system}
     if display:
         coding["display"] = display
-    elif codebook:
-        if system in codebook:
-            coding["display"] = codebook[system].get(code, "Unknown")
+    elif display_codes:
+        if system in display_codes:
+            coding["display"] = display_codes[system].get(code, "Unknown")
     return {"coding": [coding]}

tests/i2b2_mock_data.py

@@ -51,7 +51,7 @@ def condition_dim() -> transform.ObservationFact:
 
 def condition() -> dict:
     return transform.to_fhir_condition(
-        condition_dim(), codebook={"http://hl7.org/fhir/sid/icd-10-cm": {"U07.1": "COVID-19"}}
+        condition_dim(), display_codes={"http://hl7.org/fhir/sid/icd-10-cm": {"U07.1": "COVID-19"}}
     )