diff --git a/README.md b/README.md index 4938ceb..0bfc005 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,12 @@ See [qualifier repo](https://github.com/sync-for-science/qualifier/blob/master/metrics.md) for some metric definitions. +## Installing + +```sh +pip install cumulus-library-data-metrics +``` + ## Running the Metrics These metrics are designed as a @@ -25,12 +31,11 @@ root/ Here's a sample command to run against that pile of ndjson data: ```sh -PYTHONPATH=. cumulus-library build \ +cumulus-library build \ --db-type duckdb \ --database output-tables.db \ --load-ndjson-dir path/to/ndjson/root \ - --target data_metrics \ - --study-dir . + --target data_metrics ``` And then you can load `output-tables.db` in a DuckDB session and see the results. @@ -39,12 +44,11 @@ Or read below to export the counts tables. ### Athena Here's a sample command to run against your Cumulus data in Athena: ```sh -PYTHONPATH=. cumulus-library build \ +cumulus-library build \ --database your-glue-database \ --workgroup your-athena-workgroup \ --profile your-aws-credentials-profile \ - --target data_metrics \ - --study-dir . + --target data_metrics ``` And then you can see the resulting tables in Athena. @@ -62,8 +66,7 @@ cumulus-library export \ ./output-folder \ --db-type duckdb \ --database output-tables.db \ - --target data_metrics \ - --study-dir . + --target data_metrics ``` #### Aggregate counts @@ -77,7 +80,6 @@ That is, run it like: ```sh env \ DATA_METRICS_OUTPUT_MODE=aggregate \ - PYTHONPATH=. \ cumulus-library build ... ``` @@ -101,35 +103,16 @@ Across the board, we have some minor differences from the Other specific deltas will be noted in the code for the given metric. -## Metric Prioritization - -### Table stakes quality: -- `q_term_use` complies with US Core v1 -- `q_ref_target_pop` complies with US Core v1 (can be run on partial extracts) -- `q_ref_target_valid` complies with US Core v1 (only on full extracts or data lake) -- `q_valid_us_core_v4` - - numerator: resources that don't have all mandatory bits of any profile - -### Table stakes characterization: -- `c_resource_count` (by category, year, month) -- `c_pt_count` (by birth year gender, ethnicity, race) -- `c_pt_deceased_count` (by gender, by age at death) -- `c_term_coverage` (by resource type, by category) -- `c_resources_per_pt` (include combinations?) -- `c_us_core_v4_count` - - Tells how many rows match mandatory US Core support - - And for each separate must-support requirement, tells which rows have the value - -### High value quality: -- `q_date_sequence` -- `q_date_in_lifetime` -- `q_date_recent` - -### High value characterization: -- `c_element_use` for USCDI v1 “must support” elements -- `c_date_precision` (by resource type, by category, by date element, by precision level) -- `c_identifier_coverage` (by resource type) - -### Useful quality: -- `q_obs_value_range` -- `q_obs_comp_value_range` +## Implemented Metrics + +- c_pt_count +- c_pt_deceased_count +- c_resource_count +- c_resources_per_pt +- c_term_coverage +- c_us_core_v4_count +- q_date_recent +- q_ref_target_pop +- q_ref_target_valid +- q_term_use +- q_valid_us_core_v4 diff --git a/cumulus_library_data_metrics/data_metrics/base.py b/cumulus_library_data_metrics/base.py similarity index 76% rename from cumulus_library_data_metrics/data_metrics/base.py rename to cumulus_library_data_metrics/base.py index 3df9e68..6cf02d9 100644 --- a/cumulus_library_data_metrics/data_metrics/base.py +++ b/cumulus_library_data_metrics/base.py @@ -5,10 +5,10 @@ from typing import ClassVar import jinja2 -from cumulus_library import databases -from cumulus_library.template_sql import base_templates +from cumulus_library import base_utils +from cumulus_library.template_sql import sql_utils -from cumulus_library_data_metrics.data_metrics import resource_info +from cumulus_library_data_metrics import resource_info class MetricMixin: @@ -31,9 +31,7 @@ def make_summary(self) -> None: sql = self.render_sql("../base.summary", entries=self.summary_entries, metric=self.name) self.queries.append(sql) - def _query_schema( - self, cursor: databases.DatabaseCursor, schema: str, parser: databases.DatabaseParser - ) -> None: + def _query_schema(self, config: base_utils.StudyConfig) -> None: fields_to_check = copy.deepcopy(self.uses_fields) # Since so many metrics use date data, add a standard date field into the mix @@ -44,11 +42,7 @@ def _query_schema( period = context.setdefault("period", {}) period["start"] = {} - for table, cols in fields_to_check.items(): - query = base_templates.get_column_datatype_query(schema, table.lower(), cols.keys()) - cursor.execute(query) - table_schema = cursor.fetchall() - self.schemas[table] = parser.validate_table_schema(cols, table_schema) + self.schemas = sql_utils.validate_schema(config.db, fields_to_check) if ( check_docref_period @@ -56,7 +50,7 @@ def _query_schema( ): self.date_fields["DocumentReference"].remove("context.period.start") - def extra_schema_checks(self, cursor: databases.DatabaseCursor, schema: str) -> None: + def extra_schema_checks(self, config: base_utils.StudyConfig) -> None: pass def add_metric_queries(self) -> None: @@ -64,14 +58,12 @@ def add_metric_queries(self) -> None: def prepare_queries( self, - cursor: databases.DatabaseCursor, - schema: str, *args, - parser: databases.DatabaseParser, + config: base_utils.StudyConfig, **kwargs, ) -> None: - self._query_schema(cursor, schema, parser) - self.extra_schema_checks(cursor, schema) + self._query_schema(config) + self.extra_schema_checks(config) self.add_metric_queries() def render_sql(self, template: str, **kwargs) -> str: diff --git a/cumulus_library_data_metrics/data_metrics/base.summary.jinja b/cumulus_library_data_metrics/base.summary.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/base.summary.jinja rename to cumulus_library_data_metrics/base.summary.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_pt_count/README.md b/cumulus_library_data_metrics/c_pt_count/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_pt_count/README.md rename to cumulus_library_data_metrics/c_pt_count/README.md diff --git a/cumulus_library_data_metrics/data_metrics/c_pt_count/c_pt_count.jinja b/cumulus_library_data_metrics/c_pt_count/c_pt_count.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_pt_count/c_pt_count.jinja rename to cumulus_library_data_metrics/c_pt_count/c_pt_count.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_pt_count/c_pt_count.py b/cumulus_library_data_metrics/c_pt_count/c_pt_count.py similarity index 92% rename from cumulus_library_data_metrics/data_metrics/c_pt_count/c_pt_count.py rename to cumulus_library_data_metrics/c_pt_count/c_pt_count.py index b423875..0bade28 100644 --- a/cumulus_library_data_metrics/data_metrics/c_pt_count/c_pt_count.py +++ b/cumulus_library_data_metrics/c_pt_count/c_pt_count.py @@ -4,7 +4,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class PatientCountBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/README.md b/cumulus_library_data_metrics/c_pt_deceased_count/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/README.md rename to cumulus_library_data_metrics/c_pt_deceased_count/README.md diff --git a/cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/c_pt_deceased_count.jinja b/cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/c_pt_deceased_count.jinja rename to cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/c_pt_deceased_count.py b/cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.py similarity index 87% rename from cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/c_pt_deceased_count.py rename to cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.py index 4607ab0..a9b86dd 100644 --- a/cumulus_library_data_metrics/data_metrics/c_pt_deceased_count/c_pt_deceased_count.py +++ b/cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.py @@ -2,7 +2,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class DeceasedCountBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/c_resource_count/README.md b/cumulus_library_data_metrics/c_resource_count/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_resource_count/README.md rename to cumulus_library_data_metrics/c_resource_count/README.md diff --git a/cumulus_library_data_metrics/data_metrics/c_resource_count/c_resource_count.jinja b/cumulus_library_data_metrics/c_resource_count/c_resource_count.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_resource_count/c_resource_count.jinja rename to cumulus_library_data_metrics/c_resource_count/c_resource_count.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_resource_count/c_resource_count.py b/cumulus_library_data_metrics/c_resource_count/c_resource_count.py similarity index 87% rename from cumulus_library_data_metrics/data_metrics/c_resource_count/c_resource_count.py rename to cumulus_library_data_metrics/c_resource_count/c_resource_count.py index dfdb0f0..4e7b923 100644 --- a/cumulus_library_data_metrics/data_metrics/c_resource_count/c_resource_count.py +++ b/cumulus_library_data_metrics/c_resource_count/c_resource_count.py @@ -2,8 +2,8 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics import resource_info -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics import resource_info +from cumulus_library_data_metrics.base import MetricMixin class ResourceCountBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/c_resources_per_pt/README.md b/cumulus_library_data_metrics/c_resources_per_pt/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_resources_per_pt/README.md rename to cumulus_library_data_metrics/c_resources_per_pt/README.md diff --git a/cumulus_library_data_metrics/data_metrics/c_resources_per_pt/c_resources_per_pt.jinja b/cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_resources_per_pt/c_resources_per_pt.jinja rename to cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_resources_per_pt/c_resources_per_pt.py b/cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.py similarity index 85% rename from cumulus_library_data_metrics/data_metrics/c_resources_per_pt/c_resources_per_pt.py rename to cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.py index 9ed147f..23f00b7 100644 --- a/cumulus_library_data_metrics/data_metrics/c_resources_per_pt/c_resources_per_pt.py +++ b/cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.py @@ -2,8 +2,8 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics import resource_info -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics import resource_info +from cumulus_library_data_metrics.base import MetricMixin class ResourcesPerPatientBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/c_term_coverage/README.md b/cumulus_library_data_metrics/c_term_coverage/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_term_coverage/README.md rename to cumulus_library_data_metrics/c_term_coverage/README.md diff --git a/cumulus_library_data_metrics/data_metrics/c_term_coverage/c_term_coverage.jinja b/cumulus_library_data_metrics/c_term_coverage/c_term_coverage.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_term_coverage/c_term_coverage.jinja rename to cumulus_library_data_metrics/c_term_coverage/c_term_coverage.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_term_coverage/c_term_coverage.py b/cumulus_library_data_metrics/c_term_coverage/c_term_coverage.py similarity index 94% rename from cumulus_library_data_metrics/data_metrics/c_term_coverage/c_term_coverage.py rename to cumulus_library_data_metrics/c_term_coverage/c_term_coverage.py index 5cb3077..980d72a 100644 --- a/cumulus_library_data_metrics/data_metrics/c_term_coverage/c_term_coverage.py +++ b/cumulus_library_data_metrics/c_term_coverage/c_term_coverage.py @@ -2,8 +2,8 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics import systems -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics import systems +from cumulus_library_data_metrics.base import MetricMixin # Note that this CUBE is already very large / slow. # Please do not add new columns to it. diff --git a/cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/README.md b/cumulus_library_data_metrics/c_us_core_v4_count/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/README.md rename to cumulus_library_data_metrics/c_us_core_v4_count/README.md diff --git a/cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/c_us_core_v4_count.jinja b/cumulus_library_data_metrics/c_us_core_v4_count/c_us_core_v4_count.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/c_us_core_v4_count.jinja rename to cumulus_library_data_metrics/c_us_core_v4_count/c_us_core_v4_count.jinja diff --git a/cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/c_us_core_v4_count.py b/cumulus_library_data_metrics/c_us_core_v4_count/c_us_core_v4_count.py similarity index 82% rename from cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/c_us_core_v4_count.py rename to cumulus_library_data_metrics/c_us_core_v4_count/c_us_core_v4_count.py index 98e3a29..2f0b969 100644 --- a/cumulus_library_data_metrics/data_metrics/c_us_core_v4_count/c_us_core_v4_count.py +++ b/cumulus_library_data_metrics/c_us_core_v4_count/c_us_core_v4_count.py @@ -2,7 +2,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.us_core_v4 import UsCoreV4Mixin +from cumulus_library_data_metrics.us_core_v4 import UsCoreV4Mixin class UsCoreV4CountBuilder(UsCoreV4Mixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/__init__.py b/cumulus_library_data_metrics/data_metrics/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cumulus_library_data_metrics/data_metrics/manifest.toml b/cumulus_library_data_metrics/manifest.toml similarity index 100% rename from cumulus_library_data_metrics/data_metrics/manifest.toml rename to cumulus_library_data_metrics/manifest.toml diff --git a/cumulus_library_data_metrics/data_metrics/meta/dates.jinja b/cumulus_library_data_metrics/meta/dates.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/meta/dates.jinja rename to cumulus_library_data_metrics/meta/dates.jinja diff --git a/cumulus_library_data_metrics/data_metrics/meta/meta.py b/cumulus_library_data_metrics/meta/meta.py similarity index 90% rename from cumulus_library_data_metrics/data_metrics/meta/meta.py rename to cumulus_library_data_metrics/meta/meta.py index c263cde..833c36a 100644 --- a/cumulus_library_data_metrics/data_metrics/meta/meta.py +++ b/cumulus_library_data_metrics/meta/meta.py @@ -2,7 +2,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class MetadataBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/meta/version.jinja b/cumulus_library_data_metrics/meta/version.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/meta/version.jinja rename to cumulus_library_data_metrics/meta/version.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_date_recent/README.md b/cumulus_library_data_metrics/q_date_recent/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_date_recent/README.md rename to cumulus_library_data_metrics/q_date_recent/README.md diff --git a/cumulus_library_data_metrics/data_metrics/q_date_recent/q_date_recent.jinja b/cumulus_library_data_metrics/q_date_recent/q_date_recent.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_date_recent/q_date_recent.jinja rename to cumulus_library_data_metrics/q_date_recent/q_date_recent.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_date_recent/q_date_recent.py b/cumulus_library_data_metrics/q_date_recent/q_date_recent.py similarity index 91% rename from cumulus_library_data_metrics/data_metrics/q_date_recent/q_date_recent.py rename to cumulus_library_data_metrics/q_date_recent/q_date_recent.py index 4236d1f..a3d102c 100644 --- a/cumulus_library_data_metrics/data_metrics/q_date_recent/q_date_recent.py +++ b/cumulus_library_data_metrics/q_date_recent/q_date_recent.py @@ -6,7 +6,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class DateRecentBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_pop/README.md b/cumulus_library_data_metrics/q_ref_target_pop/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_pop/README.md rename to cumulus_library_data_metrics/q_ref_target_pop/README.md diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_pop/q_ref_target_pop.jinja b/cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_pop/q_ref_target_pop.jinja rename to cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_pop/q_ref_target_pop.py b/cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.py similarity index 95% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_pop/q_ref_target_pop.py rename to cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.py index b96a122..cbd3fc6 100644 --- a/cumulus_library_data_metrics/data_metrics/q_ref_target_pop/q_ref_target_pop.py +++ b/cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.py @@ -2,7 +2,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class TargetPopBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_valid/README.md b/cumulus_library_data_metrics/q_ref_target_valid/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_valid/README.md rename to cumulus_library_data_metrics/q_ref_target_valid/README.md diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_valid/denominator.jinja b/cumulus_library_data_metrics/q_ref_target_valid/denominator.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_valid/denominator.jinja rename to cumulus_library_data_metrics/q_ref_target_valid/denominator.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_valid/q_ref_target_valid.jinja b/cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_valid/q_ref_target_valid.jinja rename to cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_ref_target_valid/q_ref_target_valid.py b/cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.py similarity index 97% rename from cumulus_library_data_metrics/data_metrics/q_ref_target_valid/q_ref_target_valid.py rename to cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.py index 282e69b..f50d305 100644 --- a/cumulus_library_data_metrics/data_metrics/q_ref_target_valid/q_ref_target_valid.py +++ b/cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.py @@ -4,7 +4,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class TargetValidBuilder(MetricMixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/q_term_use/README.md b/cumulus_library_data_metrics/q_term_use/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_term_use/README.md rename to cumulus_library_data_metrics/q_term_use/README.md diff --git a/cumulus_library_data_metrics/data_metrics/q_term_use/q_term_use.jinja b/cumulus_library_data_metrics/q_term_use/q_term_use.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_term_use/q_term_use.jinja rename to cumulus_library_data_metrics/q_term_use/q_term_use.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_term_use/q_term_use.py b/cumulus_library_data_metrics/q_term_use/q_term_use.py similarity index 92% rename from cumulus_library_data_metrics/data_metrics/q_term_use/q_term_use.py rename to cumulus_library_data_metrics/q_term_use/q_term_use.py index 8145ea6..953333a 100644 --- a/cumulus_library_data_metrics/data_metrics/q_term_use/q_term_use.py +++ b/cumulus_library_data_metrics/q_term_use/q_term_use.py @@ -2,8 +2,8 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics import systems -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics import systems +from cumulus_library_data_metrics.base import MetricMixin class TermUseBuilder(MetricMixin, BaseTableBuilder): @@ -71,8 +71,8 @@ def add_metric_queries(self) -> None: systems.CPT, systems.LOINC, systems.SNOMED, - "http://www.ada.org/cdt", - "https://www.cms.gov/Medicare/Coding/HCPCSReleaseCodeSets", + "http://ada.org/cdt", + "http://www.cms.gov/Medicare/Coding/HCPCSReleaseCodeSets", "http://www.cms.gov/Medicare/Coding/ICD10", ], ) diff --git a/cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/README.md b/cumulus_library_data_metrics/q_valid_us_core_v4/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/README.md rename to cumulus_library_data_metrics/q_valid_us_core_v4/README.md diff --git a/cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.jinja b/cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.jinja rename to cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.jinja diff --git a/cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.py b/cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.py similarity index 88% rename from cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.py rename to cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.py index 931a67e..663ceb1 100644 --- a/cumulus_library_data_metrics/data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.py +++ b/cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.py @@ -2,7 +2,7 @@ from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.us_core_v4 import UsCoreV4Mixin +from cumulus_library_data_metrics.us_core_v4 import UsCoreV4Mixin class ValidUsCoreV4Builder(UsCoreV4Mixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/resource_info.py b/cumulus_library_data_metrics/resource_info.py similarity index 97% rename from cumulus_library_data_metrics/data_metrics/resource_info.py rename to cumulus_library_data_metrics/resource_info.py index 1fad727..d28f7df 100644 --- a/cumulus_library_data_metrics/data_metrics/resource_info.py +++ b/cumulus_library_data_metrics/resource_info.py @@ -1,6 +1,6 @@ """Holds various static info about resources we want to examine.""" -from cumulus_library_data_metrics.data_metrics import systems +from cumulus_library_data_metrics import systems # Categories to slice on CATEGORIES = { @@ -40,8 +40,8 @@ DATES = { "AllergyIntolerance": ["recordedDate", "onsetDateTime", "onsetPeriod.start"], "Condition": ["recordedDate", "onsetDateTime", "onsetPeriod.start"], - "DocumentReference": ["context.period.start", "date"], "DiagnosticReport": ["effectiveDateTime", "effectivePeriod.start", "issued"], + "DocumentReference": ["context.period.start", "date"], "Encounter": ["period.start"], "Immunization": ["occurrenceDateTime", "recorded"], "MedicationRequest": ["authoredOn"], diff --git a/cumulus_library_data_metrics/data_metrics/systems.py b/cumulus_library_data_metrics/systems.py similarity index 100% rename from cumulus_library_data_metrics/data_metrics/systems.py rename to cumulus_library_data_metrics/systems.py diff --git a/cumulus_library_data_metrics/data_metrics/t_us_core_v4/README.md b/cumulus_library_data_metrics/t_us_core_v4/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/t_us_core_v4/README.md rename to cumulus_library_data_metrics/t_us_core_v4/README.md diff --git a/cumulus_library_data_metrics/data_metrics/t_us_core_v4/mandatory.jinja b/cumulus_library_data_metrics/t_us_core_v4/mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/t_us_core_v4/mandatory.jinja rename to cumulus_library_data_metrics/t_us_core_v4/mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/t_us_core_v4/must_support.jinja b/cumulus_library_data_metrics/t_us_core_v4/must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/t_us_core_v4/must_support.jinja rename to cumulus_library_data_metrics/t_us_core_v4/must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/t_us_core_v4/t_us_core_v4.py b/cumulus_library_data_metrics/t_us_core_v4/t_us_core_v4.py similarity index 90% rename from cumulus_library_data_metrics/data_metrics/t_us_core_v4/t_us_core_v4.py rename to cumulus_library_data_metrics/t_us_core_v4/t_us_core_v4.py index 83330f5..b35ae6c 100644 --- a/cumulus_library_data_metrics/data_metrics/t_us_core_v4/t_us_core_v4.py +++ b/cumulus_library_data_metrics/t_us_core_v4/t_us_core_v4.py @@ -3,7 +3,7 @@ import jinja2 from cumulus_library.base_table_builder import BaseTableBuilder -from cumulus_library_data_metrics.data_metrics.us_core_v4 import UsCoreV4Mixin +from cumulus_library_data_metrics.us_core_v4 import UsCoreV4Mixin class TestUsCoreV4Builder(UsCoreV4Mixin, BaseTableBuilder): diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/README.md b/cumulus_library_data_metrics/us_core_v4/README.md similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/README.md rename to cumulus_library_data_metrics/us_core_v4/README.md diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/__init__.py b/cumulus_library_data_metrics/us_core_v4/__init__.py similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/__init__.py rename to cumulus_library_data_metrics/us_core_v4/__init__.py diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/allergyintolerance_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/allergyintolerance_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/allergyintolerance_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/allergyintolerance_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/allergyintolerance_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/allergyintolerance_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/allergyintolerance_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/allergyintolerance_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/condition_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/condition_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/condition_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/condition_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/condition_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/condition_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/condition_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/condition_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/diagnosticreport_note_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/diagnosticreport_note_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/diagnosticreport_note_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/diagnosticreport_note_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/diagnosticreport_note_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/diagnosticreport_note_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/diagnosticreport_note_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/diagnosticreport_note_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/documentreference_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/documentreference_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/documentreference_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/documentreference_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/documentreference_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/documentreference_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/documentreference_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/documentreference_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/encounter_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/encounter_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/encounter_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/encounter_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/encounter_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/encounter_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/encounter_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/encounter_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/immunization_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/immunization_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/immunization_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/immunization_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/immunization_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/immunization_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/immunization_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/immunization_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/medication_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/medication_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/medication_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/medication_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/medication_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/medication_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/medication_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/medication_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/medicationrequest_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/medicationrequest_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/medicationrequest_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/medicationrequest_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/medicationrequest_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/medicationrequest_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/medicationrequest_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/medicationrequest_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_blood_pressure_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/observation_blood_pressure_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_blood_pressure_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_blood_pressure_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_laboratory_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/observation_laboratory_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_laboratory_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_laboratory_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_laboratory_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/observation_laboratory_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_laboratory_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_laboratory_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_smoking_status_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/observation_smoking_status_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_smoking_status_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_smoking_status_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_smoking_status_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/observation_smoking_status_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_smoking_status_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_smoking_status_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_utils.jinja b/cumulus_library_data_metrics/us_core_v4/observation_utils.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_utils.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_utils.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_vital_signs_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/observation_vital_signs_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_vital_signs_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_vital_signs_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/observation_vital_signs_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/observation_vital_signs_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/observation_vital_signs_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/observation_vital_signs_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/patient_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/patient_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/patient_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/patient_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/patient_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/patient_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/patient_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/patient_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/patient_utils.jinja b/cumulus_library_data_metrics/us_core_v4/patient_utils.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/patient_utils.jinja rename to cumulus_library_data_metrics/us_core_v4/patient_utils.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/procedure_mandatory.jinja b/cumulus_library_data_metrics/us_core_v4/procedure_mandatory.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/procedure_mandatory.jinja rename to cumulus_library_data_metrics/us_core_v4/procedure_mandatory.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/procedure_must_support.jinja b/cumulus_library_data_metrics/us_core_v4/procedure_must_support.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/procedure_must_support.jinja rename to cumulus_library_data_metrics/us_core_v4/procedure_must_support.jinja diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/profiles.py b/cumulus_library_data_metrics/us_core_v4/profiles.py similarity index 98% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/profiles.py rename to cumulus_library_data_metrics/us_core_v4/profiles.py index d5d8181..da3589a 100644 --- a/cumulus_library_data_metrics/data_metrics/us_core_v4/profiles.py +++ b/cumulus_library_data_metrics/us_core_v4/profiles.py @@ -2,7 +2,7 @@ from typing import ClassVar -from cumulus_library_data_metrics.data_metrics.base import MetricMixin +from cumulus_library_data_metrics.base import MetricMixin class UsCoreV4Mixin(MetricMixin): diff --git a/cumulus_library_data_metrics/data_metrics/us_core_v4/slice.jinja b/cumulus_library_data_metrics/us_core_v4/slice.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/us_core_v4/slice.jinja rename to cumulus_library_data_metrics/us_core_v4/slice.jinja diff --git a/cumulus_library_data_metrics/data_metrics/utils.jinja b/cumulus_library_data_metrics/utils.jinja similarity index 100% rename from cumulus_library_data_metrics/data_metrics/utils.jinja rename to cumulus_library_data_metrics/utils.jinja diff --git a/pyproject.toml b/pyproject.toml index 87b51a1..574c9be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "cumulus-library-data-metrics" requires-python = ">= 3.10" dependencies = [ - "cumulus-library >= 2.1, < 3", + "cumulus-library >= 2.2, < 3", ] description = "Data quality and characterization metrics for Cumulus" readme = "README.md" diff --git a/tests/test_metrics.py b/tests/test_metrics.py index ae7831b..192af3e 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -3,6 +3,7 @@ import glob import os import shutil +import sys import tempfile import unittest from unittest import mock @@ -113,8 +114,17 @@ def run_study(self, metric: str, test: str = "general", prefix: str = "") -> Non f"{tmpdir}/cumulus_library_data_metrics", ) + # Because we reload the data-metrics study from different paths each time, + # python might be keeping the stale imports from previous test builders around. + # Manually drop em here. + stale_modules = [ + mod for mod in sys.modules if mod.startswith("cumulus_library_data_metrics") + ] + for mod in stale_modules: + del sys.modules[mod] + # But change the manifest to only run one test metric, for speed reasons - manifest_file = f"{tmpdir}/cumulus_library_data_metrics/data_metrics/manifest.toml" + manifest_file = f"{tmpdir}/cumulus_library_data_metrics/manifest.toml" with open(manifest_file, "w", encoding="utf8") as f: f.write( f"""