From e958f9604fb0e269e7213e77e19012335f2d9db4 Mon Sep 17 00:00:00 2001 From: Chris Knoll Date: Wed, 12 May 2021 21:48:55 -0400 Subject: [PATCH] Handle standard and non-standard concept sets in criteria. Fixes #160 --- .../builders/BuilderUtils.java | 14 ++-- .../CohortGeneration_5_0_0_Test.java | 46 +++++++++++ .../mixedConceptsetsExpression.json | 77 +++++++++++++++++++ .../mixedConceptsets_PREP.json | 37 +++++++++ .../mixedConceptsets_VERIFY.json | 10 +++ 5 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsetsExpression.json create mode 100644 src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_PREP.json create mode 100644 src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_VERIFY.json diff --git a/src/main/java/org/ohdsi/circe/cohortdefinition/builders/BuilderUtils.java b/src/main/java/org/ohdsi/circe/cohortdefinition/builders/BuilderUtils.java index cedbb439..47f57ec9 100644 --- a/src/main/java/org/ohdsi/circe/cohortdefinition/builders/BuilderUtils.java +++ b/src/main/java/org/ohdsi/circe/cohortdefinition/builders/BuilderUtils.java @@ -10,26 +10,26 @@ public abstract class BuilderUtils { - private final static String CODESET_JOIN_TEMPLATE = "JOIN #Codesets codesets on (@codesetClauses)"; - + private final static String CODESET_JOIN_TEMPLATE = "JOIN #Codesets %s on (%s = %s.concept_id and %s.codeset_id = %d)"; + private final static String STANARD_ALIAS = "cs"; + private final static String NON_STANARD_ALIAS = "cns"; + public static String getCodesetJoinExpression(Integer standardCodesetId, String standardConceptColumn, Integer sourceCodesetId, String sourceConceptColumn) { - final String codsetJoinClause = "(%s = codesets.concept_id and codesets.codeset_id = %d)"; String joinExpression = ""; - ArrayList codesetClauses = new ArrayList<>(); if (standardCodesetId != null) { - codesetClauses.add(String.format(codsetJoinClause, standardConceptColumn, standardCodesetId)); + codesetClauses.add(String.format(CODESET_JOIN_TEMPLATE, STANARD_ALIAS, standardConceptColumn, STANARD_ALIAS, STANARD_ALIAS, standardCodesetId)); } // conditionSourceConcept if (sourceCodesetId != null) { - codesetClauses.add(String.format(codsetJoinClause, sourceConceptColumn, sourceCodesetId)); + codesetClauses.add(String.format(CODESET_JOIN_TEMPLATE, NON_STANARD_ALIAS, sourceConceptColumn, NON_STANARD_ALIAS, NON_STANARD_ALIAS, sourceCodesetId)); } if (codesetClauses.size() > 0) { - joinExpression = StringUtils.replace(CODESET_JOIN_TEMPLATE, "@codesetClauses", StringUtils.join(codesetClauses, " AND ")); + joinExpression = StringUtils.join(codesetClauses, "\n"); } diff --git a/src/test/java/org/ohdsi/circe/cohortdefinition/CohortGeneration_5_0_0_Test.java b/src/test/java/org/ohdsi/circe/cohortdefinition/CohortGeneration_5_0_0_Test.java index 1860ea06..66403c7e 100644 --- a/src/test/java/org/ohdsi/circe/cohortdefinition/CohortGeneration_5_0_0_Test.java +++ b/src/test/java/org/ohdsi/circe/cohortdefinition/CohortGeneration_5_0_0_Test.java @@ -771,4 +771,50 @@ public void testCensorWindow() throws Exception { Assertion.assertEquals(expectedDataSet, actualDataSet); } + /** + * Other Tests + */ + @Test + public void testMixedConceptsets() throws Exception { + final String RESULTS_SCHEMA = "mixedConceptsets"; + final String[] testDataSetsPrep = new String[] { + "/datasets/vocabulary.json", + "/cohortgeneration/mixedConceptsets/mixedConceptsets_PREP.json" + }; + final IDatabaseConnection dbUnitCon = getConnection(); + + // prepare results schema for the specified options.resultSchema + prepareSchema(RESULTS_SCHEMA, RESULTS_DDL_PATH); + + // load test data into DB. + final IDataSet dsPrep = DataSetFactory.createDataSet(testDataSetsPrep); + DatabaseOperation.CLEAN_INSERT.execute(dbUnitCon, dsPrep); // clean load of the DB. Careful, clean means "delete the old stuff" + + CohortExpressionQueryBuilder.BuildExpressionQueryOptions options; + CohortExpression expression; + String cohortSql; + + // load the default expression: + // all drug exposure events, fixed offset of endDate + 31 days. + + // cohort 1 will use the default expression from JSON. + expression = CohortExpression.fromJson(ResourceHelper.GetResourceAsString("/cohortgeneration/mixedConceptsets/mixedConceptsetsExpression.json")); + options = buildExpressionQueryOptions(1, RESULTS_SCHEMA); + cohortSql = buildExpressionSql(expression, options); + // execute on database, expect no errors + jdbcTemplate.batchUpdate(SqlSplit.splitSql(cohortSql)); + + // Validate results + // Load actual records from cohort table + final ITable actualTable = dbUnitCon.createQueryTable(RESULTS_SCHEMA + ".cohort", String.format("SELECT * from %s ORDER BY cohort_definition_id, subject_id, cohort_start_date", RESULTS_SCHEMA + ".cohort")); + // Load expected data from an XML dataset + final String[] testDataSetsVerify = new String[] {"/cohortgeneration/mixedConceptsets/mixedConceptsets_VERIFY.json"}; + final IDataSet expectedDataSet = DataSetFactory.createDataSet(testDataSetsVerify); + final ITable expectedTable = expectedDataSet.getTable(RESULTS_SCHEMA + ".cohort"); + + // Assert actual database table match expected table + Assertion.assertEquals(expectedTable, actualTable); + + } + } diff --git a/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsetsExpression.json b/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsetsExpression.json new file mode 100644 index 00000000..0957bca8 --- /dev/null +++ b/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsetsExpression.json @@ -0,0 +1,77 @@ +{ + "ConceptSets": [ + { + "id": 0, + "name": "Standard Conceptset", + "expression": { + "items": [ + { + "concept": { + "CONCEPT_CLASS_ID": "Clinical Finding", + "CONCEPT_CODE": "C1P1", + "CONCEPT_ID": 2, + "CONCEPT_NAME": "Child 1 [Parent: 1]", + "DOMAIN_ID": "CONDITION", + "INVALID_REASON": "V", + "INVALID_REASON_CAPTION": "Valid", + "STANDARD_CONCEPT": "S", + "STANDARD_CONCEPT_CAPTION": "Standard", + "VOCABULARY_ID": "TestVocab" + }, + "includeDescendants": true + } + ] + } + },{ + "id": 1, + "name": "Non-Standard Conceptset", + "expression": { + "items": [ + { + "concept": { + "CONCEPT_CLASS_ID": "Clinical Finding", + "CONCEPT_CODE": "C2P1", + "CONCEPT_ID": 3, + "CONCEPT_NAME": "Child 2 [Parent 1]", + "DOMAIN_ID": "CONDITION", + "INVALID_REASON": "V", + "INVALID_REASON_CAPTION": "Valid", + "STANDARD_CONCEPT": "S", + "STANDARD_CONCEPT_CAPTION": "Standard", + "VOCABULARY_ID": "TestVocab" + }, + "includeDescendants": true + } + ] + } + } + ], + "PrimaryCriteria": { + "CriteriaList": [{ + "ConditionOccurrence": { + "CodesetId": 0, + "ConditionSourceConcept": 1 + } + }], + "ObservationWindow": { + "PriorDays": 0, + "PostDays": 0 + }, + "PrimaryCriteriaLimit": { + "Type": "First" + } + }, + "QualifiedLimit": { + "Type": "First" + }, + "ExpressionLimit": { + "Type": "First" + }, + "InclusionRules": [], + "CensoringCriteria": [], + "CollapseSettings": { + "CollapseType": "ERA", + "EraPad": 0 + }, + "CensorWindow": {} +} \ No newline at end of file diff --git a/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_PREP.json b/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_PREP.json new file mode 100644 index 00000000..2e1ac335 --- /dev/null +++ b/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_PREP.json @@ -0,0 +1,37 @@ +{ + "cdm.person": [ + { + "person_id":1, + "gender_concept_id":0, + "year_of_birth":0, + "race_concept_id":0, + "ethnicity_concept_id":0 + } + ], + "cdm.condition_occurrence": [ + { + "condition_occurrence_id": 1, + "person_id":1, + "condition_concept_id":2, + "condition_start_date":"2000-02-01", + "condition_type_concept_id":0 + }, + { + "condition_occurrence_id": 2, + "person_id":1, + "condition_concept_id":2, + "condition_source_concept_id":3, + "condition_start_date":"2000-07-01", + "condition_type_concept_id":0 + } + ], + "cdm.observation_period" : [ + { + "observation_period_id": 1, + "person_id":1, + "observation_period_start_date":"2000-01-01", + "observation_period_end_date":"2001-01-01", + "period_type_concept_id": 0 + } + ] +} \ No newline at end of file diff --git a/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_VERIFY.json b/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_VERIFY.json new file mode 100644 index 00000000..3fdfd32c --- /dev/null +++ b/src/test/resources/cohortgeneration/mixedConceptsets/mixedConceptsets_VERIFY.json @@ -0,0 +1,10 @@ +{ + "mixedConceptsets.cohort": [ + { + "cohort_definition_id":1, + "subject_id":1, + "cohort_start_date":"2000-07-01", + "cohort_end_date":"2001-01-01" + } + ] +} \ No newline at end of file