diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 79d87d438..785f4413a 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -8,13 +8,13 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Python - uses: "actions/setup-python@v2" + uses: actions/setup-python@v5 with: - python-version: "3.7" + python-version: "3.12" - name: install sqlfluff - run: "pip install sqlfluff==1.4.5" + run: "pip install sqlfluff==3.5.0" - name: Get changed files id: get_file_changes uses: trilom/file-changes-action@v1.2.4 @@ -22,28 +22,23 @@ jobs: output: ' ' - name: Get changed .sql files in mimic-iv concepts folder id: get_files_to_lint - shell: bash -l {0} + shell: bash run: | - # Set the command in the $() brackets as an output to use in later steps - echo "lintees=$( - echo \ - $(echo ${{ steps.get_file_changes.outputs.files_modified }} | - tr -s ' ' '\n' | - grep -E '^mimic-iv/concepts/.*[.]sql$' | - tr -s '\n' ' ') \ - $(echo ${{ steps.get_file_changes.outputs.files_added }} | - tr -s ' ' '\n' | - grep -E '^mimic-iv/concepts/.*[.]sql$' | - tr -s '\n' ' ') - ) >> $GITHUB_OUTPUT" + # Compose list of changed SQL files under mimic-iv/concepts and export to step output + raw="${{ steps.get_file_changes.outputs.files_modified }} ${{ steps.get_file_changes.outputs.files_added }}" + # Check for mimic-iv SQL files which may have changed + filtered="$(printf '%s' "$raw" | tr -s ' ' '\n' | grep -E '^mimic-iv/concepts/.*[.]sql$' || true)" + # Turn this into a space separated list for the next step + files="$(printf '%s' "$filtered" | tr -s '\n' ' ')" + echo "lintees=${files}" >> "$GITHUB_OUTPUT" - name: Lint SQL files id: sqlfluff_json if: steps.get_files_to_lint.outputs.lintees != '' - shell: bash -l {0} + shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate - uses: yuzutech/annotations-action@v0.3.0 + uses: yuzutech/annotations-action@v0.5.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" diff --git a/.sqlfluff b/.sqlfluff index 28663d303..62eff0dde 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -1,20 +1,21 @@ -[sqlfluff:core] -rules = core,L019 -large_file_skip_byte_limit = 40000 +[sqlfluff] dialect = bigquery +large_file_skip_byte_limit = 40000 +exclude_rules = LT02 + +# Enable core rules plus explicit layout.commas (leading commas) +[sqlfluff:core] +rules = core,layout.commas [sqlfluff:layout:type:comma] line_position = leading -[sqlfluff:indentation] -indented_joins = false -indented_using_on = true -template_blocks_indent = false - -[sqlfluff:rules:L010] -# Keywords should be upper case +# Capitalisation rules +[sqlfluff:rules:capitalisation.keywords] capitalisation_policy = upper -[sqlfluff:rules:L030] -# Functions should be upper case -extended_capitalisation_policy = upper \ No newline at end of file +[sqlfluff:rules:capitalisation.functions] +extended_capitalisation_policy = upper + +[sqlfluff:rules:capitalisation.identifiers] +extended_capitalisation_policy = lower \ No newline at end of file diff --git a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh index c012f6575..df0afa8fa 100644 --- a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh +++ b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh @@ -27,7 +27,7 @@ else fi echo "Copying tables from ${SOURCE_DATASET} to ${TARGET_DATASET}." -for TABLE in `bq ls ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`; +for TABLE in `bq ls -n 500 ${PROJECT_ID}:${SOURCE_DATASET} | cut -d' ' -f3`; do # skip the first line of dashes if [[ "${TABLE:0:2}" == '--' ]]; then diff --git a/mimic-iv/concepts/firstday/first_day_bg_art.sql b/mimic-iv/concepts/firstday/first_day_bg_art.sql index fb6276e99..e70aaf453 100644 --- a/mimic-iv/concepts/firstday/first_day_bg_art.sql +++ b/mimic-iv/concepts/firstday/first_day_bg_art.sql @@ -28,7 +28,8 @@ SELECT , MIN(sodium) AS sodium_min, MAX(sodium) AS sodium_max FROM `physionet-data.mimiciv_icu.icustays` ie LEFT JOIN `physionet-data.mimiciv_derived.bg` bg - ON ie.subject_id = bg.subject_id + ON + ie.subject_id = bg.subject_id AND bg.specimen = 'ART.' AND bg.charttime >= DATETIME_SUB(ie.intime, INTERVAL '6' HOUR) AND bg.charttime <= DATETIME_ADD(ie.intime, INTERVAL '1' DAY) diff --git a/mimic-iv/concepts/medication/arb.sql b/mimic-iv/concepts/medication/arb.sql index d83ba10e6..d825cb8d8 100644 --- a/mimic-iv/concepts/medication/arb.sql +++ b/mimic-iv/concepts/medication/arb.sql @@ -1,16 +1,17 @@ -WITH arb_drug AS ( +WITH arb_drug AS ( SELECT DISTINCT drug , CASE - WHEN UPPER(drug) LIKE '%AZILSARTAN%' OR UPPER(drug) LIKE '%EDARBI%' THEN 1 - WHEN UPPER(drug) LIKE '%CANDESARTAN%' OR UPPER(drug) LIKE '%ATACAND%' THEN 1 - WHEN UPPER(drug) LIKE '%IRBESARTAN%' OR UPPER(drug) LIKE '%AVAPRO%' THEN 1 - WHEN UPPER(drug) LIKE '%LOSARTAN%' OR UPPER(drug) LIKE '%COZAAR%' THEN 1 - WHEN UPPER(drug) LIKE '%OLMESARTAN%' OR UPPER(drug) LIKE '%BENICAR%' THEN 1 - WHEN UPPER(drug) LIKE '%TELMISARTAN%' OR UPPER(drug) LIKE '%MICARDIS%' THEN 1 - WHEN UPPER(drug) LIKE '%VALSARTAN%' OR UPPER(drug) LIKE '%DIOVAN%' THEN 1 - WHEN UPPER(drug) LIKE '%SACUBITRIL%' OR UPPER(drug) LIKE '%ENTRESTO%' THEN 1 - ELSE 0 + WHEN UPPER(drug) LIKE '%AZILSARTAN%' OR UPPER(drug) LIKE '%EDARBI%' + OR UPPER(drug) LIKE '%CANDESARTAN%' OR UPPER(drug) LIKE '%ATACAND%' + OR UPPER(drug) LIKE '%IRBESARTAN%' OR UPPER(drug) LIKE '%AVAPRO%' + OR UPPER(drug) LIKE '%LOSARTAN%' OR UPPER(drug) LIKE '%COZAAR%' + OR UPPER(drug) LIKE '%OLMESARTAN%' OR UPPER(drug) LIKE '%BENICAR%' + OR UPPER(drug) LIKE '%TELMISARTAN%' OR UPPER(drug) LIKE '%MICARDIS%' + OR UPPER(drug) LIKE '%VALSARTAN%' OR UPPER(drug) LIKE '%DIOVAN%' + OR UPPER(drug) LIKE '%SACUBITRIL%' OR UPPER(drug) LIKE '%ENTRESTO%' + THEN 1 + ELSE 0 END AS arb FROM `physionet-data.mimiciv_hosp.prescriptions` ) @@ -24,8 +25,7 @@ SELECT FROM `physionet-data.mimiciv_hosp.prescriptions` pr INNER JOIN arb_drug - ON - pr.drug = arb_drug.drug + ON pr.drug = arb_drug.drug WHERE arb_drug.arb = 1 ;