From 5e2dbd1f1c9a873355e45b5fc8a666c8cde15fc4 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 4 Feb 2025 10:43:34 -0500 Subject: [PATCH 1/6] feat: Demote PARTICIPANT_ID_MISMATCH to a warning for now --- packages/openneuro-app/src/scripts/workers/schema.worker.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/openneuro-app/src/scripts/workers/schema.worker.ts b/packages/openneuro-app/src/scripts/workers/schema.worker.ts index 197a767149..a68f1466f0 100644 --- a/packages/openneuro-app/src/scripts/workers/schema.worker.ts +++ b/packages/openneuro-app/src/scripts/workers/schema.worker.ts @@ -9,6 +9,12 @@ const config: Config = { { code: "SUBJECT_FOLDERS" }, // bids-standard/bids-specification#1928 downgrades to warning { code: "EMPTY_DATASET_NAME" }, ], + warning: [ + // Pending update to BIDS that enforces participants.tsv to be a superset + // of subject directories and phenotype participant_id columns + // https://github.com/bids-standard/bids-specification/issues/914 + { code: "PARTICIPANT_ID_MISMATCH" }, + ], } const options: ValidatorOptions = { From 2be2eda69eeb08d013d5fd13b7d56c84ca5a3f28 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 11 Feb 2025 12:02:25 -0500 Subject: [PATCH 2/6] feat: Sync datalad validation task configuration to app worker --- .../tasks/assets/validator-config.json | 11 +++++++++++ services/datalad/datalad_service/tasks/validator.py | 6 +++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 services/datalad/datalad_service/tasks/assets/validator-config.json diff --git a/services/datalad/datalad_service/tasks/assets/validator-config.json b/services/datalad/datalad_service/tasks/assets/validator-config.json new file mode 100644 index 0000000000..ada360a331 --- /dev/null +++ b/services/datalad/datalad_service/tasks/assets/validator-config.json @@ -0,0 +1,11 @@ +{ + error: [ + { code: "NO_AUTHORS" }, + { code: "SUBJECT_FOLDERS" }, + { code: "EMPTY_DATASET_NAME" }, + ], + warning: [ + { code: "PARTICIPANT_ID_MISMATCH" }, + ], + ignore: [], +} diff --git a/services/datalad/datalad_service/tasks/validator.py b/services/datalad/datalad_service/tasks/validator.py index 8fa705d4b0..22edaf9cef 100644 --- a/services/datalad/datalad_service/tasks/validator.py +++ b/services/datalad/datalad_service/tasks/validator.py @@ -1,7 +1,7 @@ import asyncio +from pathlib import Path import json import logging -import os import re import requests @@ -46,9 +46,13 @@ async def validate_dataset_deno_call(dataset_path, ref, logger=logger): Runs the deno bids-validator process. """ + # Sync to packages/openneuro-app/src/scripts/workers/schema.worker.ts + config_path = Path(__file__).parent / 'assets' / 'validator-config.json' return await run_and_decode( ['deno', 'run', '-A', f'jsr:@bids/validator@{DENO_VALIDATOR_VERSION}', + '--config', str(config_path), + '--blacklistModalities', 'micr', '--json', dataset_path], logger=logger ) From 54898f7a1ea0d88763538435096be7df5d8151e9 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Wed, 12 Feb 2025 11:07:34 -0800 Subject: [PATCH 3/6] chore: Make the linter happy about valid JSON for validator configuration --- .../tasks/assets/validator-config.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/datalad/datalad_service/tasks/assets/validator-config.json b/services/datalad/datalad_service/tasks/assets/validator-config.json index ada360a331..5bdce2ecf0 100644 --- a/services/datalad/datalad_service/tasks/assets/validator-config.json +++ b/services/datalad/datalad_service/tasks/assets/validator-config.json @@ -1,11 +1,11 @@ { - error: [ - { code: "NO_AUTHORS" }, - { code: "SUBJECT_FOLDERS" }, - { code: "EMPTY_DATASET_NAME" }, + "error": [ + { "code": "NO_AUTHORS" }, + { "code": "SUBJECT_FOLDERS" }, + { "code": "EMPTY_DATASET_NAME" } ], - warning: [ - { code: "PARTICIPANT_ID_MISMATCH" }, + "warning": [ + { "code": "PARTICIPANT_ID_MISMATCH" } ], - ignore: [], + "ignore": [] } From 20d2a8908fe23fc96ae72ba2827dd4e3d31eec87 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Wed, 12 Feb 2025 11:39:44 -0800 Subject: [PATCH 4/6] fix: Consolidate validator configuration to one JSON file and symlinks --- cli/src/commands/upload.ts | 2 ++ cli/src/validator-config.json | 1 + .../src/scripts/workers/schema.worker.ts | 15 ++------------- .../src/scripts/workers/validator-config.json | 1 + packages/openneuro-app/tsconfig.json | 2 +- validator-config.json | 11 +++++++++++ 6 files changed, 18 insertions(+), 14 deletions(-) create mode 120000 cli/src/validator-config.json create mode 120000 packages/openneuro-app/src/scripts/workers/validator-config.json create mode 100644 validator-config.json diff --git a/cli/src/commands/upload.ts b/cli/src/commands/upload.ts index 6f48dca6d9..b6566e5f6d 100644 --- a/cli/src/commands/upload.ts +++ b/cli/src/commands/upload.ts @@ -11,6 +11,7 @@ import type { CommandOptions } from "@cliffy/command" import { getRepoAccess } from "./git-credential.ts" import { readConfig } from "../config.ts" import { createDataset } from "../graphq.ts" +import validatorConfig from "../validator-config.json" with { type: "json" } async function getRepoDir(url: URL): Promise { const LOCAL_STORAGE_KEY = `openneuro_cli_${url.hostname}_` @@ -71,6 +72,7 @@ export async function uploadAction( const schemaResult = await validate( await readFileTree(dataset_directory_abs), options, + validatorConfig, ) console.log(consoleFormat(schemaResult)) diff --git a/cli/src/validator-config.json b/cli/src/validator-config.json new file mode 120000 index 0000000000..9b53d6989f --- /dev/null +++ b/cli/src/validator-config.json @@ -0,0 +1 @@ +../../validator-config.json \ No newline at end of file diff --git a/packages/openneuro-app/src/scripts/workers/schema.worker.ts b/packages/openneuro-app/src/scripts/workers/schema.worker.ts index a68f1466f0..82a25ec072 100644 --- a/packages/openneuro-app/src/scripts/workers/schema.worker.ts +++ b/packages/openneuro-app/src/scripts/workers/schema.worker.ts @@ -2,20 +2,9 @@ import { fileListToTree, validate } from "@bids/validator/main" import type { ValidationResult } from "@bids/validator/main" import type { Config, ValidatorOptions } from "@bids/validator/options" +import validatorConfig from "./validator-config.json" -const config: Config = { - error: [ - { code: "NO_AUTHORS" }, - { code: "SUBJECT_FOLDERS" }, // bids-standard/bids-specification#1928 downgrades to warning - { code: "EMPTY_DATASET_NAME" }, - ], - warning: [ - // Pending update to BIDS that enforces participants.tsv to be a superset - // of subject directories and phenotype participant_id columns - // https://github.com/bids-standard/bids-specification/issues/914 - { code: "PARTICIPANT_ID_MISMATCH" }, - ], -} +const config: Config = validatorConfig const options: ValidatorOptions = { datasetPath: "browser", diff --git a/packages/openneuro-app/src/scripts/workers/validator-config.json b/packages/openneuro-app/src/scripts/workers/validator-config.json new file mode 120000 index 0000000000..604f1a9334 --- /dev/null +++ b/packages/openneuro-app/src/scripts/workers/validator-config.json @@ -0,0 +1 @@ +../../../../../validator-config.json \ No newline at end of file diff --git a/packages/openneuro-app/tsconfig.json b/packages/openneuro-app/tsconfig.json index 19a8a01bf7..5c73b88bae 100644 --- a/packages/openneuro-app/tsconfig.json +++ b/packages/openneuro-app/tsconfig.json @@ -6,7 +6,7 @@ "tsBuildInfoFile": "../../.build-cache/app.tsbuildinfo" }, "include": ["./src"], - "files": ["./src/lerna.json"], + "files": ["./src/lerna.json", "./src/scripts/workers/validator-config.json"], "references": [ { "path": "../openneuro-components" } ] diff --git a/validator-config.json b/validator-config.json new file mode 100644 index 0000000000..5bdce2ecf0 --- /dev/null +++ b/validator-config.json @@ -0,0 +1,11 @@ +{ + "error": [ + { "code": "NO_AUTHORS" }, + { "code": "SUBJECT_FOLDERS" }, + { "code": "EMPTY_DATASET_NAME" } + ], + "warning": [ + { "code": "PARTICIPANT_ID_MISMATCH" } + ], + "ignore": [] +} From 13565b6b21aaf494e1a1038f629758da3372b369 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Wed, 12 Feb 2025 12:08:24 -0800 Subject: [PATCH 5/6] fix(worker): --blacklistModalities must be final argument --- services/datalad/datalad_service/tasks/validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/datalad/datalad_service/tasks/validator.py b/services/datalad/datalad_service/tasks/validator.py index 22edaf9cef..12ac05e49a 100644 --- a/services/datalad/datalad_service/tasks/validator.py +++ b/services/datalad/datalad_service/tasks/validator.py @@ -52,8 +52,8 @@ async def validate_dataset_deno_call(dataset_path, ref, logger=logger): ['deno', 'run', '-A', f'jsr:@bids/validator@{DENO_VALIDATOR_VERSION}', '--config', str(config_path), - '--blacklistModalities', 'micr', - '--json', dataset_path], + '--json', dataset_path, + '--blacklistModalities', 'micr'], logger=logger ) From 8477fd02d88bac95691855dd113d6c87539abf32 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Wed, 12 Feb 2025 15:39:42 -0800 Subject: [PATCH 6/6] fix: Remove validator-config symlink for Deno version and link app to Deno version --- cli/src/validator-config.json | 12 +++++++++++- .../src/scripts/workers/validator-config.json | 2 +- validator-config.json | 11 ----------- 3 files changed, 12 insertions(+), 13 deletions(-) mode change 120000 => 100644 cli/src/validator-config.json delete mode 100644 validator-config.json diff --git a/cli/src/validator-config.json b/cli/src/validator-config.json deleted file mode 120000 index 9b53d6989f..0000000000 --- a/cli/src/validator-config.json +++ /dev/null @@ -1 +0,0 @@ -../../validator-config.json \ No newline at end of file diff --git a/cli/src/validator-config.json b/cli/src/validator-config.json new file mode 100644 index 0000000000..5bdce2ecf0 --- /dev/null +++ b/cli/src/validator-config.json @@ -0,0 +1,11 @@ +{ + "error": [ + { "code": "NO_AUTHORS" }, + { "code": "SUBJECT_FOLDERS" }, + { "code": "EMPTY_DATASET_NAME" } + ], + "warning": [ + { "code": "PARTICIPANT_ID_MISMATCH" } + ], + "ignore": [] +} diff --git a/packages/openneuro-app/src/scripts/workers/validator-config.json b/packages/openneuro-app/src/scripts/workers/validator-config.json index 604f1a9334..d09d550f87 120000 --- a/packages/openneuro-app/src/scripts/workers/validator-config.json +++ b/packages/openneuro-app/src/scripts/workers/validator-config.json @@ -1 +1 @@ -../../../../../validator-config.json \ No newline at end of file +../../../../../cli/src/validator-config.json \ No newline at end of file diff --git a/validator-config.json b/validator-config.json deleted file mode 100644 index 5bdce2ecf0..0000000000 --- a/validator-config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "error": [ - { "code": "NO_AUTHORS" }, - { "code": "SUBJECT_FOLDERS" }, - { "code": "EMPTY_DATASET_NAME" } - ], - "warning": [ - { "code": "PARTICIPANT_ID_MISMATCH" } - ], - "ignore": [] -}