diff --git a/README.md b/README.md index e6e8d4ee..6072ddc5 100644 --- a/README.md +++ b/README.md @@ -145,10 +145,14 @@ You can find launch examples in the [workflow demo](https://github.com/JBZoo/Csv # Required: true report: 'table' - # Apply all schemas (also without `filename_pattern`) to all CSV files found. - # Default value: 'no' + # Apply all schemas (also without `filename_pattern`) to all CSV files found as global rules. + # Available options: + # auto: If no glob pattern (*) is used for --schema, the schema is applied to all found CSV files. + # yes: Apply all schemas to all CSV files, Schemas without `filename_pattern` are applied as a global rule. + # no: Apply only schemas with not empty `filename_pattern` and match the CSV files. + # Default value: 'auto' # Required: true - apply-all: 'no' + apply-all: 'auto' # Quick mode. It will not validate all rows. It will stop after the first error. # Default value: 'no' @@ -1446,7 +1450,13 @@ Options: -S, --skip-schema[=SKIP-SCHEMA] Skips schema validation for quicker checks when the schema's correctness is certain. Use any non-empty value or "yes" to activate [default: "no"] - -a, --apply-all[=APPLY-ALL] Apply global schemas (also without `filename_pattern`) to all CSV files found. [default: "no"] + -a, --apply-all[=APPLY-ALL] Apply all schemas (also without `filename_pattern`) to all CSV files found as global rules. + Available options: + - auto: If no glob pattern (*) is used for --schema, the schema is applied to all found CSV files. + - yes|y|1: Apply all schemas to all CSV files, Schemas without `filename_pattern` are applied as a global rule. + - no|n|0: Apply only schemas with not empty `filename_pattern` and match the CSV files. + Note. If specify the option `--apply-all` without value, it will be treated as "yes". + [default: "auto"] -r, --report=REPORT Determines the report's output format. Available options: text, table, github, gitlab, teamcity, junit [default: "table"] @@ -1557,7 +1567,7 @@ discovered errors. This format is ideal for quick reviews and sharing with team CSV Blueprint: vX.Y.Z -Found Schemas : 1 +Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 diff --git a/action.yml b/action.yml index fd254b4f..9043b20c 100644 --- a/action.yml +++ b/action.yml @@ -36,8 +36,13 @@ inputs: default: table required: true apply-all: - description: 'Apply all schemas (also without `filename_pattern`) to all CSV files found.' - default: no + description: | + Apply all schemas (also without `filename_pattern`) to all CSV files found as global rules. + Available options: + auto: If no glob pattern (*) is used for --schema, the schema is applied to all found CSV files. + yes: Apply all schemas to all CSV files, Schemas without `filename_pattern` are applied as a global rule. + no: Apply only schemas with not empty `filename_pattern` and match the CSV files. + default: auto required: true quick: description: 'Quick mode. It will not validate all rows. It will stop after the first error.' diff --git a/src/Commands/ValidateCsv.php b/src/Commands/ValidateCsv.php index f4be6cf6..18f121fc 100644 --- a/src/Commands/ValidateCsv.php +++ b/src/Commands/ValidateCsv.php @@ -88,8 +88,19 @@ protected function configure(): void 'apply-all', 'a', InputOption::VALUE_OPTIONAL, - 'Apply global schemas (also without `filename_pattern`) to all CSV files found.', - 'no', + \implode("\n", [ + "Apply all schemas (also without `filename_pattern`) to all CSV files found as global rules.\n", + 'Available options:', + ' - auto: If no glob pattern (*) is used for --schema, the schema is applied to all ' . + 'found CSV files.', + ' - yes|y|1: Apply all schemas to all CSV files, Schemas without `filename_pattern` ' . + 'are applied as a global rule.', + ' - no|n|0: Apply only schemas with not empty `filename_pattern` and ' . + 'match the CSV files.', + 'Note. If specify the option `--apply-all` without value, it will be treated as "yes".', + '', + ]), + 'auto', ); parent::configure(); @@ -101,7 +112,7 @@ protected function executeAction(): int $csvFilenames = $this->findFiles('csv', false); $schemaFilenames = $this->findFiles('schema', false); - $matchedFiles = Utils::matchSchemaAndCsvFiles($csvFilenames, $schemaFilenames, $this->isApplyGlobal()); + $matchedFiles = Utils::matchSchemaAndCsvFiles($csvFilenames, $schemaFilenames, $this->isApplyAll()); $this->printHeaderInfo($csvFilenames, $schemaFilenames, $matchedFiles); @@ -119,18 +130,6 @@ protected function executeAction(): int ); } - protected function isApplyGlobal(): bool - { - $value = $this->getOptString('apply-all'); - return $value === '' || bool($value); - } - - private function isCheckingSchema(): bool - { - $value = $this->getOptString('skip-schema'); - return !($value === '' || bool($value)); - } - /** * @param SplFileInfo[] $schemaFilenames */ @@ -318,6 +317,37 @@ private function printSummary( return $exitCode; } + private function isApplyAll(): bool + { + $applyAll = \strtolower(\trim($this->getOptString('apply-all'))); + if (\in_array($applyAll, ['', 'yes', 'y', '1'], true)) { + return true; + } + + if (\in_array($applyAll, ['no', 'n', '0'], true)) { + return false; + } + + if ($applyAll === 'auto') { + $schemaPatterns = $this->getOptArray('schema'); + foreach ($schemaPatterns as $schema) { + if (\str_contains($schema, '*')) { // glob pattern found. + return false; + } + } + + return \count($schemaPatterns) === 1; // Only one schema file found without glob pattern. + } + + throw new Exception('Invalid value for --apply-all option: ' . $applyAll); + } + + private function isCheckingSchema(): bool + { + $value = $this->getOptString('skip-schema'); + return !($value === '' || bool($value)); + } + /** * @param SplFileInfo[] $csvFilenames * @param SplFileInfo[] $schemaFilenames @@ -325,8 +355,9 @@ private function printSummary( private function printHeaderInfo(array $csvFilenames, array $schemaFilenames, array $matchedFiles): void { $validationFlag = $this->isCheckingSchema() ? '' : ' (Validation skipped)'; + $applyAllFlag = $this->isApplyAll() ? ' (Apply All)' : ''; $this->out([ - 'Found Schemas : ' . \count($schemaFilenames) . $validationFlag, + 'Found Schemas : ' . \count($schemaFilenames) . $validationFlag . $applyAllFlag, 'Found CSV files : ' . \count($csvFilenames), 'Pairs by pattern: ' . $matchedFiles['count_pairs'], ]); diff --git a/tests/Commands/ValidateCsvApplyAllTest.php b/tests/Commands/ValidateCsvApplyAllTest.php new file mode 100644 index 00000000..133008ef --- /dev/null +++ b/tests/Commands/ValidateCsvApplyAllTest.php @@ -0,0 +1,270 @@ + './tests/fixtures/demo.csv', + 'schema' => [ + Tools::DEMO_YML_VALID, + './tests/schemas/demo_invalid_no_pattern.yml', + ], + ]); + + [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); + + $expected = <<<'TXT' + CSV Blueprint: Unknown version (PhpUnit) + Found Schemas : 2 + Found CSV files : 1 + Pairs by pattern: 1 + + Check schema syntax: 2 + (1/2) OK ./tests/schemas/demo_invalid_no_pattern.yml + (2/2) OK ./tests/schemas/demo_valid.yml + + CSV file validation: 1 + Schema: ./tests/schemas/demo_valid.yml + OK ./tests/fixtures/demo.csv; Size: 123.34 MB + + Summary: + 1 pairs (schema to csv) were found based on `filename_pattern`. + No issues in 2 schemas. + No issues in 1 CSV files. + Not used schemas: + * ./tests/schemas/demo_invalid_no_pattern.yml + + + TXT; + + isSame(1, $exitCode, $actual); + isSame($expected, $actual); + } + + public function testNoPatternApplyAllAutoNegativeGlob(): void + { + $optionsAsString = Tools::arrayToOptionString([ + 'csv' => './tests/fixtures/demo.csv', + 'schema' => './tests/schemas/demo_invalid_*.yml', + ]); + + [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); + + $expected = <<<'TXT' + CSV Blueprint: Unknown version (PhpUnit) + Found Schemas : 1 + Found CSV files : 1 + Pairs by pattern: 0 + + Check schema syntax: 1 + OK ./tests/schemas/demo_invalid_no_pattern.yml + + CSV file validation: 0 + + Summary: + 0 pairs (schema to csv) were found based on `filename_pattern`. + No issues in 1 schemas. + No issues in 1 CSV files. + No schema was applied to the CSV files (filename_pattern didn't match): + * ./tests/fixtures/demo.csv + Not used schemas: + * ./tests/schemas/demo_invalid_no_pattern.yml + + + TXT; + + isSame(1, $exitCode, $actual); + isSame($expected, $actual); + } + + public function testNoPatternApplyAllAutoPositive(): void + { + $optionsAsString = Tools::arrayToOptionString([ + 'csv' => './tests/fixtures/demo.csv', + 'schema' => './tests/schemas/demo_invalid_no_pattern.yml', + ]); + + [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); + + $expected = <<<'TXT' + CSV Blueprint: Unknown version (PhpUnit) + Found Schemas : 1 (Apply All) + Found CSV files : 1 + Pairs by pattern: 1 + + Check schema syntax: 1 + OK ./tests/schemas/demo_invalid_no_pattern.yml + + CSV file validation: 1 + Schema: ./tests/schemas/demo_invalid_no_pattern.yml + 2 issues in ./tests/fixtures/demo.csv; Size: 123.34 MB + +------+-----------+---------+---------------------------------------------------+ + | Line | id:Column | Rule | Message | + +------+-----------+---------+---------------------------------------------------+ + | 4 | 2:Float | num_min | The value "-177.90" is less than the expected "0" | + | 11 | 2:Float | num_min | The value "-200.1" is less than the expected "0" | + +------+-----------+---------+---------------------------------------------------+ + + Summary: + 1 pairs (schema to csv) were found based on `filename_pattern`. + No issues in 1 schemas. + Found 2 issues in 1 out of 1 CSV files. + Schemas have no filename_pattern and are applied to all CSV files found: + * ./tests/schemas/demo_invalid_no_pattern.yml + + + TXT; + + isSame(1, $exitCode, $actual); + isSame($expected, $actual); + } + + public function testNoPatternApplyAllYes(): void + { + $optionsAsString = Tools::arrayToOptionString([ + 'csv' => './tests/fixtures/demo.csv', + 'schema' => [Tools::DEMO_YML_VALID, './tests/schemas/demo_invalid_no_pattern.yml'], + 'apply-all' => 'yes', + ]); + + [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); + + $expected = <<<'TXT' + CSV Blueprint: Unknown version (PhpUnit) + Found Schemas : 2 (Apply All) + Found CSV files : 1 + Pairs by pattern: 2 + + Check schema syntax: 2 + (1/2) OK ./tests/schemas/demo_invalid_no_pattern.yml + (2/2) OK ./tests/schemas/demo_valid.yml + + CSV file validation: 2 + Schema: ./tests/schemas/demo_invalid_no_pattern.yml + (1/2) 2 issues in ./tests/fixtures/demo.csv; Size: 123.34 MB + +------+-----------+---------+---------------------------------------------------+ + | Line | id:Column | Rule | Message | + +------+-----------+---------+---------------------------------------------------+ + | 4 | 2:Float | num_min | The value "-177.90" is less than the expected "0" | + | 11 | 2:Float | num_min | The value "-200.1" is less than the expected "0" | + +------+-----------+---------+---------------------------------------------------+ + + Schema: ./tests/schemas/demo_valid.yml + (2/2) OK ./tests/fixtures/demo.csv; Size: 123.34 MB + + Summary: + 2 pairs (schema to csv) were found based on `filename_pattern`. + No issues in 2 schemas. + Found 2 issues in 1 out of 1 CSV files. + Schemas have no filename_pattern and are applied to all CSV files found: + * ./tests/schemas/demo_invalid_no_pattern.yml + + + TXT; + + isSame(1, $exitCode, $actual); + isSame($expected, $actual); + } + + public function testNoPatternApplyAllYesGlob(): void + { + $optionsAsString = Tools::arrayToOptionString([ + 'csv' => './tests/fixtures/demo.csv', + 'schema' => './tests/schemas/demo_invalid_*.yml', + 'apply-all' => 'yes', + ]); + + [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); + + $expected = <<<'TXT' + CSV Blueprint: Unknown version (PhpUnit) + Found Schemas : 1 (Apply All) + Found CSV files : 1 + Pairs by pattern: 1 + + Check schema syntax: 1 + OK ./tests/schemas/demo_invalid_no_pattern.yml + + CSV file validation: 1 + Schema: ./tests/schemas/demo_invalid_no_pattern.yml + 2 issues in ./tests/fixtures/demo.csv; Size: 123.34 MB + +------+-----------+---------+---------------------------------------------------+ + | Line | id:Column | Rule | Message | + +------+-----------+---------+---------------------------------------------------+ + | 4 | 2:Float | num_min | The value "-177.90" is less than the expected "0" | + | 11 | 2:Float | num_min | The value "-200.1" is less than the expected "0" | + +------+-----------+---------+---------------------------------------------------+ + + Summary: + 1 pairs (schema to csv) were found based on `filename_pattern`. + No issues in 1 schemas. + Found 2 issues in 1 out of 1 CSV files. + Schemas have no filename_pattern and are applied to all CSV files found: + * ./tests/schemas/demo_invalid_no_pattern.yml + + + TXT; + + isSame(1, $exitCode, $actual); + isSame($expected, $actual); + } + + public function testNoPatternApplyAllNo(): void + { + $optionsAsString = Tools::arrayToOptionString([ + 'csv' => './tests/fixtures/demo.csv', + 'schema' => './tests/schemas/demo_invalid_no_pattern.yml', + 'apply-all' => 'no', + ]); + + [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); + + $expected = <<<'TXT' + CSV Blueprint: Unknown version (PhpUnit) + Found Schemas : 1 + Found CSV files : 1 + Pairs by pattern: 0 + + Check schema syntax: 1 + OK ./tests/schemas/demo_invalid_no_pattern.yml + + CSV file validation: 0 + + Summary: + 0 pairs (schema to csv) were found based on `filename_pattern`. + No issues in 1 schemas. + No issues in 1 CSV files. + No schema was applied to the CSV files (filename_pattern didn't match): + * ./tests/fixtures/demo.csv + Not used schemas: + * ./tests/schemas/demo_invalid_no_pattern.yml + + + TXT; + + isSame(1, $exitCode, $actual); + isSame($expected, $actual); + } +} diff --git a/tests/Commands/ValidateCsvBasicTest.php b/tests/Commands/ValidateCsvBasicTest.php index 5208861c..c11fb335 100644 --- a/tests/Commands/ValidateCsvBasicTest.php +++ b/tests/Commands/ValidateCsvBasicTest.php @@ -34,7 +34,7 @@ public function testValidateOneCsvPositive(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 @@ -67,7 +67,7 @@ public function testValidateOneCsvNegative(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 @@ -105,7 +105,7 @@ public function testValidateOneCsvWithInvalidSchemaNegative(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 @@ -164,7 +164,7 @@ public function testInvalidSchemaNotMatched(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 0 @@ -212,7 +212,7 @@ public function testInvalidSchemaAndNotFoundCSV(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 0 Pairs by pattern: 0 @@ -257,7 +257,7 @@ public function testValidateOneCsvNoHeaderNegative(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 @@ -297,7 +297,7 @@ public function testNothingFound(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 0 + Found Schemas : 0 (Apply All) Found CSV files : 0 Pairs by pattern: 0 diff --git a/tests/Commands/ValidateCsvBatchCsvTest.php b/tests/Commands/ValidateCsvBatchCsvTest.php index fe8c1aec..2a03349a 100644 --- a/tests/Commands/ValidateCsvBatchCsvTest.php +++ b/tests/Commands/ValidateCsvBatchCsvTest.php @@ -40,7 +40,7 @@ public function testValidateManyCsvPositive(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 4 Pairs by pattern: 4 @@ -79,7 +79,7 @@ public function testValidateManyCsvNegative(): void $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 3 Pairs by pattern: 3 diff --git a/tests/Commands/ValidateCsvBatchSchemaTest.php b/tests/Commands/ValidateCsvBatchSchemaTest.php index a548e7d9..8c5b32ce 100644 --- a/tests/Commands/ValidateCsvBatchSchemaTest.php +++ b/tests/Commands/ValidateCsvBatchSchemaTest.php @@ -102,88 +102,4 @@ public function testMultiSchemaDiscovery(): void isSame(1, $exitCode, $actual); isSame($expected, $actual); } - - public function testNoPatternNoApplyGlobal(): void - { - $optionsAsString = Tools::arrayToOptionString([ - 'csv' => './tests/fixtures/demo.csv', - 'schema' => [Tools::DEMO_YML_VALID, './tests/schemas/demo_invalid_no_pattern.yml'], - ]); - - [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); - - $expected = <<<'TXT' - CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 2 - Found CSV files : 1 - Pairs by pattern: 1 - - Check schema syntax: 2 - (1/2) OK ./tests/schemas/demo_invalid_no_pattern.yml - (2/2) OK ./tests/schemas/demo_valid.yml - - CSV file validation: 1 - Schema: ./tests/schemas/demo_valid.yml - OK ./tests/fixtures/demo.csv; Size: 123.34 MB - - Summary: - 1 pairs (schema to csv) were found based on `filename_pattern`. - No issues in 2 schemas. - No issues in 1 CSV files. - Not used schemas: - * ./tests/schemas/demo_invalid_no_pattern.yml - - - TXT; - - isSame(1, $exitCode, $actual); - isSame($expected, $actual); - } - - public function testNoPatternApplyGlobal(): void - { - $optionsAsString = Tools::arrayToOptionString([ - 'csv' => './tests/fixtures/demo.csv', - 'schema' => [Tools::DEMO_YML_VALID, './tests/schemas/demo_invalid_no_pattern.yml'], - 'apply-all' => 'yes', - ]); - - [$actual, $exitCode] = Tools::virtualExecution('validate:csv', $optionsAsString); - - $expected = <<<'TXT' - CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 2 - Found CSV files : 1 - Pairs by pattern: 2 - - Check schema syntax: 2 - (1/2) OK ./tests/schemas/demo_invalid_no_pattern.yml - (2/2) OK ./tests/schemas/demo_valid.yml - - CSV file validation: 2 - Schema: ./tests/schemas/demo_invalid_no_pattern.yml - (1/2) 2 issues in ./tests/fixtures/demo.csv; Size: 123.34 MB - +------+-----------+---------+---------------------------------------------------+ - | Line | id:Column | Rule | Message | - +------+-----------+---------+---------------------------------------------------+ - | 4 | 2:Float | num_min | The value "-177.90" is less than the expected "0" | - | 11 | 2:Float | num_min | The value "-200.1" is less than the expected "0" | - +------+-----------+---------+---------------------------------------------------+ - - Schema: ./tests/schemas/demo_valid.yml - (2/2) OK ./tests/fixtures/demo.csv; Size: 123.34 MB - - Summary: - 2 pairs (schema to csv) were found based on `filename_pattern`. - No issues in 2 schemas. - Found 2 issues in 1 out of 1 CSV files. - Schemas have no filename_pattern and are applied to all CSV files found: - * ./tests/schemas/demo_invalid_no_pattern.yml - - - TXT; - - isSame(1, $exitCode, $actual); - isSame($expected, $actual); - } } diff --git a/tests/Commands/ValidateCsvQuickTest.php b/tests/Commands/ValidateCsvQuickTest.php index 44870b8d..d9235acd 100644 --- a/tests/Commands/ValidateCsvQuickTest.php +++ b/tests/Commands/ValidateCsvQuickTest.php @@ -30,7 +30,7 @@ public function testEnabled(): void { $expectedFull = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 3 Pairs by pattern: 3 Quick mode enabled! @@ -63,7 +63,7 @@ public function testDisabled(): void { $expectedFull = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 3 Pairs by pattern: 3 diff --git a/tests/Commands/ValidateCsvReportsTest.php b/tests/Commands/ValidateCsvReportsTest.php index cc3aef57..a2ae10ca 100644 --- a/tests/Commands/ValidateCsvReportsTest.php +++ b/tests/Commands/ValidateCsvReportsTest.php @@ -28,7 +28,7 @@ public function testDefault(): void { $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 @@ -77,7 +77,7 @@ public function testText(): void { $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 @@ -117,7 +117,7 @@ public function testGithub(): void { $expected = <<<'TXT' CSV Blueprint: Unknown version (PhpUnit) - Found Schemas : 1 + Found Schemas : 1 (Apply All) Found CSV files : 1 Pairs by pattern: 1 diff --git a/tests/GithubActionsTest.php b/tests/GithubActionsTest.php index 265cf570..683a6f9a 100644 --- a/tests/GithubActionsTest.php +++ b/tests/GithubActionsTest.php @@ -52,7 +52,7 @@ public function testGitHubActionsReadMe(): void 'csv' => './tests/**/*.csv', 'schema' => './tests/**/*.yml', 'report' => "'" . ErrorSuite::REPORT_DEFAULT . "'", - 'apply-all' => "'no'", + 'apply-all' => "'auto'", 'quick' => "'no'", 'skip-schema' => "'no'", 'extra' => "'options: --ansi'",