From c9e051422dc61a30b359bed98e25f46d830d69ac Mon Sep 17 00:00:00 2001 From: Bjarne CALLEWAERT Date: Fri, 8 Mar 2024 15:50:31 +0100 Subject: [PATCH 1/5] Make sort sensitivity opt-in and configurable --- README.md | 1 + src/cli/cli.ts | 11 +- .../sort-by-key.post-processor.ts | 16 ++- .../sort-by-key.post-processor.spec.ts | 122 +++++++++++++++++- 4 files changed, 145 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2ec7dde1..7ae96d8b 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ Output --format, -f Format [string] [choices: "json", "namespaced-json", "pot"] [default: "json"] --format-indentation, --fi Format indentation (JSON/Namedspaced JSON) [string] [default: "\t"] --sort, -s Sort strings in alphabetical order [boolean] + --sort-sensitivity, -ss Sensitivity when sorting strings [boolean] --clean, -c Remove obsolete strings after merge [boolean] --replace, -r Replace the contents of output file if it exists (Merges by default) [boolean] --strip-prefix, -sp Strip prefix from key [string] diff --git a/src/cli/cli.ts b/src/cli/cli.ts index ded7af5d..8d08dc23 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -80,6 +80,13 @@ const cli = await y describe: 'Sort strings in alphabetical order', type: 'boolean' }) + .option('sort-sensitivity', { + alias: 'ss', + describe: 'Sort sensitivitiy of strings', + type: 'string', + choices: ['base', 'accent', 'case', 'variant'], + default: undefined + }) .option('clean', { alias: 'c', describe: 'Remove obsolete strings after merge', @@ -118,7 +125,7 @@ const cli = await y describe: 'Strip a prefix from the extracted key', type: 'string' }) - .group(['format', 'format-indentation', 'sort', 'clean', 'replace', 'strip-prefix'], 'Output') + .group(['format', 'format-indentation', 'sort', 'sort-sensitivity', 'clean', 'replace', 'strip-prefix'], 'Output') .group(['key-as-default-value', 'null-as-default-value', 'string-as-default-value'], 'Extracted key value (defaults to empty string)') .conflicts('key-as-default-value', 'null-as-default-value') .example('$0 -i ./src-a/ -i ./src-b/ -o strings.json', 'Extract (ts, html) from multiple paths') @@ -167,7 +174,7 @@ if (cli.stripPrefix) { } if (cli.sort) { - postProcessors.push(new SortByKeyPostProcessor()); + postProcessors.push(new SortByKeyPostProcessor(cli.sortSensitivity)); } extractTask.setPostProcessors(postProcessors); diff --git a/src/post-processors/sort-by-key.post-processor.ts b/src/post-processors/sort-by-key.post-processor.ts index 6786b521..621287a3 100644 --- a/src/post-processors/sort-by-key.post-processor.ts +++ b/src/post-processors/sort-by-key.post-processor.ts @@ -4,8 +4,22 @@ import { PostProcessorInterface } from './post-processor.interface.js'; export class SortByKeyPostProcessor implements PostProcessorInterface { public name: string = 'SortByKey'; + public sortSensitivity: 'base' | 'accent' | 'case' | 'variant' | undefined = undefined; + + constructor(sortSensitivity: string | undefined) { + if (isOfTypeSortSensitivity(sortSensitivity)) { + this.sortSensitivity = sortSensitivity; + } else { + throw new Error(`Unknown sortSensitivity: ${sortSensitivity}`); + } + } + public process(draft: TranslationCollection, extracted: TranslationCollection, existing: TranslationCollection): TranslationCollection { - const compareFn = new Intl.Collator('en', {sensitivity: 'base'}).compare; + const compareFn = this.sortSensitivity ? new Intl.Collator('en', { sensitivity: this.sortSensitivity }).compare : undefined; return draft.sort(compareFn); } } + +function isOfTypeSortSensitivity(keyInput: string | undefined): keyInput is 'base' | 'accent' | 'case' | 'variant' | undefined { + return ['base', 'accent', 'case', 'variant'].includes(keyInput) || keyInput === undefined; +} diff --git a/tests/post-processors/sort-by-key.post-processor.spec.ts b/tests/post-processors/sort-by-key.post-processor.spec.ts index a023a86a..dac21124 100644 --- a/tests/post-processors/sort-by-key.post-processor.spec.ts +++ b/tests/post-processors/sort-by-key.post-processor.spec.ts @@ -4,11 +4,11 @@ import { PostProcessorInterface } from '../../src/post-processors/post-processor import { SortByKeyPostProcessor } from '../../src/post-processors/sort-by-key.post-processor.js'; import { TranslationCollection } from '../../src/utils/translation.collection.js'; -describe('SortByKeyPostProcessor', () => { +describe('SortByKeyPostProcessor - base sensitivity should sort lowercase and uppercase equally', () => { let processor: PostProcessorInterface; beforeEach(() => { - processor = new SortByKeyPostProcessor(); + processor = new SortByKeyPostProcessor('base'); }); it('should sort keys alphanumerically', () => { @@ -27,6 +27,15 @@ describe('SortByKeyPostProcessor', () => { b: {value: 'another value', sourceFiles: []}, z: {value: 'last value', sourceFiles: []} }); + + expect(Object.keys(processor.process(collection, extracted, existing).values)).to.deep.equal( + Object.keys({ + '9': {value: 'a numeric key', sourceFiles: [] }, + a: {value: 'a value', sourceFiles: [] }, + b: {value: 'another value', sourceFiles: [] }, + z: {value: 'last value', sourceFiles: [] } + }) + ); }); it('should perform case insensitive sorting', () => { @@ -47,6 +56,96 @@ describe('SortByKeyPostProcessor', () => { g: { value: 'letter g', sourceFiles: [] } }); + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ + c: {value: 'letter c', sourceFiles: [] }, + j: {value: 'letter j', sourceFiles: [] }, + b: {value: 'letter b', sourceFiles: [] }, + a: {value: 'letter a', sourceFiles: [] }, + h: {value: 'letter h', sourceFiles: [] }, + B: {value: 'letter B', sourceFiles: [] }, + H: {value: 'letter H', sourceFiles: [] }, + i: {value: 'letter i', sourceFiles: [] }, + C: {value: 'letter C', sourceFiles: [] }, + e: {value: 'letter e', sourceFiles: [] }, + f: {value: 'letter f', sourceFiles: [] }, + d: {value: 'letter d', sourceFiles: [] }, + A: {value: 'letter A', sourceFiles: [] }, + g: {value: 'letter g', sourceFiles: [] } + }); + + expect(Object.keys(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values)).to.deep.equal( + Object.keys({ + a: {value: 'letter a', sourceFiles: [] }, + A: {value: 'letter A', sourceFiles: [] }, + b: {value: 'letter b', sourceFiles: [] }, + B: {value: 'letter B', sourceFiles: [] }, + c: {value: 'letter c', sourceFiles: [] }, + C: {value: 'letter C', sourceFiles: [] }, + d: {value: 'letter d', sourceFiles: [] }, + e: {value: 'letter e', sourceFiles: [] }, + f: {value: 'letter f', sourceFiles: [] }, + g: {value: 'letter g', sourceFiles: [] }, + h: {value: 'letter h', sourceFiles: [] }, + H: {value: 'letter H', sourceFiles: [] }, + i: {value: 'letter i', sourceFiles: [] }, + j: {value: 'letter j', sourceFiles: [] } + }) + ); + }); +}); + +describe('SortByKeyPostProcessor - undefined sort sensitivity should sort lowercase and uppercase separatly', () => { + let processor: PostProcessorInterface; + + beforeEach(() => { + processor = new SortByKeyPostProcessor(undefined); + }); + + it('should sort keys alphanumerically', () => { + const collection = new TranslationCollection({ + z: {value: 'last value', sourceFiles: [] }, + a: {value: 'a value', sourceFiles: [] }, + '9': {value: 'a numeric key', sourceFiles: [] }, + b: {value: 'another value', sourceFiles: [] } + }); + const extracted = new TranslationCollection(); + const existing = new TranslationCollection(); + + expect(processor.process(collection, extracted, existing).values).to.deep.equal({ + '9': {value: 'a numeric key', sourceFiles: [] }, + a: {value: 'a value', sourceFiles: [] }, + b: {value: 'another value', sourceFiles: [] }, + z: {value: 'last value', sourceFiles: [] } + }); + + expect(Object.keys(processor.process(collection, extracted, existing).values)).to.deep.equal( + Object.keys({ + '9': {value: 'a numeric key', sourceFiles: [] }, + a: {value: 'a value', sourceFiles: [] }, + b: {value: 'another value', sourceFiles: [] }, + z: {value: 'last value', sourceFiles: [] } + }) + ); + }); + + it('should perform case sensitive sorting', () => { + const collection = new TranslationCollection({ + c: {value: 'letter c', sourceFiles: [] }, + j: {value: 'letter j', sourceFiles: [] }, + b: {value: 'letter b', sourceFiles: [] }, + a: {value: 'letter a', sourceFiles: [] }, + h: {value: 'letter h', sourceFiles: [] }, + B: {value: 'letter B', sourceFiles: [] }, + H: {value: 'letter H', sourceFiles: [] }, + i: {value: 'letter i', sourceFiles: [] }, + C: {value: 'letter C', sourceFiles: [] }, + e: {value: 'letter e', sourceFiles: [] }, + f: {value: 'letter f', sourceFiles: [] }, + d: {value: 'letter d', sourceFiles: [] }, + A: {value: 'letter A', sourceFiles: [] }, + g: {value: 'letter g', sourceFiles: [] } + }); + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ A: { value: 'letter A', sourceFiles: [] }, a: { value: 'letter a', sourceFiles: [] }, @@ -63,5 +162,24 @@ describe('SortByKeyPostProcessor', () => { i: { value: 'letter i', sourceFiles: [] }, j: { value: 'letter j', sourceFiles: [] } }); + + expect(Object.keys(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values)).to.deep.equal( + Object.keys({ + A: {value: 'letter A', sourceFiles: [] }, + B: {value: 'letter B', sourceFiles: [] }, + C: {value: 'letter C', sourceFiles: [] }, + H: {value: 'letter H', sourceFiles: [] }, + a: {value: 'letter a', sourceFiles: [] }, + b: {value: 'letter b', sourceFiles: [] }, + c: {value: 'letter c', sourceFiles: [] }, + d: {value: 'letter d', sourceFiles: [] }, + e: {value: 'letter e', sourceFiles: [] }, + f: {value: 'letter f', sourceFiles: [] }, + g: {value: 'letter g', sourceFiles: [] }, + h: {value: 'letter h', sourceFiles: [] }, + i: {value: 'letter i', sourceFiles: [] }, + j: {value: 'letter j', sourceFiles: [] } + }) + ); }); }); From aa85c6f6071200a338d80144f02602170c4c2483 Mon Sep 17 00:00:00 2001 From: Bjarne CALLEWAERT Date: Fri, 8 Mar 2024 16:03:15 +0100 Subject: [PATCH 2/5] Fix type of --sort-sensitivity parameter in Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ae96d8b..b86ffe16 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Output --format, -f Format [string] [choices: "json", "namespaced-json", "pot"] [default: "json"] --format-indentation, --fi Format indentation (JSON/Namedspaced JSON) [string] [default: "\t"] --sort, -s Sort strings in alphabetical order [boolean] - --sort-sensitivity, -ss Sensitivity when sorting strings [boolean] + --sort-sensitivity, -ss Sensitivity when sorting strings [string] --clean, -c Remove obsolete strings after merge [boolean] --replace, -r Replace the contents of output file if it exists (Merges by default) [boolean] --strip-prefix, -sp Strip prefix from key [string] From 6b72ec6662205be85f9437e3ad278827257143d6 Mon Sep 17 00:00:00 2001 From: Bjarne CALLEWAERT Date: Tue, 4 Jun 2024 10:16:12 +0200 Subject: [PATCH 3/5] Update unit tests for SortByKeyPostProcessor for better readability and added more scenarios --- .../sort-by-key.post-processor.ts | 2 + .../sort-by-key.post-processor.spec.ts | 478 ++++++++++++++---- 2 files changed, 374 insertions(+), 106 deletions(-) diff --git a/src/post-processors/sort-by-key.post-processor.ts b/src/post-processors/sort-by-key.post-processor.ts index 621287a3..b1fd13fb 100644 --- a/src/post-processors/sort-by-key.post-processor.ts +++ b/src/post-processors/sort-by-key.post-processor.ts @@ -4,6 +4,8 @@ import { PostProcessorInterface } from './post-processor.interface.js'; export class SortByKeyPostProcessor implements PostProcessorInterface { public name: string = 'SortByKey'; + // More information on sort sensitivity: https://tc39.es/ecma402/#sec-collator-comparestrings + // Passing undefined will be treated as 'variant' by default: https://tc39.es/ecma402/#sec-intl.collator public sortSensitivity: 'base' | 'accent' | 'case' | 'variant' | undefined = undefined; constructor(sortSensitivity: string | undefined) { diff --git a/tests/post-processors/sort-by-key.post-processor.spec.ts b/tests/post-processors/sort-by-key.post-processor.spec.ts index a4d5f7cd..29b2c016 100644 --- a/tests/post-processors/sort-by-key.post-processor.spec.ts +++ b/tests/post-processors/sort-by-key.post-processor.spec.ts @@ -4,46 +4,48 @@ import { PostProcessorInterface } from '../../src/post-processors/post-processor import { SortByKeyPostProcessor } from '../../src/post-processors/sort-by-key.post-processor.js'; import { TranslationCollection } from '../../src/utils/translation.collection.js'; -describe('SortByKeyPostProcessor - base sensitivity should sort lowercase and uppercase equally', () => { +describe('SortByKeyPostProcessor - should throw error if sort sensitivity is not known', () => { + it('should throw error', () => { + expect(() => new SortByKeyPostProcessor('invalidSortSensitivityOption')).throw('Unknown sortSensitivity: invalidSortSensitivityOption'); + }); +}); + +describe('SortByKeyPostProcessor - undefined sort sensitivity should sort as variant sort sensitivity', () => { let processor: PostProcessorInterface; beforeEach(() => { - processor = new SortByKeyPostProcessor('base'); + processor = new SortByKeyPostProcessor(undefined); }); it('should sort keys alphanumerically', () => { const collection = new TranslationCollection({ - z: {value: 'last value', sourceFiles: []}, - a: {value: 'a value', sourceFiles: []}, - '9': {value: 'a numeric key', sourceFiles: []}, - b: {value: 'another value', sourceFiles: []} + z: { value: 'last value', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + '9': { value: 'a numeric key', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] } }); const extracted = new TranslationCollection(); const existing = new TranslationCollection(); + // Assert all values are processed correctly expect(processor.process(collection, extracted, existing).values).to.deep.equal({ - '9': {value: 'a numeric key', sourceFiles: []}, - a: {value: 'a value', sourceFiles: []}, - b: {value: 'another value', sourceFiles: []}, - z: {value: 'last value', sourceFiles: []} + '9': { value: 'a numeric key', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] }, + z: { value: 'last value', sourceFiles: [] } }); - expect(Object.keys(processor.process(collection, extracted, existing).values)).to.deep.equal( - Object.keys({ - '9': {value: 'a numeric key', sourceFiles: [] }, - a: {value: 'a value', sourceFiles: [] }, - b: {value: 'another value', sourceFiles: [] }, - z: {value: 'last value', sourceFiles: [] } - }) - ); + // Assert all keys are in the correct order + expect(processor.process(collection, extracted, existing).keys()).toStrictEqual(['9', 'a', 'b', 'z']); }); - it('should perform case insensitive sorting', () => { + it('should perform variant sensitive sorting', () => { const collection = new TranslationCollection({ c: { value: 'letter c', sourceFiles: [] }, j: { value: 'letter j', sourceFiles: [] }, b: { value: 'letter b', sourceFiles: [] }, a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, h: { value: 'letter h', sourceFiles: [] }, B: { value: 'letter B', sourceFiles: [] }, H: { value: 'letter H', sourceFiles: [] }, @@ -56,130 +58,394 @@ describe('SortByKeyPostProcessor - base sensitivity should sort lowercase and up g: { value: 'letter g', sourceFiles: [] } }); + // Assert all values are processed correctly expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ - c: {value: 'letter c', sourceFiles: [] }, - j: {value: 'letter j', sourceFiles: [] }, - b: {value: 'letter b', sourceFiles: [] }, - a: {value: 'letter a', sourceFiles: [] }, - h: {value: 'letter h', sourceFiles: [] }, - B: {value: 'letter B', sourceFiles: [] }, - H: {value: 'letter H', sourceFiles: [] }, - i: {value: 'letter i', sourceFiles: [] }, - C: {value: 'letter C', sourceFiles: [] }, - e: {value: 'letter e', sourceFiles: [] }, - f: {value: 'letter f', sourceFiles: [] }, - d: {value: 'letter d', sourceFiles: [] }, - A: {value: 'letter A', sourceFiles: [] }, - g: {value: 'letter g', sourceFiles: [] } - }); - - expect(Object.keys(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values)).to.deep.equal( - Object.keys({ - a: {value: 'letter a', sourceFiles: [] }, - A: {value: 'letter A', sourceFiles: [] }, - b: {value: 'letter b', sourceFiles: [] }, - B: {value: 'letter B', sourceFiles: [] }, - c: {value: 'letter c', sourceFiles: [] }, - C: {value: 'letter C', sourceFiles: [] }, - d: {value: 'letter d', sourceFiles: [] }, - e: {value: 'letter e', sourceFiles: [] }, - f: {value: 'letter f', sourceFiles: [] }, - g: {value: 'letter g', sourceFiles: [] }, - h: {value: 'letter h', sourceFiles: [] }, - H: {value: 'letter H', sourceFiles: [] }, - i: {value: 'letter i', sourceFiles: [] }, - j: {value: 'letter j', sourceFiles: [] } - }) - ); + A: { value: 'letter A', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + c: { value: 'letter c', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).keys()).toStrictEqual([ + 'A', + 'B', + 'C', + 'H', + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'à' + ]); }); }); -describe('SortByKeyPostProcessor - undefined sort sensitivity should sort lowercase and uppercase separatly', () => { +describe('SortByKeyPostProcessor - base sensitivity should treat all base characters as equal', () => { let processor: PostProcessorInterface; beforeEach(() => { - processor = new SortByKeyPostProcessor(undefined); + processor = new SortByKeyPostProcessor('base'); }); it('should sort keys alphanumerically', () => { const collection = new TranslationCollection({ - z: {value: 'last value', sourceFiles: [] }, - a: {value: 'a value', sourceFiles: [] }, - '9': {value: 'a numeric key', sourceFiles: [] }, - b: {value: 'another value', sourceFiles: [] } + z: { value: 'last value', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + '9': { value: 'a numeric key', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] } }); const extracted = new TranslationCollection(); const existing = new TranslationCollection(); + // Assert all values are processed correctly expect(processor.process(collection, extracted, existing).values).to.deep.equal({ - '9': {value: 'a numeric key', sourceFiles: [] }, - a: {value: 'a value', sourceFiles: [] }, - b: {value: 'another value', sourceFiles: [] }, - z: {value: 'last value', sourceFiles: [] } + '9': { value: 'a numeric key', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] }, + z: { value: 'last value', sourceFiles: [] } }); - expect(Object.keys(processor.process(collection, extracted, existing).values)).to.deep.equal( - Object.keys({ - '9': {value: 'a numeric key', sourceFiles: [] }, - a: {value: 'a value', sourceFiles: [] }, - b: {value: 'another value', sourceFiles: [] }, - z: {value: 'last value', sourceFiles: [] } - }) - ); + // Assert all keys are in the correct order + expect(processor.process(collection, extracted, existing).keys()).toStrictEqual(['9', 'a', 'b', 'z']); }); - it('should perform case sensitive sorting', () => { + it('should perform base sensitive sorting', () => { const collection = new TranslationCollection({ - c: {value: 'letter c', sourceFiles: [] }, - j: {value: 'letter j', sourceFiles: [] }, - b: {value: 'letter b', sourceFiles: [] }, - a: {value: 'letter a', sourceFiles: [] }, - h: {value: 'letter h', sourceFiles: [] }, - B: {value: 'letter B', sourceFiles: [] }, - H: {value: 'letter H', sourceFiles: [] }, - i: {value: 'letter i', sourceFiles: [] }, - C: {value: 'letter C', sourceFiles: [] }, - e: {value: 'letter e', sourceFiles: [] }, - f: {value: 'letter f', sourceFiles: [] }, - d: {value: 'letter d', sourceFiles: [] }, - A: {value: 'letter A', sourceFiles: [] }, - g: {value: 'letter g', sourceFiles: [] } + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } }); + // Assert all values are processed correctly expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).keys()).toStrictEqual([ + 'a', + 'à', + 'A', + 'b', + 'B', + 'c', + 'C', + 'd', + 'e', + 'f', + 'g', + 'h', + 'H', + 'i', + 'j' + ]); + }); +}); + +describe('SortByKeyPostProcessor - accent sensitivity should sort treat lowercase and uppercase as equal but accents and diacretics as not equal', () => { + let processor: PostProcessorInterface; + + beforeEach(() => { + processor = new SortByKeyPostProcessor('accent'); + }); + + it('should sort keys alphanumerically', () => { + const collection = new TranslationCollection({ + z: { value: 'last value', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + '9': { value: 'a numeric key', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] } + }); + const extracted = new TranslationCollection(); + const existing = new TranslationCollection(); + + // Assert all values are processed correctly + expect(processor.process(collection, extracted, existing).values).to.deep.equal({ + '9': { value: 'a numeric key', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] }, + z: { value: 'last value', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, extracted, existing).keys()).toStrictEqual(['9', 'a', 'b', 'z']); + }); + + it('should perform accent sensitive sorting', () => { + const collection = new TranslationCollection({ + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } + }); + + // Assert all values are processed correctly + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).keys()).toStrictEqual([ + 'a', + 'A', + 'à', + 'b', + 'B', + 'c', + 'C', + 'd', + 'e', + 'f', + 'g', + 'h', + 'H', + 'i', + 'j' + ]); + }); +}); + +describe('SortByKeyPostProcessor - case sensitivity should treat lowercase and uppercase as not equal but accents and diacretics as equal', () => { + let processor: PostProcessorInterface; + + beforeEach(() => { + processor = new SortByKeyPostProcessor('case'); + }); + + it('should sort keys alphanumerically', () => { + const collection = new TranslationCollection({ + z: { value: 'last value', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + '9': { value: 'a numeric key', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] } + }); + const extracted = new TranslationCollection(); + const existing = new TranslationCollection(); + + // Assert all values are processed correctly + expect(processor.process(collection, extracted, existing).values).to.deep.equal({ + '9': { value: 'a numeric key', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] }, + z: { value: 'last value', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, extracted, existing).keys()).toStrictEqual(['9', 'a', 'b', 'z']); + }); + + it('should perform case sensitive sorting', () => { + const collection = new TranslationCollection({ c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } + }); + + // Assert all values are processed correctly + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, e: { value: 'letter e', sourceFiles: [] }, f: { value: 'letter f', sourceFiles: [] }, - g: { value: 'letter g', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).keys()).toStrictEqual([ + 'a', + 'à', + 'A', + 'b', + 'B', + 'c', + 'C', + 'd', + 'e', + 'f', + 'g', + 'h', + 'H', + 'i', + 'j' + ]); + }); +}); + +describe('SortByKeyPostProcessor - variant sensitivity should treat lowercase, uppercase, accents and diacretics as not equal', () => { + let processor: PostProcessorInterface; + + beforeEach(() => { + processor = new SortByKeyPostProcessor('variant'); + }); + + it('should sort keys alphanumerically', () => { + const collection = new TranslationCollection({ + z: { value: 'last value', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + '9': { value: 'a numeric key', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] } + }); + const extracted = new TranslationCollection(); + const existing = new TranslationCollection(); + + // Assert all values are processed correctly + expect(processor.process(collection, extracted, existing).values).to.deep.equal({ + '9': { value: 'a numeric key', sourceFiles: [] }, + a: { value: 'a value', sourceFiles: [] }, + b: { value: 'another value', sourceFiles: [] }, + z: { value: 'last value', sourceFiles: [] } + }); + + // Assert all keys are in the correct order + expect(processor.process(collection, extracted, existing).keys()).toStrictEqual(['9', 'a', 'b', 'z']); + }); + + it('should perform variant sensitive sorting', () => { + const collection = new TranslationCollection({ + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, + h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, H: { value: 'letter H', sourceFiles: [] }, + i: { value: 'letter i', sourceFiles: [] }, + C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } + }); + + // Assert all values are processed correctly + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values).to.deep.equal({ + c: { value: 'letter c', sourceFiles: [] }, + j: { value: 'letter j', sourceFiles: [] }, + b: { value: 'letter b', sourceFiles: [] }, + a: { value: 'letter a', sourceFiles: [] }, + à: { value: 'letter à', sourceFiles: [] }, h: { value: 'letter h', sourceFiles: [] }, + B: { value: 'letter B', sourceFiles: [] }, + H: { value: 'letter H', sourceFiles: [] }, i: { value: 'letter i', sourceFiles: [] }, - j: { value: 'letter j', sourceFiles: [] } + C: { value: 'letter C', sourceFiles: [] }, + e: { value: 'letter e', sourceFiles: [] }, + f: { value: 'letter f', sourceFiles: [] }, + d: { value: 'letter d', sourceFiles: [] }, + A: { value: 'letter A', sourceFiles: [] }, + g: { value: 'letter g', sourceFiles: [] } }); - expect(Object.keys(processor.process(collection, new TranslationCollection(), new TranslationCollection()).values)).to.deep.equal( - Object.keys({ - A: {value: 'letter A', sourceFiles: [] }, - B: {value: 'letter B', sourceFiles: [] }, - C: {value: 'letter C', sourceFiles: [] }, - H: {value: 'letter H', sourceFiles: [] }, - a: {value: 'letter a', sourceFiles: [] }, - b: {value: 'letter b', sourceFiles: [] }, - c: {value: 'letter c', sourceFiles: [] }, - d: {value: 'letter d', sourceFiles: [] }, - e: {value: 'letter e', sourceFiles: [] }, - f: {value: 'letter f', sourceFiles: [] }, - g: {value: 'letter g', sourceFiles: [] }, - h: {value: 'letter h', sourceFiles: [] }, - i: {value: 'letter i', sourceFiles: [] }, - j: {value: 'letter j', sourceFiles: [] } - }) - ); + // Assert all keys are in the correct order + expect(processor.process(collection, new TranslationCollection(), new TranslationCollection()).keys()).toStrictEqual([ + 'a', + 'A', + 'à', + 'b', + 'B', + 'c', + 'C', + 'd', + 'e', + 'f', + 'g', + 'h', + 'H', + 'i', + 'j' + ]); }); }); From 53755f4083fc19bf78389fab6b6d9dd32c3720ea Mon Sep 17 00:00:00 2001 From: Bjarne CALLEWAERT Date: Tue, 4 Jun 2024 10:47:12 +0200 Subject: [PATCH 4/5] Add README section dedicated to sorting of extracted keys --- README.md | 12 +++++++++++- src/cli/cli.ts | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b86ffe16..4e9eb618 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,16 @@ If you want to use spaces instead, you can do the following: ngx-translate-extract --input ./src --output ./src/i18n/en.json --format-indentation ' ' ``` +### Sorting + +Extracted keys are by default not sorted. You can enable sorting by using the `--sort` or `-s` flag. + +If sorting is enabled, the keys will be sorted using the default variant sort sensitivity. Other sort sensitivity options are also available using the `--sort-sensitivity` or `-s` flag: +- `base`: Strings that differ in base letters are unequal. For example `a !== b`, `a === á`, `a === A` +- `accent`: Strings that differ in base letters and accents are unequal. For example `a !== b`, `a !== á`, `a === A` +- `case`: Strings that differ in base letters or casing are unequal. For example `a !== b`, `a === á`, `a !== A` +- `variant`: Strings that differ in base letters, accents, or casing are unequal. For example `a !== b`, `a !== á`, `a !== A` + ### Marker function If you want to extract strings that are not passed directly to `NgxTranslate.TranslateService`'s @@ -110,7 +120,7 @@ Output --format, -f Format [string] [choices: "json", "namespaced-json", "pot"] [default: "json"] --format-indentation, --fi Format indentation (JSON/Namedspaced JSON) [string] [default: "\t"] --sort, -s Sort strings in alphabetical order [boolean] - --sort-sensitivity, -ss Sensitivity when sorting strings [string] + --sort-sensitivity, -ss Sensitivity when sorting strings (only when sort is enabled) [string] --clean, -c Remove obsolete strings after merge [boolean] --replace, -r Replace the contents of output file if it exists (Merges by default) [boolean] --strip-prefix, -sp Strip prefix from key [string] diff --git a/src/cli/cli.ts b/src/cli/cli.ts index 8d08dc23..1893563f 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -82,7 +82,7 @@ const cli = await y }) .option('sort-sensitivity', { alias: 'ss', - describe: 'Sort sensitivitiy of strings', + describe: 'Sort sensitivitiy of strings (only to be used when sorting)', type: 'string', choices: ['base', 'accent', 'case', 'variant'], default: undefined From 3f38fd23695858afc31b35940836d936359ff7a7 Mon Sep 17 00:00:00 2001 From: Bjarne CALLEWAERT Date: Tue, 4 Jun 2024 14:15:31 +0200 Subject: [PATCH 5/5] Fix type in Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbb0503e..839d43b4 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ ngx-translate-extract --input ./src --output ./src/i18n/en.json --format-indenta Extracted keys are by default not sorted. You can enable sorting by using the `--sort` or `-s` flag. -If sorting is enabled, the keys will be sorted using the default variant sort sensitivity. Other sort sensitivity options are also available using the `--sort-sensitivity` or `-s` flag: +If sorting is enabled, the keys will be sorted using the default variant sort sensitivity. Other sort sensitivity options are also available using the `--sort-sensitivity` or `-ss` flag: - `base`: Strings that differ in base letters are unequal. For example `a !== b`, `a === á`, `a === A` - `accent`: Strings that differ in base letters and accents are unequal. For example `a !== b`, `a !== á`, `a === A` - `case`: Strings that differ in base letters or casing are unequal. For example `a !== b`, `a === á`, `a !== A`