diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index 34230e8764..d68681a7e4 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -759,12 +759,12 @@ describe('InformationExtraction', () => { status: 'processing', state: { labeled: true, + match: null, withValue: true, withSuggestion: true, - match: false, hasContext: true, processing: true, - obsolete: false, + obsolete: true, error: false, }, }) @@ -1011,9 +1011,9 @@ describe('InformationExtraction', () => { error: 'Issue calculation suggestion', state: { labeled: true, + match: null, withValue: true, withSuggestion: false, - match: false, hasContext: false, processing: false, obsolete: false, diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index 5ed882491b..384f63b780 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -93,7 +93,7 @@ const fixtures: DBFixture = { fixtureFactory.ixExtractor('fungusKindExtractor', 'kind', ['fungusTemplate']), ], ixsuggestions: [ - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh1_en', 'existingExtractor', 'shared1', @@ -101,7 +101,7 @@ const fixtures: DBFixture = { 'F3', 'kind' ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh1_es', 'existingExtractor', 'shared1', @@ -110,7 +110,7 @@ const fixtures: DBFixture = { 'kind', { language: 'es' } ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh3_en', 'existingExtractor', 'shared3', @@ -118,7 +118,7 @@ const fixtures: DBFixture = { 'F5', 'kind' ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh4_en', 'fungusKindExtractor', 'shared4', @@ -126,7 +126,7 @@ const fixtures: DBFixture = { 'F7', 'kind' ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh4_es', 'fungusKindExtractor', 'shared4', diff --git a/app/api/suggestions/pipelineStages.ts b/app/api/suggestions/pipelineStages.ts index c222e9c315..26702c2e61 100644 --- a/app/api/suggestions/pipelineStages.ts +++ b/app/api/suggestions/pipelineStages.ts @@ -14,46 +14,23 @@ export const baseQueryFragment = (extractorId: ObjectId, ignoreProcessing = true }; export const filterFragments = { - labeled: { - _fragment: { 'state.labeled': true }, - match: { 'state.labeled': true, 'state.match': true }, - mismatch: { 'state.labeled': true, 'state.match': false }, - }, - nonLabeled: { - _fragment: { 'state.labeled': false }, - withSuggestion: { 'state.labeled': false, 'state.withSuggestion': true }, - noSuggestion: { 'state.labeled': false, 'state.withSuggestion': false }, - noContext: { 'state.labeled': false, 'state.hasContext': false }, - obsolete: { 'state.labeled': false, 'state.obsolete': true }, - others: { 'state.labeled': false, 'state.error': true }, - }, + labeled: { 'state.labeled': true }, + nonLabeled: { 'state.labeled': false }, + match: { 'state.match': true }, + mismatch: { 'state.match': false }, + obsolete: { 'state.obsolete': true }, + error: { 'state.error': true }, }; export const translateCustomFilter = (customFilter: SuggestionCustomFilter) => { const orFilters = []; - if (customFilter.labeled.match) { - orFilters.push(filterFragments.labeled.match); - } - if (customFilter.labeled.mismatch) { - orFilters.push(filterFragments.labeled.mismatch); - } - - if (customFilter.nonLabeled.withSuggestion) { - orFilters.push(filterFragments.nonLabeled.withSuggestion); - } + if (customFilter.labeled) orFilters.push(filterFragments.labeled); + if (customFilter.nonLabeled) orFilters.push(filterFragments.nonLabeled); + if (customFilter.match) orFilters.push(filterFragments.match); + if (customFilter.mismatch) orFilters.push(filterFragments.mismatch); + if (customFilter.obsolete) orFilters.push(filterFragments.obsolete); + if (customFilter.error) orFilters.push(filterFragments.error); - if (customFilter.nonLabeled.noSuggestion) { - orFilters.push(filterFragments.nonLabeled.noSuggestion); - } - if (customFilter.nonLabeled.noContext) { - orFilters.push(filterFragments.nonLabeled.noContext); - } - if (customFilter.nonLabeled.obsolete) { - orFilters.push(filterFragments.nonLabeled.obsolete); - } - if (customFilter.nonLabeled.others) { - orFilters.push(filterFragments.nonLabeled.others); - } return orFilters; }; diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index bd8fc83228..045d17352a 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -1,38 +1,34 @@ -import db from 'api/utils/testing_db'; +import { testingDB } from 'api/utils/testing_db'; import { SuggestionCustomFilter } from 'shared/types/suggestionType'; -import { factory, stateFilterFixtures } from './fixtures'; +import { factory as f, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; const blankCustomFilter: SuggestionCustomFilter = { - labeled: { - match: false, - mismatch: false, - }, - nonLabeled: { - withSuggestion: false, - noSuggestion: false, - noContext: false, - obsolete: false, - others: false, - }, + labeled: false, + nonLabeled: false, + match: false, + mismatch: false, + obsolete: false, + error: false, }; beforeAll(async () => { - await db.setupFixturesAndContext(stateFilterFixtures); + await testingDB.setupFixturesAndContext(stateFilterFixtures); await Suggestions.updateStates({}); }); -afterAll(async () => db.disconnect()); +afterAll(async () => testingDB.disconnect()); describe('suggestions with CustomFilters', () => { describe('get()', () => { it('should return all suggestions (except processing) when no custom filter is provided', async () => { const result = await Suggestions.get( { - extractorId: factory.id('test_extractor').toString(), + extractorId: f.id('test_extractor').toString(), }, {} ); + expect(result.suggestions.length).toBe(12); expect(result.suggestions).toMatchObject([ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, @@ -40,10 +36,10 @@ describe('suggestions with CustomFilters', () => { { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ]); @@ -52,7 +48,7 @@ describe('suggestions with CustomFilters', () => { it('should be able to paginate', async () => { const result = await Suggestions.get( { - extractorId: factory.id('test_extractor').toString(), + extractorId: f.id('test_extractor').toString(), }, { page: { number: 3, size: 2 } } ); @@ -64,163 +60,114 @@ describe('suggestions with CustomFilters', () => { it.each([ { - description: 'filtering for labeled - match', - customFilter: { - ...blankCustomFilter, - labeled: { - match: true, - mismatch: false, - }, - }, + description: 'filtering for labeled', + customFilter: { ...blankCustomFilter, labeled: true }, expectedSuggestions: [ { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, - ], - }, - { - description: 'filtering for labeled - mismatch', - customFilter: { - ...blankCustomFilter, - labeled: { - match: false, - mismatch: true, - }, - }, - expectedSuggestions: [ { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - withSuggestion', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - withSuggestion: true, - }, - }, + description: 'filtering for nonLabeled', + customFilter: { ...blankCustomFilter, nonLabeled: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, { sharedId: 'unlabeled-error', language: 'en' }, { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + { + description: 'filtering for match', + customFilter: { ...blankCustomFilter, match: true }, + expectedSuggestions: [ + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - noSuggestion', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - noSuggestion: true, - }, - }, + description: 'filtering for mismatch', + customFilter: { ...blankCustomFilter, mismatch: true }, expectedSuggestions: [ + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - withSuggestions and noSuggestion', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - withSuggestion: true, - noSuggestion: true, - }, - }, + description: 'filtering for obsolete', + customFilter: { ...blankCustomFilter, obsolete: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, - { sharedId: 'unlabeled-no-context', language: 'en' }, - { sharedId: 'unlabeled-no-context', language: 'es' }, - { sharedId: 'unlabeled-no-suggestion', language: 'en' }, - { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - noContext', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - noContext: true, - }, - }, + description: 'filtering for error', + customFilter: { ...blankCustomFilter, error: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-no-context', language: 'en' }, - { sharedId: 'unlabeled-no-context', language: 'es' }, - { sharedId: 'unlabeled-no-suggestion', language: 'en' }, - { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - obsolete', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - obsolete: true, - }, - }, + description: 'filtering for OR combinations like: error OR obsolete', + customFilter: { ...blankCustomFilter, error: true, obsolete: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - others', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - others: true, - }, - }, + description: 'filtering for OR combinations like: mismatch OR error', + customFilter: { ...blankCustomFilter, mismatch: true, error: true }, expectedSuggestions: [ + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, { sharedId: 'unlabeled-error', language: 'en' }, { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { - description: 'filtering for labeled - match and nonLabeled - obsolete', - customFilter: { - ...blankCustomFilter, - labeled: { - match: true, - mismatch: false, - }, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - obsolete: true, - }, - }, + description: 'filtering for OR combinations like: labeled OR match', + customFilter: { ...blankCustomFilter, labeled: true, match: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-obsolete', language: 'en' }, - { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - noSuggestion and nonLabeled - noContext', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - noSuggestion: true, - noContext: true, - }, - }, + description: + 'filtering for OR combinations of complimentary filters like: labeled OR nonLabeled, which would result in all suggestions', + customFilter: { ...blankCustomFilter, labeled: true, nonLabeled: true }, expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], @@ -229,35 +176,107 @@ describe('suggestions with CustomFilters', () => { 'should use the custom filter properly when $description', async ({ customFilter, expectedSuggestions }) => { const result = await Suggestions.get( - { - extractorId: factory.id('test_extractor').toString(), - customFilter, - }, + { extractorId: f.id('test_extractor').toString(), customFilter }, {} ); + + expect(result.suggestions.length).toBe(expectedSuggestions.length); expect(result.suggestions).toMatchObject(expectedSuggestions); } ); }); describe('aggreagate()', () => { - it('should return correct aggregation', async () => { - const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); - expect(result).toEqual({ - total: 12, - labeled: { - _count: 4, - match: 2, - mismatch: 2, - }, - nonLabeled: { - _count: 8, - withSuggestion: 6, - noSuggestion: 2, - noContext: 4, - obsolete: 2, - others: 2, - }, + it('should return count of labeled and non labeled suggestions', async () => { + await testingDB.setupFixturesAndContext({ + ixsuggestions: [ + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { labeled: true } }), + f.ixSuggestion({ + extractorId: f.id('another_extractor'), + state: { labeled: true }, + }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { labeled: false } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { labeled: false } }), + ], + }); + + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 3, + labeled: 1, + nonLabeled: 2, + }); + }); + + it('should return count of match and missmatch', async () => { + await testingDB.setupFixturesAndContext({ + ixsuggestions: [ + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { match: true } }), + f.ixSuggestion({ extractorId: f.id('another_extractor'), state: { match: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { match: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { match: false } }), + ], + }); + + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 3, + match: 2, + mismatch: 1, + }); + }); + + it('should return count of obsolete suggestions', async () => { + await testingDB.setupFixturesAndContext({ + ixsuggestions: [ + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { obsolete: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { obsolete: true } }), + f.ixSuggestion({ + extractorId: f.id('another_extractor'), + state: { obsolete: true }, + }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { obsolete: false } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: {} }), + ], + }); + + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 4, + obsolete: 2, + }); + }); + + it('should return count of errors in suggestions', async () => { + await testingDB.setupFixturesAndContext({ + ixsuggestions: [ + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { error: true } }), + f.ixSuggestion({ extractorId: f.id('another_extractor'), state: { error: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: {} }), + ], + }); + + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 2, + error: 1, + }); + }); + + it('should correctly return all zeroes if no suggestions found', async () => { + await testingDB.setupFixturesAndContext({ + ixsuggestions: [], + }); + + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 0, + labeled: 0, + nonLabeled: 0, + match: 0, + mismatch: 0, + obsolete: 0, + error: 0, }); }); }); diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts index 0c9d5e7820..14406dd055 100644 --- a/app/api/suggestions/specs/eventListeners.spec.ts +++ b/app/api/suggestions/specs/eventListeners.spec.ts @@ -99,7 +99,7 @@ const fixtures: DBFixture = { fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]), ], ixsuggestions: [ - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_prop_1_suggestion', 'extractor1', 'entity for new file', @@ -107,7 +107,7 @@ const fixtures: DBFixture = { 'entfile', 'extracted_property_1' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_prop_2_suggestion', 'extractor2', 'entity for new file', @@ -115,7 +115,7 @@ const fixtures: DBFixture = { 'entfile', 'extracted_property_2' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_title_suggestion', 'title_extractor', 'entity for new file', @@ -123,7 +123,7 @@ const fixtures: DBFixture = { 'entfile', 'title' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_select_suggestion', 'extractor6', 'entity for new file', @@ -131,7 +131,7 @@ const fixtures: DBFixture = { 'entfile', 'select_property' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_multiselect_suggestion', 'extractor7', 'entity for new file', @@ -139,7 +139,7 @@ const fixtures: DBFixture = { 'entfile', 'multiselect_property' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_relationship_suggestion', 'extractor8', 'entity for new file', diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index cb68ca189a..79dc3fc101 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1102,7 +1102,7 @@ const stateFilterFixtures: DBFixture = { factory.ixExtractor('unused_extractor', 'unused_prop', ['template1']), ], ixsuggestions: [ - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-match-suggestion-en', 'test_extractor', 'labeled-match', @@ -1116,7 +1116,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-match', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-match-suggestion-es', 'test_extractor', 'labeled-match', @@ -1130,7 +1130,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-match', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-mismatch-suggestion-en', 'test_extractor', 'labeled-mismatch', @@ -1144,7 +1144,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-mismatch-mismatch', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-mismatch-suggestion-es', 'test_extractor', 'labeled-mismatch', @@ -1158,7 +1158,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-mismatch-mismatch', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-suggestion-suggestion-en', 'test_extractor', 'unlabeled-no-suggestion', @@ -1172,7 +1172,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: '', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-suggestion-suggestion-es', 'test_extractor', 'unlabeled-no-suggestion', @@ -1186,7 +1186,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: '', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-context-suggestion-en', 'test_extractor', 'unlabeled-no-context', @@ -1200,7 +1200,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-unlabeled-no-context', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-context-suggestion-es', 'test_extractor', 'unlabeled-no-context', @@ -1214,7 +1214,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-unlabeled-no-context', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-obsolete-suggestion-en', 'test_extractor', 'unlabeled-obsolete', @@ -1229,7 +1229,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-obsolete', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-obsolete-suggestion-es', 'test_extractor', 'unlabeled-obsolete', @@ -1244,7 +1244,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-obsolete', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-processing-suggestion-en', 'test_extractor', 'unlabeled-processing', @@ -1259,7 +1259,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-processing', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-processing-suggestion-es', 'test_extractor', 'unlabeled-processing', @@ -1274,7 +1274,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-processing', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-error-suggestion-en', 'test_extractor', 'unlabeled-error', @@ -1290,7 +1290,7 @@ const stateFilterFixtures: DBFixture = { error: 'some error happened', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-error-suggestion-es', 'test_extractor', 'unlabeled-error', @@ -1306,7 +1306,7 @@ const stateFilterFixtures: DBFixture = { error: 'some error happened', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unusedsuggestion', 'unused_extractor', 'unused', @@ -1320,6 +1320,15 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-unused', } ), + factory.ixSuggestion({ + extractorId: factory.id('unused_extractor'), + state: { + labeled: true, + match: true, + obsolete: true, + error: true, + }, + }), ], }; diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index 68b97b1756..e5d92fcc7c 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -208,36 +208,24 @@ describe('suggestions routes', () => { filter: { extractorId: factory.id('enemy_extractor').toString(), customFilter: { - labeled: { - match: false, - mismatch: false, - }, - nonLabeled: { - withSuggestion: false, - noSuggestion: true, - noContext: false, - obsolete: false, - others: false, - }, + labeled: true, + nonLabeled: false, + match: false, + mismatch: false, + obsolete: false, + error: false, }, }, }) .expect(200); expect(response.body.suggestions).toEqual([ expect.objectContaining({ - entityTitle: 'Catwoman', - state: { - labeled: false, - withValue: false, - withSuggestion: false, - match: false, - hasContext: true, - obsolete: false, - processing: false, - error: false, - }, - suggestedValue: '', - currentValue: '', + entityTitle: 'The Penguin', + language: 'en', + }), + expect.objectContaining({ + entityTitle: 'The Penguin', + language: 'es', }), ]); }); @@ -427,21 +415,15 @@ describe('aggregation routes', () => { extractorId: factory.id('test_extractor').toString(), }) .expect(200); + expect(response.body).toEqual({ - total: 12, - labeled: { - _count: 4, - match: 2, - mismatch: 2, - }, - nonLabeled: { - _count: 8, - withSuggestion: 6, - noSuggestion: 2, - noContext: 4, - obsolete: 2, - others: 2, - }, + total: 14, + labeled: 4, + nonLabeled: 10, + match: 4, + mismatch: 4, + obsolete: 4, + error: 2, }); }); }); diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index cd54b6124d..09ac72ccce 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-statements */ import db from 'api/utils/testing_db'; import { @@ -6,6 +7,7 @@ import { IXSuggestionType, IXSuggestionsFilter, } from 'shared/types/suggestionType'; +import { ObjectId } from 'mongodb'; import { Suggestions } from '../suggestions'; import { factory, @@ -20,7 +22,6 @@ import { selectAcceptanceFixtureBase, relationshipAcceptanceFixtureBase, } from './fixtures'; -import { ObjectId } from 'mongodb'; const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => Suggestions.get(filter, { page: { size, number: 1 } }); diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 05db7b7ab8..3f57f55fd7 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { ObjectId } from 'mongodb'; import { files } from 'api/files/files'; @@ -28,14 +29,11 @@ import { import { Extractors } from 'api/services/informationextraction/ixextractors'; import { registerEventListeners } from './eventListeners'; import { - baseQueryFragment, - filterFragments, getCurrentValueStage, getEntityStage, getFileStage, getLabeledValueStage, getMatchStage, - groupByAndCount, } from './pipelineStages'; import { postProcessCurrentValues, updateStates } from './updateState'; import { @@ -44,6 +42,8 @@ import { updateEntitiesWithSuggestion, } from './updateEntities'; +const DEFAULT_LIMIT = 30; + const updateExtractedMetadata = async ( suggestions: IXSuggestionType[], property: PropertySchema @@ -90,10 +90,12 @@ const buildListQuery = ( extractorId: ObjectId, customFilter: SuggestionCustomFilter | undefined, setLanguages: LanguagesListSchema | undefined, - offset: number, - limit: number, - sort?: IXSuggestionsQuery['sort'] + options: { page?: IXSuggestionsQuery['page']; sort?: IXSuggestionsQuery['sort'] } ) => { + const offset = options && options.page ? options.page.size * (options.page.number - 1) : 0; + const limit = options.page?.size || DEFAULT_LIMIT; + const { sort } = options; + const sortOrder = sort?.order === 'desc' ? -1 : 1; const sorting = sort?.property ? { [sort.property]: sortOrder } : { date: 1, state: -1 }; @@ -136,63 +138,6 @@ const buildListQuery = ( return pipeline; }; -async function getLabeledCounts(extractorId: ObjectId) { - const labeledAggregationQuery = [ - { - $match: { - ...baseQueryFragment(extractorId), - ...filterFragments.labeled._fragment, - }, - }, - ...groupByAndCount('$state.match'), - ]; - const labeledAggregation: { _id: boolean; count: number }[] = - await IXSuggestionsModel.db.aggregate(labeledAggregationQuery); - const matchCount = - labeledAggregation.find((aggregation: any) => aggregation._id === true)?.count || 0; - const mismatchCount = - labeledAggregation.find((aggregation: any) => aggregation._id === false)?.count || 0; - const labeledCount = matchCount + mismatchCount; - return { labeledCount, matchCount, mismatchCount }; -} - -const getNonLabeledCounts = async (_extractorId: ObjectId) => { - const extractorId = new ObjectId(_extractorId); - const unlabeledMatch = { - ...baseQueryFragment(extractorId), - ...filterFragments.nonLabeled._fragment, - }; - const nonLabeledCount = await IXSuggestionsModel.count(unlabeledMatch); - const noContextCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.noContext, - }); - const withSuggestionCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.withSuggestion, - }); - const noSuggestionCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.noSuggestion, - }); - const obsoleteCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.obsolete, - }); - const othersCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.others, - }); - return { - nonLabeledCount, - noContextCount, - withSuggestionCount, - noSuggestionCount, - obsoleteCount, - othersCount, - }; -}; - const readFilter = (filter: IXSuggestionsFilter) => { const { customFilter, extractorId: _extractorId } = filter; const extractorId = new ObjectId(_extractorId); @@ -247,9 +192,6 @@ const Suggestions = { sort?: IXSuggestionsQuery['sort']; } ) => { - const offset = options && options.page ? options.page.size * (options.page.number - 1) : 0; - const DEFAULT_LIMIT = 30; - const limit = options.page?.size || DEFAULT_LIMIT; const { languages: setLanguages } = await settings.get(); const { customFilter, extractorId } = readFilter(filter); @@ -258,44 +200,78 @@ const Suggestions = { .then(result => (result?.length ? result[0].count : 0)); let suggestions = await IXSuggestionsModel.db.aggregate( - buildListQuery(extractorId, customFilter, setLanguages, offset, limit, options.sort) + buildListQuery(extractorId, customFilter, setLanguages, options) ); suggestions = await postProcessSuggestions(suggestions, extractorId); return { suggestions, - totalPages: Math.ceil(count / limit), + totalPages: Math.ceil(count / (options.page?.size || DEFAULT_LIMIT)), }; }, aggregate: async (_extractorId: ObjectIdSchema): Promise => { const extractorId = new ObjectId(_extractorId); - const { labeledCount, matchCount, mismatchCount } = await getLabeledCounts(extractorId); - const { - nonLabeledCount, - noContextCount, - withSuggestionCount, - noSuggestionCount, - obsoleteCount, - othersCount, - } = await getNonLabeledCounts(extractorId); - const totalCount = labeledCount + nonLabeledCount; - return { - total: totalCount, - labeled: { - _count: labeledCount, - match: matchCount, - mismatch: mismatchCount, - }, - nonLabeled: { - _count: nonLabeledCount, - noContext: noContextCount, - withSuggestion: withSuggestionCount, - noSuggestion: noSuggestionCount, - obsolete: obsoleteCount, - others: othersCount, - }, + + const aggregations: (IXSuggestionAggregation & { _id: ObjectId })[] = + await IXSuggestionsModel.db.aggregate([ + { + $match: { extractorId }, + }, + { + $group: { + _id: null, + total: { $sum: 1 }, + labeled: { $sum: { $cond: ['$state.labeled', 1, 0] } }, + nonLabeled: { + $sum: { + $cond: [ + { + $and: [ + { $ne: ['$state.labeled', undefined] }, + { $ne: ['$state.labeled', null] }, + { $not: '$state.labeled' }, + ], + }, + 1, + 0, + ], + }, + }, + match: { $sum: { $cond: ['$state.match', 1, 0] } }, + mismatch: { + $sum: { + $cond: [ + { + $and: [ + { $ne: ['$state.match', undefined] }, + { $ne: ['$state.match', null] }, + { $not: '$state.match' }, + ], + }, + 1, + 0, + ], + }, + }, + obsolete: { $sum: { $cond: ['$state.obsolete', 1, 0] } }, + error: { $sum: { $cond: ['$state.error', 1, 0] } }, + }, + }, + ]); + + const { _id, ...results } = aggregations[0] || { + _id: null, + total: 0, + labeled: 0, + nonLabeled: 0, + match: 0, + mismatch: 0, + obsolete: 0, + error: 0, }; + + return results; }, updateStates, diff --git a/app/api/suggestions/updateState.ts b/app/api/suggestions/updateState.ts index a352033ed0..f38f6886a6 100644 --- a/app/api/suggestions/updateState.ts +++ b/app/api/suggestions/updateState.ts @@ -109,6 +109,7 @@ export const postProcessCurrentValues = ( propertyType: PropertyTypeSchema ) => suggestions.map(s => postProcessCurrentValue(s, propertyType)); +// eslint-disable-next-line max-statements export const updateStates = async (query: any) => { const { languages } = await settings.get(); const propertyTypes = objectIndex( diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index a9495886c2..cb6e411490 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import { ObjectId } from 'mongodb'; -import db from 'api/utils/testing_db'; +import { testingDB } from 'api/utils/testing_db'; import { EntitySchema } from 'shared/types/entityType'; import { FileType } from 'shared/types/fileType'; import { UserRole } from 'shared/types/userSchema'; @@ -24,11 +24,15 @@ import { getV2FixturesFactoryElements } from 'api/common.v2/testing/fixturesFact import { IXModelType } from 'shared/types/IXModelType'; import { PermissionSchema } from 'shared/types/permissionType'; +type PartialSuggestion = Partial> & { + state?: Partial; +}; + function getIdMapper() { const map = new Map(); return function setAndGet(key: string) { - if (!map.has(key)) map.set(key, db.id() as ObjectId); + if (!map.has(key)) map.set(key, testingDB.id() as ObjectId); return map.get(key)!; }; @@ -196,7 +200,7 @@ function getFixturesFactory() { originalname: string | undefined = undefined, extractedMetadata: ExtractedMetadataSchema[] = [] ): WithId => ({ - _id: idMapper(`${id}`), + _id: idMapper(id), entity, language, type, @@ -314,7 +318,38 @@ function getFixturesFactory() { extractorId: idMapper(extractor), }), - ixSuggestion: ( + ixSuggestion(props: PartialSuggestion): IXSuggestionType { + const { state, ...otherProps } = props; + + return { + _id: testingDB.id(), + entityId: testingDB.id().toString(), + status: 'ready' as const, + entityTemplate: testingDB.id().toString(), + language: 'en', + fileId: testingDB.id().toString(), + propertyName: 'propertyName', + extractorId: testingDB.id(), + error: '', + segment: '', + suggestedValue: '', + date: 1001, + state: { + labeled: false, + withValue: true, + withSuggestion: false, + hasContext: false, + processing: false, + obsolete: false, + error: false, + ...state, + }, + ...otherProps, + }; + }, + + // eslint-disable-next-line max-params + ixSuggestion_deprecated: ( suggestionId: string, extractor: string, entity: string, @@ -339,10 +374,9 @@ function getFixturesFactory() { labeled: false, withValue: true, withSuggestion: false, - match: false, hasContext: false, - obsolete: false, processing: false, + obsolete: false, error: false, }, ...otherProps, diff --git a/app/react/App/styles/globals.css b/app/react/App/styles/globals.css index 960970b326..97deede448 100644 --- a/app/react/App/styles/globals.css +++ b/app/react/App/styles/globals.css @@ -1740,11 +1740,6 @@ input[type="range"]::-ms-fill-lower { margin-right: 0.5rem; } -.mx-4 { - margin-left: 1rem; - margin-right: 1rem; -} - .mx-auto { margin-left: auto; margin-right: auto; @@ -2444,12 +2439,6 @@ input[type="range"]::-ms-fill-lower { margin-left: calc(1rem * calc(1 - var(--tw-space-x-reverse))); } -.space-y-1 > :not([hidden]) ~ :not([hidden]) { - --tw-space-y-reverse: 0; - margin-top: calc(0.25rem * calc(1 - var(--tw-space-y-reverse))); - margin-bottom: calc(0.25rem * var(--tw-space-y-reverse)); -} - .space-y-3 > :not([hidden]) ~ :not([hidden]) { --tw-space-y-reverse: 0; margin-top: calc(0.75rem * calc(1 - var(--tw-space-y-reverse))); @@ -2856,11 +2845,6 @@ input[type="range"]::-ms-fill-lower { background-color: rgb(222 247 236 / var(--tw-bg-opacity)); } -.bg-green-200 { - --tw-bg-opacity: 1; - background-color: rgb(188 240 218 / var(--tw-bg-opacity)); -} - .bg-green-400 { --tw-bg-opacity: 1; background-color: rgb(49 196 141 / var(--tw-bg-opacity)); @@ -3329,11 +3313,6 @@ input[type="range"]::-ms-fill-lower { color: rgb(209 213 219 / var(--tw-text-opacity)) !important; } -.text-black { - --tw-text-opacity: 1; - color: rgb(0 0 0 / var(--tw-text-opacity)); -} - .text-blue-600 { --tw-text-opacity: 1; color: rgb(79 70 229 / var(--tw-text-opacity)); @@ -3344,6 +3323,11 @@ input[type="range"]::-ms-fill-lower { color: rgb(55 48 163 / var(--tw-text-opacity)); } +.text-error-500 { + --tw-text-opacity: 1; + color: rgb(236 72 153 / var(--tw-text-opacity)); +} + .text-error-600 { --tw-text-opacity: 1; color: rgb(219 39 119 / var(--tw-text-opacity)); diff --git a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx index a8029adba7..ccd9f409f3 100644 --- a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx +++ b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx @@ -24,7 +24,7 @@ import { ClientPropertySchema, ClientTemplateSchema } from 'app/istore'; import { notificationAtom } from 'app/V2/atoms'; import { socket } from 'app/socket'; import { SuggestionsTitle } from './components/SuggestionsTitle'; -import { FiltersSidepanel } from './components/FiltersSidepanel.old'; +import { FiltersSidepanel } from './components/FiltersSidepanel'; import { suggestionsTableColumnsBuilder } from './components/TableElements'; import { PDFSidepanel } from './components/PDFSidepanel'; import { @@ -112,6 +112,7 @@ const IXSuggestions = () => { const [sidepanelSuggestion, setSidepanelSuggestion] = useState(); const [selected, setSelected] = useState([]); const [sorting, setSorting] = useState([]); + const [aggregations, setAggregations] = useState(aggregation); const { revalidate } = useRevalidator(); const setNotifications = useSetAtom(notificationAtom); const [status, setStatus] = useState<{ @@ -139,6 +140,10 @@ const IXSuggestions = () => { }; }, [extractor._id, revalidate]); + useEffect(() => { + setAggregations(aggregation); + }, [aggregation]); + useEffect(() => { if (searchParams.has('sort') && !sorting.length) { navigate(location.pathname, { replace: true }); @@ -187,6 +192,8 @@ const IXSuggestions = () => { }); await suggestionsAPI.accept(preparedSuggestions); + const newAggregations = await suggestionsAPI.aggregation(extractor._id); + setAggregations(newAggregations); setCurrentSuggestions(current => updateSuggestions(current, acceptedSuggestions)); setNotifications({ type: 'success', @@ -272,7 +279,7 @@ const IXSuggestions = () => {
@@ -353,7 +360,7 @@ const IXSuggestions = () => { >; - aggregation: any; -} - -const Header = ({ label, total }: { label: string; total: number }) => ( -
-
{label}
-
-
{total}
-
-); - -const getPercentage = (match: number, total: number): string => { - const percentage = (match / total) * 100; - - if (Number.isNaN(percentage)) { - return '-'; - } - - return `${Math.round(percentage)}%`; -}; - -const FiltersSidepanel = ({ - showSidepanel, - setShowSidepanel, - aggregation, -}: FiltersSidepanelProps) => { - const [searchParams, setSearchParams] = useSearchParams(); - - const defaultFilter: SuggestionCustomFilter = { - labeled: { - match: false, - mismatch: false, - }, - nonLabeled: { - noContext: false, - withSuggestion: false, - noSuggestion: false, - obsolete: false, - others: false, - }, - }; - - let initialFilters: SuggestionCustomFilter = defaultFilter; - - try { - if (searchParams.has('filter')) { - initialFilters = JSON.parse(searchParams.get('filter')!); - } - } catch (e) {} - - const { register, handleSubmit, reset, setValue } = useForm({ - values: initialFilters, - defaultValues: defaultFilter, - }); - - const submitFilters = async (filters: SuggestionCustomFilter) => { - setSearchParams((prev: URLSearchParams) => { - prev.set('page', '1'); - prev.set('filter', JSON.stringify(filters)); - return prev; - }); - setShowSidepanel(false); - }; - - const checkOption = (e: any, optionName: any) => { - const { checked } = e.target; - setValue(optionName, checked); - }; - - const clearFilters = () => { - setSearchParams(prev => { - prev.delete('filter'); - return prev; - }); - setShowSidepanel(false); - reset(); - }; - - return ( - setShowSidepanel(false)} - title={ - - Stats & Filters - - } - > -
- - } - > -
-
-
- Accuracy -
-
-
- {getPercentage(aggregation.labeled.match, aggregation.labeled._count)} -
-
-
- { - checkOption(e, 'labeled.match'); - }} - /> -
-
{aggregation.labeled.match}
-
-
- { - checkOption(e, 'labeled.mismatch'); - }} - /> -
-
{aggregation.labeled.mismatch}
-
-
- - - } - > -
-
-
- Pending -
-
-
- {getPercentage(aggregation.nonLabeled._count, aggregation.total)} -
-
-
- With suggestion} - {...register('nonLabeled.withSuggestion')} - onChange={e => { - checkOption(e, 'nonLabeled.withSuggestion'); - }} - /> -
-
{aggregation.nonLabeled.withSuggestion}
-
-
- No suggestion} - {...register('nonLabeled.noSuggestion')} - onChange={e => { - checkOption(e, 'nonLabeled.noSuggestion'); - }} - /> -
-
{aggregation.nonLabeled.noSuggestion}
-
-
- No context} - {...register('nonLabeled.noContext')} - onChange={e => { - checkOption(e, 'nonLabeled.noContext'); - }} - /> -
-
{aggregation.nonLabeled.noContext}
-
-
- Obsolete} - {...register('nonLabeled.obsolete')} - onChange={e => { - checkOption(e, 'nonLabeled.obsolete'); - }} - /> -
-
{aggregation.nonLabeled.obsolete}
-
-
- Others} - {...register('nonLabeled.others')} - onChange={e => { - checkOption(e, 'nonLabeled.others'); - }} - /> -
-
{aggregation.nonLabeled.others}
-
-
- - - -
- - -
-
- - - ); -}; - -export { FiltersSidepanel }; diff --git a/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx b/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx index f03ad2d8e5..8f96df6399 100644 --- a/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx +++ b/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx @@ -89,13 +89,6 @@ const FiltersSidepanel = ({
-
-
- Accuracy (labeled data) -
-
-
{aggregation.accuracy}%
-
Labeled} diff --git a/app/react/V2/Routes/Settings/IX/components/TableElements.tsx b/app/react/V2/Routes/Settings/IX/components/TableElements.tsx index 8cf3714660..ff249a3d71 100644 --- a/app/react/V2/Routes/Settings/IX/components/TableElements.tsx +++ b/app/react/V2/Routes/Settings/IX/components/TableElements.tsx @@ -77,6 +77,44 @@ const PropertyCell = ({ cell }: CellContext { + const suggestions = suggestion.subRows; + const ammountOfSuggestions = suggestions.length; + const amountOfValues = suggestions.filter(s => s.currentValue).length; + const amountOfMatches = suggestions.filter(s => s.currentValue === s.suggestedValue).length; + const amountOfMissmatches = ammountOfSuggestions - amountOfMatches; + + return ( +
+ + {amountOfValues} values + + | + + {ammountOfSuggestions} suggestions + + {amountOfMatches > 0 && ( + <> + | + + {amountOfMatches}{' '} + matching + + + )} + {amountOfMissmatches > 0 && ( + <> + | + + {amountOfMissmatches}{' '} + mismatching + + + )} +
+ ); +}; + const CurrentValueCell = ({ cell, allProperties, @@ -87,43 +125,29 @@ const CurrentValueCell = ({ >; allProperties: ClientPropertySchema[]; }) => { - if ('subRows' in cell.row.original) { - const suggestions = cell.row.original.subRows; - const ammountOfSuggestions = suggestions.length; - const amountOfValues = suggestions.filter(suggestion => suggestion.currentValue).length; - const amountOfMatches = suggestions.filter(s => s.currentValue === s.suggestedValue).length; - const amountOfMissmatches = ammountOfSuggestions - amountOfMatches; - + if (cell.row.original.state.obsolete) { return (
- {amountOfValues} values + Obsolete - | +
+ ); + } + + if (cell.row.original.state.error) { + return ( +
- {ammountOfSuggestions} suggestions + Error - {amountOfMatches > 0 && ( - <> - | - - {amountOfMatches}{' '} - matching - - - )} - {amountOfMissmatches > 0 && ( - <> - | - - {amountOfMissmatches}{' '} - mismatching - - - )}
); } + + if ('subRows' in cell.row.original) { + return ; + } return ( { @@ -86,7 +87,7 @@ describe('getIXSuggestionState', () => { labeled: false, withValue: true, withSuggestion: false, - match: false, + match: undefined, hasContext: false, obsolete: false, processing: false, @@ -140,7 +141,7 @@ describe('getIXSuggestionState', () => { }); }); - it('should mark when currentValue != suggestedValue, labeledValue are empty', () => { + it('should mark when currentValue != suggestedValue, labeledValue is empty', () => { const values = { currentValue: 'some other value', date: 1234, @@ -224,12 +225,36 @@ describe('getIXSuggestionState', () => { labeled: false, withValue: false, withSuggestion: true, - match: false, + match: undefined, hasContext: false, obsolete: true, processing: false, error: false, }); }); + + it('should mark processing when status is processing and set obsolete as true', () => { + const values = { + currentValue: '', + date: 1234, + labeledValue: '', + suggestedValue: 'some value', + modelCreationDate: 1, + status: 'processing', + }; + + const state = getSuggestionState(values, 'text'); + + expect(state).toEqual({ + labeled: false, + withValue: false, + withSuggestion: true, + match: undefined, + hasContext: false, + obsolete: true, + processing: true, + error: false, + }); + }); }); }); diff --git a/app/shared/types/suggestionSchema.ts b/app/shared/types/suggestionSchema.ts index 8a0f80f393..e0ce40cf38 100644 --- a/app/shared/types/suggestionSchema.ts +++ b/app/shared/types/suggestionSchema.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { objectIdSchema, propertyValueSchema, @@ -5,14 +6,14 @@ import { } from 'shared/types/commonSchemas'; import { propertyTypes } from 'shared/propertyTypes'; -export const emitSchemaTypes = true; - const commonSuggestionMessageProperties = { tenant: { type: 'string', minLength: 1 }, id: { type: 'string', minLength: 1 }, xml_file_name: { type: 'string', minLength: 1 }, }; +export const emitSchemaTypes = true; + export const CommonSuggestionSchema = { type: 'object', title: 'CommonSuggestion', @@ -88,10 +89,9 @@ export const IXSuggestionStateSchema = { 'labeled', 'withValue', 'withSuggestion', - 'match', 'hasContext', - 'obsolete', 'processing', + 'obsolete', 'error', ], }; @@ -204,29 +204,14 @@ export const SuggestionCustomFilterSchema = { title: 'SuggestionCustomFilter', additionalProperties: false, properties: { - labeled: { - type: 'object', - properties: { - match: { type: 'boolean' }, - mismatch: { type: 'boolean' }, - }, - additionalProperties: false, - required: ['match', 'mismatch'], - }, - nonLabeled: { - type: 'object', - properties: { - withSuggestion: { type: 'boolean' }, - noSuggestion: { type: 'boolean' }, - noContext: { type: 'boolean' }, - obsolete: { type: 'boolean' }, - others: { type: 'boolean' }, - }, - additionalProperties: false, - required: ['withSuggestion', 'noSuggestion', 'noContext', 'obsolete', 'others'], - }, + labeled: { type: 'boolean' }, + match: { type: 'boolean' }, + mismatch: { type: 'boolean' }, + nonLabeled: { type: 'boolean' }, + obsolete: { type: 'boolean' }, + error: { type: 'boolean' }, }, - required: ['labeled', 'nonLabeled'], + required: ['labeled', 'nonLabeled', 'match', 'mismatch', 'obsolete', 'error'], }; export const SuggestionsQueryFilterSchema = { @@ -285,31 +270,14 @@ export const IXSuggestionAggregationSchema = { type: 'object', title: 'IXSuggestionAggregation', additionalProperties: false, - required: ['labeled', 'nonLabeled', 'total'], + required: ['total', 'labeled', 'nonLabeled', 'match', 'mismatch', 'obsolete', 'error'], properties: { total: { type: 'number' }, - labeled: { - type: 'object', - additionalProperties: false, - required: ['_count', 'match', 'mismatch'], - properties: { - _count: { type: 'number' }, - match: { type: 'number' }, - mismatch: { type: 'number' }, - }, - }, - nonLabeled: { - type: 'object', - additionalProperties: false, - required: ['_count', 'withSuggestion', 'noSuggestion', 'noContext', 'obsolete', 'others'], - properties: { - _count: { type: 'number' }, - withSuggestion: { type: 'number' }, - noSuggestion: { type: 'number' }, - noContext: { type: 'number' }, - obsolete: { type: 'number' }, - others: { type: 'number' }, - }, - }, + labeled: { type: 'number' }, + nonLabeled: { type: 'number' }, + match: { type: 'number' }, + mismatch: { type: 'number' }, + obsolete: { type: 'number' }, + error: { type: 'number' }, }, }; diff --git a/app/shared/types/suggestionType.d.ts b/app/shared/types/suggestionType.d.ts index 504cb24983..be4626461a 100644 --- a/app/shared/types/suggestionType.d.ts +++ b/app/shared/types/suggestionType.d.ts @@ -47,19 +47,12 @@ export interface IXAggregationQuery { export interface IXSuggestionAggregation { total: number; - labeled: { - _count: number; - match: number; - mismatch: number; - }; - nonLabeled: { - _count: number; - withSuggestion: number; - noSuggestion: number; - noContext: number; - obsolete: number; - others: number; - }; + labeled: number; + nonLabeled: number; + match: number; + mismatch: number; + obsolete: number; + error: number; } export interface IXSuggestionType { @@ -91,7 +84,7 @@ export interface IXSuggestionStateType { labeled: boolean; withValue: boolean; withSuggestion: boolean; - match: boolean; + match?: boolean; hasContext: boolean; obsolete: boolean; processing: boolean; @@ -112,17 +105,12 @@ export interface IXSuggestionsQuery { } export interface SuggestionCustomFilter { - labeled: { - match: boolean; - mismatch: boolean; - }; - nonLabeled: { - withSuggestion: boolean; - noSuggestion: boolean; - noContext: boolean; - obsolete: boolean; - others: boolean; - }; + labeled: boolean; + match: boolean; + mismatch: boolean; + nonLabeled: boolean; + obsolete: boolean; + error: boolean; } export interface IXSuggestionsFilter { diff --git a/contents/ui-translations/ar.csv b/contents/ui-translations/ar.csv index 0e1dfd6608..26694581e5 100644 --- a/contents/ui-translations/ar.csv +++ b/contents/ui-translations/ar.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,عمل/ إجراء Activate,Activate Activated,Activated @@ -957,7 +955,6 @@ Welcome to Uwazi,مرحباً بكم في أوازي will be updated with the same value.,سيحدث بالقيمة نفسها with,مع With great power comes great responsibility!,كلما ازدادت القوة، ازدادت معها المسؤولية! -With suggestion,With suggestion x less,X أقل x more,X أكثر Year,سنة diff --git a/contents/ui-translations/en.csv b/contents/ui-translations/en.csv index 9f39892291..e032c667de 100644 --- a/contents/ui-translations/en.csv +++ b/contents/ui-translations/en.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Action Activate,Activate Activated,Activated @@ -960,7 +958,6 @@ Welcome to Uwazi,Welcome to Uwazi will be updated with the same value.,will be updated with the same value. with,with With great power comes great responsibility!,With great power comes great responsibility! -With suggestion,With suggestion x less,less x more,more Year,Year diff --git a/contents/ui-translations/es.csv b/contents/ui-translations/es.csv index 6ba6d9beac..4430fdb383 100644 --- a/contents/ui-translations/es.csv +++ b/contents/ui-translations/es.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Cuenta bloqueada. Revisa tu correo electrónico para desbloquearla. Account unlocked successfully,Cuenta desbloqueada satisfactoriamente Account updated,Account updated -Accuracy,Precisión -Accuracy (labeled data),Accuracy (labeled data) Action,Acción Activate,Activate Activated,Activated @@ -955,7 +953,6 @@ Welcome to Uwazi,Bienvenido a Uwazi will be updated with the same value.,se actualizará con el mismo valor with,con With great power comes great responsibility!,"¡Con un gran poder, viene una gran responsabilidad!" -With suggestion,With suggestion x less,menos x more,más Year,Año diff --git a/contents/ui-translations/fr.csv b/contents/ui-translations/fr.csv index 550f8525a9..f7ff8cd65d 100644 --- a/contents/ui-translations/fr.csv +++ b/contents/ui-translations/fr.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Action Activate,Activate Activated,Activated @@ -957,7 +955,6 @@ Welcome to Uwazi,Bienvenue à Uwazi will be updated with the same value.,sera mis à jour avec la même valeur. with,avec With great power comes great responsibility!,De grands pouvoirs impliquent de grandes responsabilités ! -With suggestion,With suggestion x less,x en moins x more,x en plus Year,Année diff --git a/contents/ui-translations/ko.csv b/contents/ui-translations/ko.csv index 4f3b372d3e..dde1cc601e 100644 --- a/contents/ui-translations/ko.csv +++ b/contents/ui-translations/ko.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,활동 Activate,Activate Activated,Activated @@ -958,7 +956,6 @@ Welcome to Uwazi,Uwazi 에 오신 것을 환영합니다. will be updated with the same value.,동일 값으로 업데이트됩니다. with,with With great power comes great responsibility!,큰 일에는 큰 책임이 따르는 법! -With suggestion,With suggestion x less,덜 보기 x more,더 보기 Year,연도 diff --git a/contents/ui-translations/my.csv b/contents/ui-translations/my.csv index 2add7198d2..51c1e8fa7b 100644 --- a/contents/ui-translations/my.csv +++ b/contents/ui-translations/my.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,လုပ်ဆောင်ချက် Activate,Activate Activated,Activated @@ -958,7 +956,6 @@ Welcome to Uwazi,Uwazi မှ ကြိုဆိုပါသည် will be updated with the same value.,တူညီသော တန်ဖိုးဖြင့် အပ်ဒိတ်လုပ်ပါမည်။ with,ဖြင့် With great power comes great responsibility!,လုပ်ပိုင်ခွင့်ကြီးလျှင် တာဝန်ပိုကြီးသည်! -With suggestion,With suggestion x less,x လျော့ x more,x ပို Year,နှစ် diff --git a/contents/ui-translations/ru.csv b/contents/ui-translations/ru.csv index ddc4b7a716..33f9a84742 100644 --- a/contents/ui-translations/ru.csv +++ b/contents/ui-translations/ru.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Действие Activate,Activate Activated,Activated @@ -955,7 +953,6 @@ Welcome to Uwazi,Добро пожаловать в Uwazi will be updated with the same value.,будет обновляться с тем же значением with,с With great power comes great responsibility!,С большой силой приходит большая ответственность! -With suggestion,With suggestion x less,x меньше x more,х больше Year,Год diff --git a/contents/ui-translations/th.csv b/contents/ui-translations/th.csv index 370829a7e5..beea2c4b79 100644 --- a/contents/ui-translations/th.csv +++ b/contents/ui-translations/th.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,เริ่มทำ Activate,Activate Activated,Activated @@ -958,7 +956,6 @@ Welcome to Uwazi,ยินดีต้อนรับสู่ Uwazi will be updated with the same value.,จะถูกอัปเดทด้วยค่าเดียวกัน with,กับ With great power comes great responsibility!,พลังที่ยิ่งใหญ่มากับความรับผิดชอบที่ใหญ่ยิ่ง! -With suggestion,With suggestion x less,ลดขนาดลง x more,เพิ่มขนาดขึ้น Year,ปี diff --git a/contents/ui-translations/tr.csv b/contents/ui-translations/tr.csv index 8839cbc6a7..aa491c5dd3 100644 --- a/contents/ui-translations/tr.csv +++ b/contents/ui-translations/tr.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Eylem Activate,Activate Activated,Activated @@ -958,7 +956,6 @@ Welcome to Uwazi,Uwazi'ye hoş geldiniz will be updated with the same value.,aynı değerle güncellenecektir. with,ile With great power comes great responsibility!,Büyük güç büyük sorumluluk getirir! -With suggestion,With suggestion x less,daha az x more,daha az Year,Yıl