From cde0cf4e1551164d874d628d3afec1ea3e0733bd Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 6 Oct 2023 22:03:59 +0300 Subject: [PATCH 1/6] IX suggestions UI Improvement (#6056) * initial layout * added initial table * Table with icons, links and translations * initial state and checkboxes * styles * dynamic footer * flow to delete extractors * list extractors to be deleted * initial layout * added initial table * Table with icons, links and translations * initial state and checkboxes * styles * dynamic footer * flow to delete extractors * list extractors to be deleted * restored headers * reuse current extractor modal * save and update logic * moved extractor modal into V2 folder * refactor * refactor * fixed bug in 'from all templates' button * minimal validation + disable save button * typo fix * created a shared types file * deleted old dashboard * refactor * simplified conditional * moved ix test to settings folder * updated e2e * moved save/update logic to api * removed obsolete key * added new keys + sorted .CSVs * migration * inject Axe for the dashboard test * Added suggestions dashboard basic table * Added a few embelishments * Added dot component * Added suggestions title * WIP: Adding popup on suggestions * remove stats endpoint * change state representation, rewrite updateStates * Revert "change state representation, rewrite updateStates" This reverts commit 9bbac58853569d0db0c7b0d2527aea6b9689e14a. * change state representation, rewrite updateStates * correct type * Fixed the popover * Fixed up stuff * Added popover on hover * Removed comments * Added dates and numerics support * Changed font weight * get simple use cases * Added functionality for action button * suggestions get with filters * Added dates support * fix suggestions test * Added filters button and sidepanel * Refactored ix api * update route tests * update ix tests * comment out aggregation functions * add dummy aggregation route * fix endpoint * Added titles to filters sidepanel * Added basic filters sidepanel * Updated checkbox * Updated suggestion filters * Tried pagination * mass accept suggestions on manager level * suggestions mass accept in routes * correct get filters * getting status on load * aggregations on Suggestions object * aggregation route * updated information extraction test * eslint cleanup * migration to remove obsolete index * added new indices * Added support for accepting bulk suggestions * Added pagination to suggestions table * Added pagination support * Refactor * Fixed paginator * Changed paginator test * Fixed eslint issues * Fixed more eslint errors * Wrapped text with translate element * Styled paginator more * Bump mongoose from 7.2.4 to 7.4.1 (#6054) * Bump mongoose from 7.2.4 to 7.4.1 Bumps [mongoose](https://github.com/Automattic/mongoose) from 7.2.4 to 7.4.1. - [Release notes](https://github.com/Automattic/mongoose/releases) - [Changelog](https://github.com/Automattic/mongoose/blob/master/CHANGELOG.md) - [Commits](https://github.com/Automattic/mongoose/compare/7.2.4...7.4.1) --- updated-dependencies: - dependency-name: mongoose dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump mongodb and fix type * Fix test on MongoDataSource * Force github refresh * fix type too deep --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Federico Nocetti * Bump mongoose from 7.2.4 to 7.4.1 (#6054) * Bump mongoose from 7.2.4 to 7.4.1 Bumps [mongoose](https://github.com/Automattic/mongoose) from 7.2.4 to 7.4.1. - [Release notes](https://github.com/Automattic/mongoose/releases) - [Changelog](https://github.com/Automattic/mongoose/blob/master/CHANGELOG.md) - [Commits](https://github.com/Automattic/mongoose/compare/7.2.4...7.4.1) --- updated-dependencies: - dependency-name: mongoose dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump mongodb and fix type * Fix test on MongoDataSource * Force github refresh * fix type too deep --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Federico Nocetti * Added more UI improvements * Added find suggestions countdown * Fixed linting issues * Added cancel capabilities * Fixed lint issue * cleanup + wip pdf sidepanel * getting file for render * Fixed lint issues * disable rendering links and anotations on PDF * Added element data-testid for table * Updated fixtures * Updated e2e tests * Removed previous implementation of IX * Fixed some types errors * Renamed a migration and added another one * Updated translation files * Added missing keys * removed duplicate key * added simple loading message to pdf render * adjusted size of sidepanel on bigger screens * basic render of existing selections * fix issue with TS type depth * cleanup * more cleanup * imrpoved existing selections calculation * Added new button * arranged stories * inproved existing selection formatter * click to fill dummy + property name for input * added message in case selection is empty * click to fill updates selected text * added clear selection button * Date and Number fields for PDFsidepanel * refactor * notify empty selection with styled error * account for empty selections on property * scroll to page + visuals WIP * clear selections and higlights when closing * pdf viewer refactor * load/unload visible pages for optimization * scroll to page functionality * Go to first highlight page * correctly set height of pdf container * adjusted translatable texts * function to update selections * function to delete selection * apply function for updating selections * set entity for the sidepanel * form with validation and click to fill * update file selections on save * allow creating new selections * improved file selections update * upped migrations number * use utc time for timestamps * set dates in fields * Use first template for property name * coerce date selections * use timezoned date strings for timestamps * update entity from sidepanel * refactor * disable buttons while submitting * hide error to avoid layout shifting * small fixes * notify success or error * Move selection functions into pdf module * formatter for entity endpoint * refactor + testing * type correction + better type guard * increased suggestions per page * changed used entity * organized template pills + visual fixes * center pdf on the sidepanel * change socket subscribe effect deps * clear modal state when no extractor * fixed a11y issues on review table * more a11y fixes * more a11y * extended ix e2e * updated snapshot since entity has pdf * disable actions if suggestion has no entity * upped migrations number * allow table sorting to be manual * updated type with correct value * allow sorting by property * extended tests + fixed pipeline order * table sorts all suggestions * replace method * allow sorting by current value * disable buttons while editing extractors * fixtures update * fixed number of expected results + screenshots * Updated entity for internal link --------- Signed-off-by: dependabot[bot] Co-authored-by: Santiago Co-authored-by: Santiago <71732018+Zasa-san@users.noreply.github.com> Co-authored-by: Laszlo Kecskes Co-authored-by: A happy cat Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Federico Nocetti --- app/api/files/specs/fixtures.ts | 4 - .../145-remove_obsolete_mongo_index/index.js | 37 + .../145-remove_obsolete_mongo_index.spec.js | 57 ++ .../specs/fixtures.js | 4 + .../146-update_translations/index.js | 108 +++ .../specs/146-update_translations.spec.js | 75 ++ .../146-update_translations/specs/fixtures.js | 115 +++ .../InformationExtraction.ts | 2 +- .../informationextraction/getFiles.ts | 3 +- .../specs/InformationExtraction.spec.ts | 62 +- .../informationextraction/specs/fixtures.ts | 22 +- .../specs/ixextractors.spec.ts | 36 +- app/api/suggestions/IXSuggestionsModel.ts | 7 +- app/api/suggestions/pipelineStages.ts | 82 +- app/api/suggestions/routes.ts | 86 +- .../suggestions/specs/customFilters.spec.ts | 223 +++++ app/api/suggestions/specs/fixtures.ts | 488 ++++++++++- app/api/suggestions/specs/routes.spec.ts | 415 +++++----- app/api/suggestions/specs/stats.spec.ts | 194 ----- app/api/suggestions/specs/suggestions.spec.ts | 554 +++++++++---- app/api/suggestions/stats.ts | 95 --- app/api/suggestions/suggestions.ts | 332 +++++--- app/api/suggestions/updateState.ts | 5 +- app/api/templates/specs/templates.spec.js | 44 +- app/api/templates/templates.ts | 34 +- app/api/utils/fixturesFactory.ts | 37 +- app/react/App/App.js | 2 +- app/react/App/scss/layout/_nav.scss | 6 +- app/react/App/styles/globals.css | 760 +++++++++++++++++- .../CancelFindingSuggestionsModal.tsx | 41 - .../MetadataExtraction/EntitySuggestions.tsx | 507 ------------ .../MetadataExtraction/FilterSidePanel.tsx | 86 -- app/react/MetadataExtraction/GridChart.tsx | 121 --- app/react/MetadataExtraction/PDFSidePanel.tsx | 123 --- .../PropertyConfigurationModal.tsx | 115 --- .../SuggestionAcceptanceModal.tsx | 64 -- .../MetadataExtraction/SuggestionsAPI.ts | 71 -- .../SuggestionsContainer.tsx | 75 -- .../MetadataExtraction/SuggestionsTable.tsx | 230 ------ .../TrainingHealthDashboard.tsx | 45 -- .../TrainingHealthLegend.tsx | 39 - .../MetadataExtraction/actions/actions.ts | 89 -- .../specs/EntitySuggestions.spec.tsx | 378 --------- .../specs/GridChart.spec.tsx | 82 -- .../specs/PDFSidePanel.spec.tsx | 76 -- .../specs/PDFSidepanelFixtures.ts | 64 -- .../specs/PropertyConfigurationModal.spec.tsx | 98 --- .../specs/SuggestionsAPI.spec.ts | 79 -- .../specs/TrainingHealthDashboard.spec.tsx | 63 -- .../MetadataExtraction/specs/fixtures.ts | 88 -- .../useContainerWidthHook.ts | 22 - app/react/Routes.tsx | 2 +- app/react/V2/Components/Forms/Checkbox.tsx | 45 ++ app/react/V2/Components/Forms/InputField.tsx | 2 +- app/react/V2/Components/Forms/index.ts | 1 + app/react/V2/Components/PDFViewer/PDF.tsx | 82 +- app/react/V2/Components/PDFViewer/PDFPage.tsx | 111 ++- .../functions/handleTextSelection.ts | 192 +++++ .../PDFViewer/functions/specs/fixtures.ts | 139 ++++ .../specs/handleTextSelection.spec.ts | 310 +++++++ app/react/V2/Components/PDFViewer/index.ts | 3 +- app/react/V2/Components/UI/EmbededButton.tsx | 17 +- app/react/V2/Components/UI/Paginator.tsx | 64 +- app/react/V2/Components/UI/Sidepanel.tsx | 14 +- app/react/V2/Components/UI/Table.tsx | 29 +- app/react/V2/Components/UI/TableElements.tsx | 4 +- .../V2/Components/UI/specs/Paginator.cy.tsx | 24 +- app/react/V2/Components/UI/specs/Table.cy.tsx | 9 + .../V2/Routes/Settings/IX/IXDashboard.tsx | 43 +- .../V2/Routes/Settings/IX/IXSuggestions.tsx | 357 ++++++++ .../V2/Routes/Settings/IX/components/Dot.tsx | 21 + .../Settings/IX/components/ExtractorModal.tsx | 7 +- .../IX/components/FiltersSidepanel.tsx | 217 +++++ .../Settings/IX/components/PDFSidepanel.tsx | 385 +++++++++ .../Settings/IX/components/SelectionError.tsx | 18 + .../Settings/IX/components/SuggestedValue.tsx | 107 +++ .../IX/components/SuggestionsTitle.tsx | 72 ++ .../Settings/IX/components/TableElements.tsx | 180 ++++- app/react/V2/Routes/Settings/IX/types.ts | 5 +- .../Settings/Languages/LanguagesList.tsx | 8 +- .../Translations/TranslationsList.tsx | 6 +- .../components/TableComponents.tsx | 2 +- .../Users/components/PermissionsListModal.tsx | 8 +- .../Users/components/TableComponents.tsx | 16 +- app/react/V2/api/entities/formatter.ts | 49 ++ app/react/V2/api/entities/index.ts | 64 ++ .../V2/api/entities/specs/formatter.spec.ts | 44 + app/react/V2/api/files/index.ts | 26 + app/react/V2/api/ix/extractors.ts | 45 ++ app/react/V2/api/ix/suggestions.ts | 57 ++ app/react/V2/shared/helpers.ts | 7 + app/react/V2/shared/types.ts | 2 +- app/react/stories/Forms/Checkbox.stories.tsx | 49 ++ .../stories/Forms/InputField.stories.tsx | 1 + app/react/stories/PDF.stories.tsx | 27 +- app/react/stories/Paginator.stories.tsx | 4 +- app/react/stories/Pill.stories.tsx | 5 +- app/react/stories/Table.stories.tsx | 14 +- app/shared/data_utils/promiseUtils.ts | 10 + .../data_utils/specs/promiseUtils.spec.ts | 20 + app/shared/getIXSuggestionState.ts | 129 +-- app/shared/specs/getIXSuggestionState.spec.ts | 130 ++- app/shared/types/react-table-config.d.ts | 4 +- app/shared/types/suggestionSchema.ts | 135 +++- app/shared/types/suggestionType.d.ts | 97 ++- contents/ui-translations/ar.csv | 49 +- contents/ui-translations/en.csv | 49 +- contents/ui-translations/es.csv | 49 +- contents/ui-translations/fr.csv | 49 +- contents/ui-translations/ko.csv | 49 +- contents/ui-translations/my.csv | 49 +- contents/ui-translations/ru.csv | 47 +- contents/ui-translations/th.csv | 49 +- contents/ui-translations/tr.csv | 49 +- ...ate an entity and check it is saved #0.png | Bin 0 -> 18973 bytes ...publishing status with mixed access #0.png | Bin 0 -> 6447 bytes ...llaborator should create an entity #0.png | Bin 13097 -> 0 bytes ...ing status if mixed access selected #0.png | Bin 6363 -> 0 bytes ...uld not be able to share the entity #0.png | Bin 23653 -> 0 bytes ...e able to unshare entities publicly #0.png | Bin 25782 -> 0 bytes ...laborator should show mixed access #0.png | Bin 20391 -> 0 bytes ...olaborator in the shared User Group #0.png | Bin 45242 -> 0 bytes ...y should unshare entities publicly #0.png | Bin 37712 -> 0 bytes ...eck table display and accessibility #0.png | Bin 45303 -> 47503 bytes ...ith the pdf and selection rectangle #0.png | Bin 0 -> 158600 bytes ...hould find suggestions successfully #0.png | Bin 59090 -> 0 bytes ...view should show filters sidepanel #0.png | Bin 13572 -> 0 bytes ...l suggestion states as Empty Label #0.png | Bin 59437 -> 0 bytes ...splay suggestions and be accessible #0.png | Bin 0 -> 103000 bytes .../e2e/settings/information-extraction.cy.ts | 146 +++- cypress/e2e/share-publicly.cy.ts | 8 +- ...isplay-entity-relationship-page-1-snap.png | Bin 111169 -> 109739 bytes e2e/suite1/entity-view-page.test.ts | 38 +- e2e/suite1/entity.test.ts | 8 +- tailwind.config.js | 1 + .../dump/uwazi_development/activitylogs.bson | Bin 633237 -> 633671 bytes .../activitylogs.metadata.json | 2 +- .../connections.metadata.json | 2 +- .../dictionaries.metadata.json | 2 +- .../dump/uwazi_development/entities.bson | Bin 759094 -> 759136 bytes .../uwazi_development/entities.metadata.json | 2 +- .../dump/uwazi_development/files.bson | Bin 9910 -> 1029210 bytes .../uwazi_development/files.metadata.json | 2 +- .../migrationHubRecords.bson | 0 .../migrationHubRecords.metadata.json | 1 + .../dump/uwazi_development/migrations.bson | Bin 26173 -> 27924 bytes .../migrations.metadata.json | 2 +- .../dump/uwazi_development/ocr_records.bson | 0 .../ocr_records.metadata.json | 1 + .../uwazi_development/pages.metadata.json | 2 +- .../passwordrecoveries.metadata.json | 2 +- .../relationshipMigrationFields.bson | 0 .../relationshipMigrationFields.metadata.json | 1 + .../relationships.metadata.json | 2 +- .../relationtypes.metadata.json | 2 +- .../uwazi_development/sessions.metadata.json | 2 +- .../uwazi_development/settings.metadata.json | 2 +- .../uwazi_development/templates.metadata.json | 2 +- .../dump/uwazi_development/translations.bson | Bin 320596 -> 311171 bytes .../translations.metadata.json | 2 +- .../uwazi_development/translationsV2.bson | Bin 0 -> 600932 bytes .../translationsV2.metadata.json | 1 + .../dump/uwazi_development/updatelogs.bson | Bin 333293 -> 658509 bytes .../updatelogs.metadata.json | 2 +- .../usergroups.metadata.json | 2 +- .../uwazi_development/users.metadata.json | 2 +- .../1696528618412uustivo365g.pdf | Bin 0 -> 863670 bytes .../1696528628407n5zieoppbvs.pdf | Bin 0 -> 2452193 bytes .../1696528636296ltasage85s.pdf | Bin 0 -> 2463578 bytes .../1696528645940jvx0yj1nmfp.pdf | Bin 0 -> 958026 bytes .../1696528655857yah2eb3ubuc.pdf | Bin 0 -> 693895 bytes .../1696528667419l50huzjqp2.pdf | Bin 0 -> 2035633 bytes .../651ef8ea056e7eed35fc5e9c.jpg | Bin 0 -> 17990 bytes .../651ef8f4056e7eed35fc5ee4.jpg | Bin 0 -> 17972 bytes .../651ef8fc056e7eed35fc5f2a.jpg | Bin 0 -> 19680 bytes .../651ef905056e7eed35fc5f70.jpg | Bin 0 -> 21044 bytes .../651ef90f056e7eed35fc5fb6.jpg | Bin 0 -> 19357 bytes .../651ef91b056e7eed35fc5ffc.jpg | Bin 0 -> 20049 bytes 178 files changed, 6438 insertions(+), 4104 deletions(-) create mode 100644 app/api/migrations/migrations/145-remove_obsolete_mongo_index/index.js create mode 100644 app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/145-remove_obsolete_mongo_index.spec.js create mode 100644 app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/fixtures.js create mode 100644 app/api/migrations/migrations/146-update_translations/index.js create mode 100644 app/api/migrations/migrations/146-update_translations/specs/146-update_translations.spec.js create mode 100644 app/api/migrations/migrations/146-update_translations/specs/fixtures.js create mode 100644 app/api/suggestions/specs/customFilters.spec.ts delete mode 100644 app/api/suggestions/specs/stats.spec.ts delete mode 100644 app/api/suggestions/stats.ts delete mode 100644 app/react/MetadataExtraction/CancelFindingSuggestionsModal.tsx delete mode 100644 app/react/MetadataExtraction/EntitySuggestions.tsx delete mode 100644 app/react/MetadataExtraction/FilterSidePanel.tsx delete mode 100644 app/react/MetadataExtraction/GridChart.tsx delete mode 100644 app/react/MetadataExtraction/PDFSidePanel.tsx delete mode 100644 app/react/MetadataExtraction/PropertyConfigurationModal.tsx delete mode 100644 app/react/MetadataExtraction/SuggestionAcceptanceModal.tsx delete mode 100644 app/react/MetadataExtraction/SuggestionsAPI.ts delete mode 100644 app/react/MetadataExtraction/SuggestionsContainer.tsx delete mode 100644 app/react/MetadataExtraction/SuggestionsTable.tsx delete mode 100644 app/react/MetadataExtraction/TrainingHealthDashboard.tsx delete mode 100644 app/react/MetadataExtraction/TrainingHealthLegend.tsx delete mode 100644 app/react/MetadataExtraction/actions/actions.ts delete mode 100644 app/react/MetadataExtraction/specs/EntitySuggestions.spec.tsx delete mode 100644 app/react/MetadataExtraction/specs/GridChart.spec.tsx delete mode 100644 app/react/MetadataExtraction/specs/PDFSidePanel.spec.tsx delete mode 100644 app/react/MetadataExtraction/specs/PDFSidepanelFixtures.ts delete mode 100644 app/react/MetadataExtraction/specs/PropertyConfigurationModal.spec.tsx delete mode 100644 app/react/MetadataExtraction/specs/SuggestionsAPI.spec.ts delete mode 100644 app/react/MetadataExtraction/specs/TrainingHealthDashboard.spec.tsx delete mode 100644 app/react/MetadataExtraction/specs/fixtures.ts delete mode 100644 app/react/MetadataExtraction/useContainerWidthHook.ts create mode 100644 app/react/V2/Components/Forms/Checkbox.tsx create mode 100644 app/react/V2/Components/PDFViewer/functions/handleTextSelection.ts create mode 100644 app/react/V2/Components/PDFViewer/functions/specs/fixtures.ts create mode 100644 app/react/V2/Components/PDFViewer/functions/specs/handleTextSelection.spec.ts create mode 100644 app/react/V2/Routes/Settings/IX/IXSuggestions.tsx create mode 100644 app/react/V2/Routes/Settings/IX/components/Dot.tsx create mode 100644 app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx create mode 100644 app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx create mode 100644 app/react/V2/Routes/Settings/IX/components/SelectionError.tsx create mode 100644 app/react/V2/Routes/Settings/IX/components/SuggestedValue.tsx create mode 100644 app/react/V2/Routes/Settings/IX/components/SuggestionsTitle.tsx create mode 100644 app/react/V2/api/entities/formatter.ts create mode 100644 app/react/V2/api/entities/index.ts create mode 100644 app/react/V2/api/entities/specs/formatter.spec.ts create mode 100644 app/react/V2/api/files/index.ts create mode 100644 app/react/V2/api/ix/extractors.ts create mode 100644 app/react/V2/api/ix/suggestions.ts create mode 100644 app/react/V2/shared/helpers.ts create mode 100644 app/react/stories/Forms/Checkbox.stories.tsx create mode 100644 app/shared/data_utils/promiseUtils.ts create mode 100644 app/shared/data_utils/specs/promiseUtils.spec.ts create mode 100644 cypress/e2e/__image_snapshots__/Permisions system as a collaborator should create an entity and check it is saved #0.png create mode 100644 cypress/e2e/__image_snapshots__/Permisions system mixed permissions should keep publishing status with mixed access #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly as a collaborator should create an entity #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly as a collaborator should keep publishing status if mixed access selected #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly as a collaborator should not be able to share the entity #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly as a collaborator should not be able to unshare entities publicly #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly as a collaborator should show mixed access #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly should create a colaborator in the shared User Group #0.png delete mode 100644 cypress/e2e/__image_snapshots__/Share Publicly should unshare entities publicly #0.png create mode 100644 cypress/e2e/settings/__image_snapshots__/Information Extraction PDF sidepanel should display the PDF sidepanel with the pdf and selection rectangle #0.png delete mode 100644 cypress/e2e/settings/__image_snapshots__/Information Extraction Review should find suggestions successfully #0.png delete mode 100644 cypress/e2e/settings/__image_snapshots__/Information Extraction Review should show filters sidepanel #0.png delete mode 100644 cypress/e2e/settings/__image_snapshots__/Information Extraction Review should show title initial suggestion states as Empty Label #0.png create mode 100644 cypress/e2e/settings/__image_snapshots__/Information Extraction Suggestions review should display suggestions and be accessible #0.png create mode 100644 uwazi-fixtures/dump/uwazi_development/migrationHubRecords.bson create mode 100644 uwazi-fixtures/dump/uwazi_development/migrationHubRecords.metadata.json create mode 100644 uwazi-fixtures/dump/uwazi_development/ocr_records.bson create mode 100644 uwazi-fixtures/dump/uwazi_development/ocr_records.metadata.json create mode 100644 uwazi-fixtures/dump/uwazi_development/relationshipMigrationFields.bson create mode 100644 uwazi-fixtures/dump/uwazi_development/relationshipMigrationFields.metadata.json create mode 100644 uwazi-fixtures/dump/uwazi_development/translationsV2.bson create mode 100644 uwazi-fixtures/dump/uwazi_development/translationsV2.metadata.json create mode 100644 uwazi-fixtures/uploaded_documents/1696528618412uustivo365g.pdf create mode 100644 uwazi-fixtures/uploaded_documents/1696528628407n5zieoppbvs.pdf create mode 100644 uwazi-fixtures/uploaded_documents/1696528636296ltasage85s.pdf create mode 100644 uwazi-fixtures/uploaded_documents/1696528645940jvx0yj1nmfp.pdf create mode 100644 uwazi-fixtures/uploaded_documents/1696528655857yah2eb3ubuc.pdf create mode 100644 uwazi-fixtures/uploaded_documents/1696528667419l50huzjqp2.pdf create mode 100644 uwazi-fixtures/uploaded_documents/651ef8ea056e7eed35fc5e9c.jpg create mode 100644 uwazi-fixtures/uploaded_documents/651ef8f4056e7eed35fc5ee4.jpg create mode 100644 uwazi-fixtures/uploaded_documents/651ef8fc056e7eed35fc5f2a.jpg create mode 100644 uwazi-fixtures/uploaded_documents/651ef905056e7eed35fc5f70.jpg create mode 100644 uwazi-fixtures/uploaded_documents/651ef90f056e7eed35fc5fb6.jpg create mode 100644 uwazi-fixtures/uploaded_documents/651ef91b056e7eed35fc5ffc.jpg diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 11e60ffe78..89099bd8ec 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -166,7 +166,6 @@ const fixtures: DBFixture = { propertyName: 'property 1', extractorId: fixturesFactory.id('property_1_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, @@ -179,7 +178,6 @@ const fixtures: DBFixture = { propertyName: 'property 2', extractorId: fixturesFactory.id('property_2_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, @@ -192,7 +190,6 @@ const fixtures: DBFixture = { propertyName: 'property 1', extractorId: fixturesFactory.id('property_1_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, @@ -205,7 +202,6 @@ const fixtures: DBFixture = { propertyName: 'property 2', extractorId: fixturesFactory.id('property_2_extractor'), date: 1654002449676, - state: 'Empty / Label', segment: '', suggestedValue: '', }, diff --git a/app/api/migrations/migrations/145-remove_obsolete_mongo_index/index.js b/app/api/migrations/migrations/145-remove_obsolete_mongo_index/index.js new file mode 100644 index 0000000000..9139c390e3 --- /dev/null +++ b/app/api/migrations/migrations/145-remove_obsolete_mongo_index/index.js @@ -0,0 +1,37 @@ +const INDICES_TO_REMOVE = { + ixsuggestions: ['extractorId_1_date_1_state_-1', 'extractorId_1_entityTemplate_1_state_1'], +}; + +const handleCollection = async (collection, indexNames) => { + for (let j = 0; j < indexNames.length; j += 1) { + const indexName = indexNames[j]; + // eslint-disable-next-line no-await-in-loop + if (await collection.indexExists(indexName)) await collection.dropIndex(indexName); + } +}; + +export default { + delta: 145, + + name: 'remove_obsolete_mongo_index', + + description: 'Removes one or more obsolete indices from mongodb.', + + reindex: false, + + async up(db) { + process.stdout.write(`${this.name}...\r\n`); + + const existingCollections = new Set( + (await db.collections()).map(collection => collection.collectionName) + ); + const indices = Object.entries(INDICES_TO_REMOVE).filter(pair => + existingCollections.has(pair[0]) + ); + + for (let i = 0; i < indices.length; i += 1) { + // eslint-disable-next-line no-await-in-loop + await handleCollection(await db.collection(indices[i][0]), indices[i][1]); + } + }, +}; diff --git a/app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/145-remove_obsolete_mongo_index.spec.js b/app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/145-remove_obsolete_mongo_index.spec.js new file mode 100644 index 0000000000..fb0ad4ff25 --- /dev/null +++ b/app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/145-remove_obsolete_mongo_index.spec.js @@ -0,0 +1,57 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import { fixtures } from './fixtures.js'; + +describe('migration remove_obsolete_mongo_index', () => { + let suggestionsIndexInfo; + + const createIndexes = async db => { + await db.collection('ixsuggestions').createIndex({ entityId: 1 }); + await db.collection('ixsuggestions').createIndex({ fileId: 1 }); + await db.collection('ixsuggestions').createIndex({ extractorId: 1, entityId: 1, fileId: 1 }); + await db.collection('ixsuggestions').createIndex({ extractorId: 1, date: 1, state: -1 }); + await db + .collection('ixsuggestions') + .createIndex({ extractorId: 1, entityTemplate: 1, state: 1 }); + }; + + const getIndexInfo = async db => { + suggestionsIndexInfo = await db.collection('ixsuggestions').indexInformation(); + }; + + beforeAll(async () => { + jest.spyOn(process.stdout, 'write').mockImplementation(() => {}); + await testingDB.setupFixturesAndContext(fixtures); + const db = testingDB.mongodb; + await createIndexes(db); + await migration.up(db); + await getIndexInfo(db); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(145); + }); + + it('should remove the targeted indices', async () => { + expect(suggestionsIndexInfo['extractorId_1_date_1_state_-1']).toBe(undefined); + expect(suggestionsIndexInfo.extractorId_1_entityTemplate_1_state_1).toBe(undefined); + }); + + it('should leave the other indices intact', async () => { + expect(suggestionsIndexInfo.entityId_1).toEqual([['entityId', 1]]); + expect(suggestionsIndexInfo.fileId_1).toEqual([['fileId', 1]]); + expect(suggestionsIndexInfo.extractorId_1_entityId_1_fileId_1).toEqual([ + ['extractorId', 1], + ['entityId', 1], + ['fileId', 1], + ]); + }); + + it('should check if a reindex is needed', async () => { + expect(migration.reindex).toBe(false); + }); +}); diff --git a/app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/fixtures.js b/app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/fixtures.js new file mode 100644 index 0000000000..cbf2144390 --- /dev/null +++ b/app/api/migrations/migrations/145-remove_obsolete_mongo_index/specs/fixtures.js @@ -0,0 +1,4 @@ +export const fixtures = { + entities: [{ title: 'test_doc' }], + ixsuggestions: [{ propertyName: 'some name', state: 'Obsolete' }], +}; diff --git a/app/api/migrations/migrations/146-update_translations/index.js b/app/api/migrations/migrations/146-update_translations/index.js new file mode 100644 index 0000000000..ac68b4eeb5 --- /dev/null +++ b/app/api/migrations/migrations/146-update_translations/index.js @@ -0,0 +1,108 @@ +const newKeys = [ + { key: 'Suggestion accepted.' }, + { key: 'Showing' }, + { key: 'Accept suggestion' }, + { key: 'Stats & Filters' }, + { key: 'Labeled' }, + { key: 'Non-labeled' }, + { key: 'Pending' }, + { key: 'Clear all' }, + { key: 'Apply' }, + { key: 'Current value:' }, + { key: 'Suggestion:' }, + { key: 'Current Value/Suggestion' }, + { key: 'No context' }, +]; + +const deletedKeys = [ + { key: 'Reviewing' }, + { key: 'Confirm suggestion acceptance' }, + { key: 'Apply to all languages' }, + { key: 'Back to dashboard' }, + { key: 'Match / Label' }, + { key: 'Mismatch / Label' }, + { key: 'Match / Value' }, + { key: 'Mismatch / Value' }, + { key: 'Empty / Label' }, + { key: 'Empty / Value' }, + { key: 'State Legend' }, + { key: 'labelMatchDesc' }, + { key: 'labelMismatchDesc' }, + { key: 'labelEmptyDesc' }, + { key: 'valueMatchDesc' }, + { key: 'valueMismatchDesc' }, + { key: 'valueEmptyDesc' }, + { key: 'obsoleteDesc' }, + { key: 'emptyDesc' }, + { key: 'This will update the entity across all languages' }, + { key: 'Mismatch / Empty' }, + { key: 'Empty / Empty' }, + { key: 'emptyMismatchDesc' }, + { key: 'Non-matching' }, + { key: 'Empty / Obsolete' }, + { key: 'This will cancel the finding suggestion process' }, + { key: 'Add properties' }, + { key: 'Show Filters' }, +]; +const updateTranslation = (currentTranslation, keysToUpdate, loc) => { + const translation = { ...currentTranslation }; + const newTranslation = keysToUpdate.find(row => row.key === currentTranslation.key); + if (newTranslation) { + translation.key = newTranslation.newKey; + if (loc === 'en' || currentTranslation.value === newTranslation.oldValue) { + translation.value = newTranslation.newValue; + } + } + return translation; +}; + +export default { + delta: 146, + + reindex: false, + + name: 'update_translations', + + description: 'Updates some translations for new User/Groups UI in settings', + + async up(db) { + const keysToInsert = newKeys; + const keysToDelete = deletedKeys; + const translations = await db.collection('translations').find().toArray(); + const locToSystemContext = {}; + translations.forEach(tr => { + locToSystemContext[tr.locale] = tr.contexts.find(c => c.id === 'System'); + }); + + const alreadyInDB = []; + Object.entries(locToSystemContext).forEach(([loc, context]) => { + const contextValues = context.values.reduce((newValues, currentTranslation) => { + const deleted = keysToDelete.find( + deletedTranslation => deletedTranslation.key === currentTranslation.key + ); + if (!deleted) { + const translation = updateTranslation(currentTranslation, [], loc); + newValues.push(translation); + } + keysToInsert.forEach(newEntry => { + if (newEntry.key === currentTranslation.key) { + alreadyInDB.push(currentTranslation.key); + } + }); + return newValues; + }, []); + keysToInsert + .filter(k => !alreadyInDB.includes(k.key)) + .forEach(newEntry => { + contextValues.push({ key: newEntry.key, value: newEntry.key }); + }); + context.values = contextValues; + }); + + await Promise.all( + translations.map(tr => db.collection('translations').replaceOne({ _id: tr._id }, tr)) + ); + + process.stdout.write(`${this.name}...\r\n`); + }, +}; diff --git a/app/api/migrations/migrations/146-update_translations/specs/146-update_translations.spec.js b/app/api/migrations/migrations/146-update_translations/specs/146-update_translations.spec.js new file mode 100644 index 0000000000..74362f062c --- /dev/null +++ b/app/api/migrations/migrations/146-update_translations/specs/146-update_translations.spec.js @@ -0,0 +1,75 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures, { templateContext } from './fixtures.js'; + +describe('migration update translations of settings new Users/Groups UI', () => { + beforeEach(async () => { + jest.spyOn(process.stdout, 'write').mockImplementation(() => {}); + await testingDB.setupFixturesAndContext(fixtures); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(146); + }); + + it('should update the keys that have changed', async () => { + await migration.up(testingDB.mongodb); + const allTranslations = await testingDB.mongodb.collection('translations').find().toArray(); + + const uwaziUI = allTranslations.filter(tr => + tr.contexts.filter(ctx => ctx.type === 'Uwazi UI') + ); + + const previousSystemValues = { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }; + + const addedKeys = [ + expect.objectContaining({ + key: 'Suggestion accepted.', + value: 'Suggestion accepted.', + }), + expect.objectContaining({ + key: 'Accept suggestion', + value: 'Accept suggestion', + }), + expect.objectContaining({ + key: 'Showing', + value: 'Showing', + }), + expect.objectContaining({ + key: 'Stats & Filters', + value: 'Stats & Filters', + }), + ]; + const defaultContextContent = expect.objectContaining({ + type: 'Uwazi UI', + values: expect.arrayContaining([previousSystemValues, ...addedKeys]), + }); + expect(uwaziUI).toMatchObject([ + expect.objectContaining({ + locale: 'en', + contexts: [defaultContextContent, templateContext], + }), + expect.objectContaining({ + locale: 'es', + contexts: [ + expect.objectContaining({ + type: 'Uwazi UI', + values: expect.arrayContaining([previousSystemValues, ...addedKeys]), + }), + templateContext, + ], + }), + expect.objectContaining({ + locale: 'pt', + contexts: [defaultContextContent, templateContext], + }), + ]); + }); +}); diff --git a/app/api/migrations/migrations/146-update_translations/specs/fixtures.js b/app/api/migrations/migrations/146-update_translations/specs/fixtures.js new file mode 100644 index 0000000000..f9a29257a5 --- /dev/null +++ b/app/api/migrations/migrations/146-update_translations/specs/fixtures.js @@ -0,0 +1,115 @@ +import db from 'api/utils/testing_db'; + +const templateContext = { + id: db.id(), + label: 'default template', + type: 'Entity', + values: [ + { + key: 'default template', + value: 'default template', + }, + { + key: 'Title', + value: 'Title', + }, + ], +}; + +const fixturesDB = { + translations: [ + { + _id: db.id(), + locale: 'en', + contexts: [ + { + _id: db.id(), + type: 'Uwazi UI', + label: 'User Interface', + id: 'System', + values: [ + { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }, + { + _id: db.id(), + key: 'Can not delete template:', + value: 'Can not delete template: changed', + }, + { + _id: db.id(), + key: '- Site page:', + value: '- Site page:', + }, + { _id: db.id(), key: 'Document OCR trigger', value: 'Document OCR trigger' }, + ], + }, + templateContext, + ], + }, + { + _id: db.id(), + locale: 'es', + contexts: [ + { + _id: db.id(), + type: 'Uwazi UI', + label: 'User Interface', + id: 'System', + values: [ + { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }, + { + _id: db.id(), + key: 'Confirm delete relationship type:', + value: 'Confirmar eliminación de tipo de relación:', + }, + { + _id: db.id(), + key: '- Site page:', + value: '- Sito:', + }, + { _id: db.id(), key: 'Document OCR trigger', value: 'Document OCR trigger' }, + ], + }, + templateContext, + ], + }, + { + _id: db.id(), + locale: 'pt', + contexts: [ + { + _id: db.id(), + type: 'Uwazi UI', + label: 'User Interface', + id: 'System', + values: [ + { + key: 'existing-key-in-system', + value: 'existing-key-in-system', + }, + { + _id: db.id(), + key: 'Can not delete template:', + value: 'Can not delete template:', + }, + { + _id: db.id(), + key: '- Site page:', + value: '- Site page:', + }, + { _id: db.id(), key: 'Document OCR trigger', value: 'Document OCR trigger' }, + ], + }, + templateContext, + ], + }, + ], +}; + +export { templateContext }; +export default fixturesDB; diff --git a/app/api/services/informationextraction/InformationExtraction.ts b/app/api/services/informationextraction/InformationExtraction.ts index fa68352937..9403417b57 100644 --- a/app/api/services/informationextraction/InformationExtraction.ts +++ b/app/api/services/informationextraction/InformationExtraction.ts @@ -370,7 +370,7 @@ class InformationExtraction { return { status: 'ready', message: 'Ready' }; } - return { status: 'error', message: '' }; + return { status: 'error', message: 'No model found' }; }; materialsForModel = async (extractor: IXExtractorType, serviceUrl: string) => { diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index 1664d904c0..c4500dec24 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -14,7 +14,6 @@ import settings from 'api/settings/settings'; import templatesModel from 'api/templates/templates'; import { propertyTypes } from 'shared/propertyTypes'; import languages from 'shared/languages'; -import { SuggestionState } from 'shared/types/suggestionSchema'; const BATCH_SIZE = 50; const MAX_TRAINING_FILES_NUMBER = 500; @@ -130,7 +129,7 @@ async function getFilesForSuggestions(extractorId: ObjectIdSchema) { { extractorId, date: { $lt: currentModel.creationDate }, - state: { $ne: SuggestionState.error }, + 'state.error': { $ne: true }, }, 'fileId', { limit: BATCH_SIZE } diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index 7a20ee654e..8df449e769 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -1,12 +1,13 @@ /* eslint-disable max-lines */ +// eslint-disable-next-line node/no-restricted-import +import fs from 'fs/promises'; + import { testingEnvironment } from 'api/utils/testingEnvironment'; import { testingTenants } from 'api/utils/testingTenants'; import { IXSuggestionsModel } from 'api/suggestions/IXSuggestionsModel'; -import { SuggestionState } from 'shared/types/suggestionSchema'; import { ResultsMessage } from 'api/services/tasksmanager/TaskManager'; import * as setupSockets from 'api/socketio/setupSockets'; -// eslint-disable-next-line node/no-restricted-import -import fs from 'fs/promises'; + import { factory, fixtures } from './fixtures'; import { InformationExtraction } from '../InformationExtraction'; import { ExternalDummyService } from '../../tasksmanager/specs/ExternalDummyService'; @@ -301,7 +302,16 @@ describe('InformationExtraction', () => { expect.objectContaining({ entityId: 'A1', status: 'processing', - state: SuggestionState.processing, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: true, + obsolete: false, + error: false, + }, }) ); }); @@ -379,7 +389,16 @@ describe('InformationExtraction', () => { suggestedValue: 'suggestion_text_1', segment: 'segment_text_1', status: 'ready', - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, }) ); }); @@ -444,7 +463,16 @@ describe('InformationExtraction', () => { propertyName: 'property1', status: 'ready', suggestedValue: 'text_in_other_language', - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, }) ); @@ -454,7 +482,16 @@ describe('InformationExtraction', () => { propertyName: 'property1', status: 'ready', suggestedValue: 'text_in_eng_language', - state: SuggestionState.valueMismatch, + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, }) ); }); @@ -491,7 +528,16 @@ describe('InformationExtraction', () => { segment: '', status: 'failed', error: 'Issue calculation suggestion', - state: SuggestionState.error, + state: { + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + processing: false, + obsolete: false, + error: true, + }, }) ); }); diff --git a/app/api/services/informationextraction/specs/fixtures.ts b/app/api/services/informationextraction/specs/fixtures.ts index 7b40c53432..0155e1542e 100644 --- a/app/api/services/informationextraction/specs/fixtures.ts +++ b/app/api/services/informationextraction/specs/fixtures.ts @@ -289,7 +289,16 @@ const fixtures: DBFixture = { suggestedValue: '', segment: '', status: 'ready', - state: 'Obsolete', + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: true, + processing: false, + error: false, + }, date: 100, }, { @@ -302,7 +311,16 @@ const fixtures: DBFixture = { suggestedValue: '', segment: '', status: 'ready', - state: 'Error', + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: true, + processing: false, + error: true, + }, date: 100, }, ], diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index a9a2826f2f..99075f6a98 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -4,7 +4,7 @@ import { Suggestions } from 'api/suggestions/suggestions'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import db, { DBFixture, testingDB } from 'api/utils/testing_db'; -import { SuggestionState } from 'shared/types/suggestionSchema'; +import { IXSuggestionStateType } from 'shared/types/suggestionType'; import { Extractors } from '../ixextractors'; const fixtureFactory = getFixturesFactory(); @@ -125,6 +125,28 @@ const fixtures: DBFixture = { ], }; +const emptyState: IXSuggestionStateType = { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, +}; + +const expectedStates: Record = { + onlyLabeled: { + ...emptyState, + labeled: true, + }, + onlyValue: { + ...emptyState, + withValue: true, + }, +}; + describe('ixextractors', () => { beforeEach(async () => { await testingEnvironment.setUp(fixtures); @@ -161,7 +183,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.labelEmpty, + state: expectedStates.onlyLabeled, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, { @@ -173,7 +195,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.labelEmpty, + state: expectedStates.onlyLabeled, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, ], @@ -196,7 +218,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('animalTemplate').toString(), }, { @@ -208,7 +230,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('animalTemplate').toString(), }, { @@ -220,7 +242,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, { @@ -232,7 +254,7 @@ describe('ixextractors', () => { error: '', segment: '', suggestedValue: '', - state: SuggestionState.valueEmpty, + state: expectedStates.onlyValue, entityTemplate: fixtureFactory.id('personTemplate').toString(), }, ], diff --git a/app/api/suggestions/IXSuggestionsModel.ts b/app/api/suggestions/IXSuggestionsModel.ts index b5c295b66b..070e0aceb2 100644 --- a/app/api/suggestions/IXSuggestionsModel.ts +++ b/app/api/suggestions/IXSuggestionsModel.ts @@ -15,8 +15,11 @@ const mongoSchema = new mongoose.Schema(props, { mongoSchema.index({ entityId: 1 }); mongoSchema.index({ fileId: 1 }); mongoSchema.index({ extractorId: 1, entityId: 1, fileId: 1 }); -mongoSchema.index({ extractorId: 1, date: 1, state: -1 }); -mongoSchema.index({ extractorId: 1, entityTemplate: 1, state: 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.match': 1 }); +mongoSchema.index({ extractorId: 1, 'tate.labeled': 1, 'state.withSuggestion': 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.hasContext': 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.obsolete': 1 }); +mongoSchema.index({ extractorId: 1, 'state.labeled': 1, 'state.error': 1 }); const IXSuggestionsModel = instanceModel('ixsuggestions', mongoSchema); diff --git a/app/api/suggestions/pipelineStages.ts b/app/api/suggestions/pipelineStages.ts index 695c62ffbf..3e065c1e74 100644 --- a/app/api/suggestions/pipelineStages.ts +++ b/app/api/suggestions/pipelineStages.ts @@ -1,16 +1,79 @@ import { ObjectId } from 'mongodb'; import { FilterQuery } from 'mongoose'; import { LanguagesListSchema } from 'shared/types/commonTypes'; -import { IXSuggestionType } from 'shared/types/suggestionType'; +import { IXSuggestionType, SuggestionCustomFilter } from 'shared/types/suggestionType'; -export const getMatchStage = (filters: FilterQuery) => [ - { - $match: { - ...filters, - status: { $ne: 'processing' }, - }, +export const baseQueryFragment = (extractorId: ObjectId, ignoreProcessing = true) => { + const query: FilterQuery = { + extractorId, + }; + if (ignoreProcessing) { + query.status = { $ne: 'processing' }; + } + return query; +}; + +export const filterFragments = { + labeled: { + _fragment: { 'state.labeled': true }, + match: { 'state.labeled': true, 'state.match': true }, + mismatch: { 'state.labeled': true, 'state.match': false }, }, -]; + nonLabeled: { + _fragment: { 'state.labeled': false }, + noSuggestion: { 'state.labeled': false, 'state.withSuggestion': false }, + noContext: { 'state.labeled': false, 'state.hasContext': false }, + obsolete: { 'state.labeled': false, 'state.obsolete': true }, + others: { 'state.labeled': false, 'state.error': true }, + }, +}; + +export const translateCustomFilter = (customFilter: SuggestionCustomFilter) => { + const orFilters = []; + if (customFilter.labeled.match) { + orFilters.push(filterFragments.labeled.match); + } + if (customFilter.labeled.mismatch) { + orFilters.push(filterFragments.labeled.mismatch); + } + + if (customFilter.nonLabeled.noSuggestion) { + orFilters.push(filterFragments.nonLabeled.noSuggestion); + } + if (customFilter.nonLabeled.noContext) { + orFilters.push(filterFragments.nonLabeled.noContext); + } + if (customFilter.nonLabeled.obsolete) { + orFilters.push(filterFragments.nonLabeled.obsolete); + } + if (customFilter.nonLabeled.others) { + orFilters.push(filterFragments.nonLabeled.others); + } + return orFilters; +}; + +export const getMatchStage = ( + extractorId: ObjectId, + customFilter: SuggestionCustomFilter | undefined, + countOnly = false +) => { + const matchQuery: FilterQuery = baseQueryFragment(extractorId); + if (customFilter) { + const orFilters = translateCustomFilter(customFilter); + if (orFilters.length > 0) matchQuery.$or = orFilters; + } + + const countExpression = countOnly ? [{ $count: 'count' }] : []; + + const matchStage = [ + { + $match: matchQuery, + }, + ...countExpression, + ]; + + return matchStage; +}; export const getEntityStage = (languages: LanguagesListSchema) => { const defaultLanguage = languages.find(l => l.default)?.key; @@ -152,12 +215,11 @@ export const getEntityTemplateFilterStage = (entityTemplates: string[] | undefin ] : []; -export const groupByAndSort = (field: string) => [ +export const groupByAndCount = (field: string) => [ { $group: { _id: field, count: { $sum: 1 }, }, }, - { $sort: { _id: 1 } }, ]; diff --git a/app/api/suggestions/routes.ts b/app/api/suggestions/routes.ts index f59f8aeb6f..01bc8d1679 100644 --- a/app/api/suggestions/routes.ts +++ b/app/api/suggestions/routes.ts @@ -1,5 +1,5 @@ /* eslint-disable max-lines */ -import { Application, NextFunction, Request, Response } from 'express'; +import { Application, Request, Response } from 'express'; import { ObjectId } from 'mongodb'; import { Suggestions } from 'api/suggestions/suggestions'; @@ -7,14 +7,15 @@ import { InformationExtraction } from 'api/services/informationextraction/Inform import { validateAndCoerceRequest } from 'api/utils/validateRequest'; import { needsAuthorization } from 'api/auth'; import { parseQuery } from 'api/utils/parseQueryMiddleware'; -import { - IXSuggestionsStatsQuerySchema, - SuggestionsQueryFilterSchema, -} from 'shared/types/suggestionSchema'; +import { ObjectIdSchema } from 'shared/types/commonTypes'; +import { SuggestionsQueryFilterSchema } from 'shared/types/suggestionSchema'; import { objectIdSchema } from 'shared/types/commonSchemas'; -import { IXSuggestionsFilter, IXSuggestionsStatsQuery } from 'shared/types/suggestionType'; +import { + IXAggregationQuery, + IXSuggestionAggregation, + IXSuggestionsQuery, +} from 'shared/types/suggestionType'; import { serviceMiddleware } from './serviceMiddleware'; -import { ObjectIdSchema } from 'shared/types/commonTypes'; const IX = new InformationExtraction(); @@ -72,42 +73,59 @@ export const suggestionsRoutes = (app: Application) => { size: { type: 'number', minimum: 1, maximum: 500 }, }, }, + sort: { + type: 'object', + properties: { + property: { type: 'string' }, + order: { type: 'string' }, + }, + }, }, }, }, }), async ( req: Request & { - query: { filter: IXSuggestionsFilter; page: { number: number; size: number } }; + query: IXSuggestionsQuery; }, - res: Response, - _next: NextFunction + res: Response ) => { - const suggestionsList = await Suggestions.get( - { language: req.language, ...req.query.filter }, - { page: req.query.page } - ); + const suggestionsList = await Suggestions.get(req.query.filter, { + page: req.query.page, + sort: req.query.sort, + }); res.json(suggestionsList); } ); app.get( - '/api/suggestions/stats', + '/api/suggestions/aggregation', serviceMiddleware, needsAuthorization(['admin']), parseQuery, validateAndCoerceRequest({ + type: 'object', + definitions: { objectIdSchema }, properties: { - query: IXSuggestionsStatsQuerySchema, + query: { + type: 'object', + additionalProperties: false, + required: ['extractorId'], + properties: { + extractorId: objectIdSchema, + }, + }, }, }), async ( - req: Request & { query: IXSuggestionsStatsQuery }, - res: Response, - _next: NextFunction + req: Request & { + query: IXAggregationQuery; + }, + res: Response ) => { - const stats = await Suggestions.getStats(req.query.extractorId); - res.json(stats); + const { extractorId } = req.query; + const aggregation = await Suggestions.aggregate(extractorId); + res.json(aggregation); } ); @@ -151,26 +169,28 @@ export const suggestionsRoutes = (app: Application) => { body: { type: 'object', additionalProperties: false, - required: ['suggestion', 'allLanguages'], + required: ['suggestions'], properties: { - suggestion: { - type: 'object', - additionalProperties: false, - required: ['_id', 'sharedId', 'entityId'], - properties: { - _id: objectIdSchema, - sharedId: { type: 'string' }, - entityId: { type: 'string' }, + suggestions: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['_id', 'sharedId', 'entityId'], + properties: { + _id: objectIdSchema, + sharedId: { type: 'string' }, + entityId: { type: 'string' }, + }, }, }, - allLanguages: { type: 'boolean' }, }, }, }, }), async (req, res, _next) => { - const { suggestion, allLanguages } = req.body; - await Suggestions.accept(suggestion, allLanguages); + const { suggestions } = req.body; + await Suggestions.accept(suggestions); res.json({ success: true }); } ); diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts new file mode 100644 index 0000000000..1ecab21715 --- /dev/null +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -0,0 +1,223 @@ +import db from 'api/utils/testing_db'; +import { SuggestionCustomFilter } from 'shared/types/suggestionType'; +import { factory, stateFilterFixtures } from './fixtures'; +import { Suggestions } from '../suggestions'; + +const blankCustomFilter: SuggestionCustomFilter = { + labeled: { + match: false, + mismatch: false, + }, + nonLabeled: { + noSuggestion: false, + noContext: false, + obsolete: false, + others: false, + }, +}; + +beforeAll(async () => { + await db.setupFixturesAndContext(stateFilterFixtures); + await Suggestions.updateStates({}); +}); + +afterAll(async () => db.disconnect()); + +describe('suggestions with CustomFilters', () => { + describe('get()', () => { + it('should return all suggestions (except processing) when no custom filter is provided', async () => { + const result = await Suggestions.get( + { + extractorId: factory.id('test_extractor').toString(), + }, + {} + ); + expect(result.suggestions).toMatchObject([ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ]); + }); + + it('should be able to paginate', async () => { + const result = await Suggestions.get( + { + extractorId: factory.id('test_extractor').toString(), + }, + { page: { number: 3, size: 2 } } + ); + expect(result.suggestions).toMatchObject([ + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + ]); + }); + + it.each([ + { + description: 'filtering for labeled - match', + customFilter: { + ...blankCustomFilter, + labeled: { + match: true, + mismatch: false, + }, + }, + expectedSuggestions: [ + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + ], + }, + { + description: 'filtering for labeled - mismatch', + customFilter: { + ...blankCustomFilter, + labeled: { + match: false, + mismatch: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - noSuggestion', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + noSuggestion: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - noContext', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + noContext: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - obsolete', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + obsolete: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - others', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + others: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + ], + }, + { + description: 'filtering for labeled - match and nonLabeled - obsolete', + customFilter: { + ...blankCustomFilter, + labeled: { + match: true, + mismatch: false, + }, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + obsolete: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + ], + }, + { + description: 'filtering for nonLabeled - noSuggestion and nonLabeled - noContext', + customFilter: { + ...blankCustomFilter, + nonLabeled: { + ...blankCustomFilter.nonLabeled, + noSuggestion: true, + noContext: true, + }, + }, + expectedSuggestions: [ + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + ])( + 'should use the custom filter properly when $description', + async ({ customFilter, expectedSuggestions }) => { + const result = await Suggestions.get( + { + extractorId: factory.id('test_extractor').toString(), + customFilter, + }, + {} + ); + expect(result.suggestions).toMatchObject(expectedSuggestions); + } + ); + }); + + describe('aggreagate()', () => { + it('should return correct aggregation', async () => { + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 12, + labeled: { + _count: 4, + match: 2, + mismatch: 2, + }, + nonLabeled: { + _count: 8, + noSuggestion: 2, + noContext: 4, + obsolete: 2, + others: 2, + }, + }); + }); + }); +}); diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index d555806626..5a55c7a4a8 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1,4 +1,6 @@ /* eslint-disable max-lines */ +import _ from 'lodash'; + import { getFixturesFactory } from 'api/utils/fixturesFactory'; import { testingDB, DBFixture } from 'api/utils/testing_db'; @@ -22,27 +24,29 @@ const shared2AgeSuggestionId = testingDB.id(); const file2Id = factory.id('F2'); const file3Id = factory.id('F3'); -const fixtures: DBFixture = { - settings: [ - { - languages: [ - { - default: true, - key: 'en', - label: 'English', - }, - { - key: 'es', - label: 'Spanish', - }, - ], - features: { - metadataExtraction: { - url: 'https://metadataextraction.com', - }, +const ixSettings = [ + { + languages: [ + { + default: true, + key: 'en' as 'en', + label: 'English', + }, + { + key: 'es' as 'es', + label: 'Spanish', + }, + ], + features: { + metadataExtraction: { + url: 'https://metadataextraction.com', }, }, - ], + }, +]; + +const fixtures: DBFixture = { + settings: _.cloneDeep(ixSettings), ixextractors: [ factory.ixExtractor('age_extractor', 'age', ['personTemplate', 'heroTemplate', 'template1']), factory.ixExtractor('title_extractor', 'title', ['heroTemplate']), @@ -122,6 +126,20 @@ const fixtures: DBFixture = { status: 'ready', error: '', }, + { + entityId: 'shared1', + fileId: factory.id('F1'), + entityTemplate: personTemplateId.toString(), + propertyName: 'age', + extractorId: factory.id('age_extractor'), + suggestedValue: '17', + segment: 'Robin is 17.', + language: 'en', + date: 5, + page: 2, + status: 'ready', + error: '', + }, { entityId: 'shared2', entityTemplate: personTemplateId.toString(), @@ -205,6 +223,20 @@ const fixtures: DBFixture = { status: 'ready', error: '', }, + { + entityId: 'shared3', + fileId: factory.id('F7'), + entityTemplate: personTemplateId.toString(), + propertyName: 'super_powers', + extractorId: factory.id('super_powers_extractor'), + suggestedValue: 'puts up with Bruce Wayne', + segment: 'he puts up with Bruce Wayne', + language: 'en', + date: 4000, + page: 3, + status: 'ready', + error: '', + }, { entityId: 'shared4', entityTemplate: personTemplateId.toString(), @@ -427,7 +459,7 @@ const fixtures: DBFixture = { sharedId: 'shared1', title: 'Robin', language: 'en', - metadata: { enemy: [{ value: 'Red Robin' }] }, + metadata: { enemy: [{ value: 'Red Robin' }], age: [{ value: 99 }] }, template: personTemplateId, }, { @@ -436,6 +468,7 @@ const fixtures: DBFixture = { title: 'Robin es', language: 'es', template: personTemplateId, + metadata: { age: [{ value: 99 }] }, }, { _id: testingDB.id(), @@ -462,11 +495,11 @@ const fixtures: DBFixture = { template: personTemplateId, }, { - _id: testingDB.id(), + _id: factory.id('Alfred-english-entity'), sharedId: 'shared3', title: 'Alfred', language: 'en', - metadata: { age: [{ value: 23 }] }, + metadata: { age: [{ value: 23 }], super_powers: [{ value: 'no super powers' }] }, template: personTemplateId, }, { @@ -513,7 +546,9 @@ const fixtures: DBFixture = { sharedId: 'shared7', title: 'The Riddler', language: 'en', - metadata: { first_encountered: [{ value: 1654732800 }] }, + metadata: { + first_encountered: [{ value: 1654732800 }], + }, template: heroTemplateId, }, { @@ -550,6 +585,15 @@ const fixtures: DBFixture = { }, ], files: [ + factory.file('F1', 'shared1', 'document', 'documentRedRobin.pdf', 'eng', '', [ + { + name: 'age', + selection: { + text: '99', + selectionRectangles: [{ top: 0, left: 0, width: 1, height: 2, page: '2' }], + }, + }, + ]), factory.file('F2', 'shared2', 'document', 'documentB.pdf', 'eng', '', [ { name: 'super_powers', @@ -611,6 +655,23 @@ const fixtures: DBFixture = { }, ]), factory.file('F6', 'shared8', 'document', 'documentRiddler.pdf', 'eng', '', []), + factory.file('F7', 'shared3', 'document', 'documentAlfred.pdf', 'eng', '', [ + { + name: 'super_powers', + selection: { + text: 'no super powers', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + ]), ], templates: [ { @@ -686,11 +747,392 @@ const fixtures: DBFixture = { ], }; +const stateFilterFixtures: DBFixture = { + settings: _.cloneDeep(ixSettings), + templates: [ + factory.template('template1', [ + factory.property('testprop', 'text'), + factory.property('unusedprop', 'text'), + ]), + ], + entities: [ + ...factory.entityInMultipleLanguages(['es', 'en'], 'labeled-match', 'template1', { + testprop: [{ value: 'test-labeled-match' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'labeled-mismatch', 'template1', { + testprop: [{ value: 'test-labeled-mismatch' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-no-suggestion', 'template1', { + testprop: [{ value: 'test-unlabeled-no-suggestion' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-no-context', 'template1', { + testprop: [{ value: 'test-unlabeled-no-context' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-obsolete', 'template1', { + testprop: [{ value: 'test-unlabeled-obsolete' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-processing', 'template1', { + testprop: [{ value: 'test-unlabeled-processing' }], + }), + ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-error', 'template1', { + testprop: [{ value: 'test-unlabeled-error' }], + }), + ], + files: [ + factory.file('label-match-file-en', 'labeled-match', 'document', 'lmfen.pdf', 'en', undefined, [ + factory.fileExtractedMetadata('testprop', 'test-labeled-match'), + ]), + factory.file('label-match-file-es', 'labeled-match', 'document', 'lmfes.pdf', 'es', undefined, [ + factory.fileExtractedMetadata('testprop', 'test-labeled-match'), + ]), + factory.file( + 'label-mismatch-file-en', + 'labeled-mismatch', + 'document', + 'lmismfen.pdf', + 'en', + undefined, + [factory.fileExtractedMetadata('testprop', 'test-labeled-mismatch')] + ), + factory.file( + 'label-mismatch-file-es', + 'labeled-mismatch', + 'document', + 'lmismfes.pdf', + 'es', + undefined, + [factory.fileExtractedMetadata('testprop', 'test-labeled-mismatch')] + ), + factory.file( + 'unlabeled-no-suggestion-file-en', + 'unlabeled-no-suggestion', + 'document', + 'unslfen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-no-suggestion-file-es', + 'unlabeled-no-suggestion', + 'document', + 'unslfes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-no-context-file-en', + 'unlabeled-no-context', + 'document', + 'unlcen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-no-context-file-es', + 'unlabeled-no-context', + 'document', + 'unlces.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-obsolete-file-en', + 'unlabeled-obsolete', + 'document', + 'unloen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-obsolete-file-es', + 'unlabeled-obsolete', + 'document', + 'unloes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-others-file-en', + 'unlabeled-others', + 'document', + 'unlothen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-others-file-es', + 'unlabeled-others', + 'document', + 'unlotes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-processing-file-en', + 'unlabeled-processing', + 'document', + 'unlpen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-processing-file-es', + 'unlabeled-processing', + 'document', + 'unlpes.pdf', + 'es', + undefined + ), + factory.file( + 'unlabeled-error-file-en', + 'unlabeled-error', + 'document', + 'unleen.pdf', + 'en', + undefined + ), + factory.file( + 'unlabeled-error-file-es', + 'unlabeled-error', + 'document', + 'unlees.pdf', + 'es', + undefined + ), + ], + ixmodels: [factory.ixModel('test_model', 'test_extractor', 1000)], + ixextractors: [ + factory.ixExtractor('test_extractor', 'testprop', ['template1']), + factory.ixExtractor('unused_extractor', 'unused_prop', ['template1']), + ], + ixsuggestions: [ + factory.ixSuggestion( + 'label-match-suggestion-en', + 'test_extractor', + 'labeled-match', + 'template1', + 'label-match-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-labeled-match', + } + ), + factory.ixSuggestion( + 'label-match-suggestion-es', + 'test_extractor', + 'labeled-match', + 'template1', + 'label-match-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: 'test-labeled-match', + } + ), + factory.ixSuggestion( + 'label-mismatch-suggestion-en', + 'test_extractor', + 'labeled-mismatch', + 'template1', + 'label-mismatch-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-labeled-mismatch-mismatch', + } + ), + factory.ixSuggestion( + 'label-mismatch-suggestion-es', + 'test_extractor', + 'labeled-mismatch', + 'template1', + 'label-mismatch-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: 'test-labeled-mismatch-mismatch', + } + ), + factory.ixSuggestion( + 'unlabeled-no-suggestion-suggestion-en', + 'test_extractor', + 'unlabeled-no-suggestion', + 'template1', + 'unlabeled-no-suggestion-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: '', + } + ), + factory.ixSuggestion( + 'unlabeled-no-suggestion-suggestion-es', + 'test_extractor', + 'unlabeled-no-suggestion', + 'template1', + 'unlabeled-no-suggestion-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: '', + } + ), + factory.ixSuggestion( + 'unlabeled-no-context-suggestion-en', + 'test_extractor', + 'unlabeled-no-context', + 'template1', + 'unlabeled-no-context-file-en', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-unlabeled-no-context', + } + ), + factory.ixSuggestion( + 'unlabeled-no-context-suggestion-es', + 'test_extractor', + 'unlabeled-no-context', + 'template1', + 'unlabeled-no-context-file-es', + 'testprop', + { + status: 'ready', + date: 1001, + language: 'es', + suggestedValue: 'test-unlabeled-no-context', + } + ), + factory.ixSuggestion( + 'unlabeled-obsolete-suggestion-en', + 'test_extractor', + 'unlabeled-obsolete', + 'template1', + 'unlabeled-obsolete-file-en', + 'testprop', + { + status: 'ready', + date: 999, + language: 'en', + suggestedValue: 'test-unlabeled-obsolete', + segment: 'test-unlabeled-obsolete', + } + ), + factory.ixSuggestion( + 'unlabeled-obsolete-suggestion-es', + 'test_extractor', + 'unlabeled-obsolete', + 'template1', + 'unlabeled-obsolete-file-es', + 'testprop', + { + status: 'ready', + date: 999, + language: 'es', + suggestedValue: 'test-unlabeled-obsolete', + segment: 'test-unlabeled-obsolete', + } + ), + factory.ixSuggestion( + 'unlabeled-processing-suggestion-en', + 'test_extractor', + 'unlabeled-processing', + 'template1', + 'unlabeled-processing-file-en', + 'testprop', + { + status: 'processing', + date: 1001, + language: 'en', + suggestedValue: 'test-unlabeled-processing', + segment: 'test-unlabeled-processing', + } + ), + factory.ixSuggestion( + 'unlabeled-processing-suggestion-es', + 'test_extractor', + 'unlabeled-processing', + 'template1', + 'unlabeled-processing-file-es', + 'testprop', + { + status: 'processing', + date: 1001, + language: 'es', + suggestedValue: 'test-unlabeled-processing', + segment: 'test-unlabeled-processing', + } + ), + factory.ixSuggestion( + 'unlabeled-error-suggestion-en', + 'test_extractor', + 'unlabeled-error', + 'template1', + 'unlabeled-error-file-en', + 'testprop', + { + status: 'failed', + date: 1001, + language: 'en', + suggestedValue: 'test-unlabeled-error', + segment: 'test-unlabeled-error', + error: 'some error happened', + } + ), + factory.ixSuggestion( + 'unlabeled-error-suggestion-es', + 'test_extractor', + 'unlabeled-error', + 'template1', + 'unlabeled-error-file-es', + 'testprop', + { + status: 'failed', + date: 1001, + language: 'es', + suggestedValue: 'test-unlabeled-error', + segment: 'test-unlabeled-error', + error: 'some error happened', + } + ), + factory.ixSuggestion( + 'unusedsuggestion', + 'unused_extractor', + 'unused', + 'template1', + 'unused-file', + 'unusedprop', + { + status: 'ready', + date: 1001, + language: 'en', + suggestedValue: 'test-unused', + } + ), + ], +}; + export { factory, file2Id, file3Id, fixtures, + stateFilterFixtures, shared2esId, shared2enId, shared6enId, diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index ab44bed6ae..79d84052ff 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -3,24 +3,20 @@ import request from 'supertest'; import { Application, NextFunction, Request, Response } from 'express'; import entities from 'api/entities'; -import { WithId } from 'api/odm'; import { search } from 'api/search'; import { factory, fixtures, - heroTemplateId, - personTemplateId, shared2enId, shared2esId, shared6enId, + stateFilterFixtures, suggestionSharedId6Enemy, suggestionSharedId6Title, } from 'api/suggestions/specs/fixtures'; import { suggestionsRoutes } from 'api/suggestions/routes'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import { setUpApp } from 'api/utils/testingRoutes'; -import { EntitySchema } from 'shared/types/entityType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; import { Suggestions } from '../suggestions'; jest.mock( @@ -39,31 +35,31 @@ jest.mock('api/services/informationextraction/InformationExtraction', () => ({ }, })); -const sortAggregateById = (array: { _id: string; count: number }[]) => - array.sort((a, b) => a._id.localeCompare(b._id)); +let user: { username: string; role: string } | undefined; +const getUser = () => user; -describe('suggestions routes', () => { - let user: { username: string; role: string } | undefined; - const getUser = () => user; +beforeEach(async () => { + user = { username: 'user 1', role: 'admin' }; + jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); +}); + +const app: Application = setUpApp( + suggestionsRoutes, + (req: Request, _res: Response, next: NextFunction) => { + (req as any).user = getUser(); + next(); + } +); +afterAll(async () => { + await testingEnvironment.tearDown(); +}); + +describe('suggestions routes', () => { beforeAll(async () => { await testingEnvironment.setUp(fixtures); await Suggestions.updateStates({}); }); - beforeEach(async () => { - user = { username: 'user 1', role: 'admin' }; - jest.spyOn(search, 'indexEntities').mockImplementation(async () => Promise.resolve()); - }); - - const app: Application = setUpApp( - suggestionsRoutes, - (req: Request, _res: Response, next: NextFunction) => { - (req as any).user = getUser(); - next(); - } - ); - - afterAll(async () => testingEnvironment.tearDown()); describe('GET /api/suggestions', () => { it('should return the suggestions filtered by the request language and the property name', async () => { @@ -76,6 +72,26 @@ describe('suggestions routes', () => { }) .expect(200); expect(response.body.suggestions).toMatchObject([ + { + entityId: shared2enId.toString(), + sharedId: 'shared2', + entityTitle: 'Batman en', + propertyName: 'super_powers', + suggestedValue: 'scientific knowledge', + segment: 'he relies on his own scientific knowledge', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, + language: 'en', + page: 5, + }, { entityId: shared2esId.toString(), sharedId: 'shared2', @@ -83,30 +99,41 @@ describe('suggestions routes', () => { propertyName: 'super_powers', suggestedValue: 'scientific knowledge es', segment: 'el confía en su propio conocimiento científico', - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, language: 'es', page: 5, }, { - entityId: shared2enId.toString(), - sharedId: 'shared2', - entityTitle: 'Batman en', + entityId: factory.id('Alfred-english-entity').toString(), + sharedId: 'shared3', + entityTitle: 'Alfred', propertyName: 'super_powers', - suggestedValue: 'scientific knowledge', - segment: 'he relies on his own scientific knowledge', - state: SuggestionState.labelMatch, + suggestedValue: 'puts up with Bruce Wayne', + segment: 'he puts up with Bruce Wayne', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, language: 'en', - page: 5, + page: 3, }, ]); expect(response.body.totalPages).toBe(1); - expect(response.body.aggregations).toMatchObject({ - template: [{ _id: personTemplateId.toString(), count: 2 }], - state: [ - { _id: 'Match / Label', count: 1 }, - { _id: 'Mismatch / Label', count: 1 }, - ], - }); }); it('should include failed suggestions but not processing ones', async () => { @@ -118,30 +145,23 @@ describe('suggestions routes', () => { }, }) .expect(200); - expect(response.body.suggestions).toMatchObject( - expect.arrayContaining([ - expect.objectContaining({ - entityTitle: 'Joker', - propertyName: 'age', - segment: 'Joker age is 45', - sharedId: 'shared4', - state: 'Error', - suggestedValue: null, - }), - ]) + const joker = response.body.suggestions.find( + (suggestion: any) => suggestion.entityTitle === 'Joker' ); - expect(response.body.suggestions).not.toMatchObject( - expect.arrayContaining([ - expect.objectContaining({ - entityTitle: 'Alfred', - propertyName: 'age', - segment: 'Alfred 67 years old processing', - currentValue: 23, - sharedId: 'shared3', - state: 'Mismatch / Value', - }), - ]) + expect(joker).toMatchObject({ + entityTitle: 'Joker', + propertyName: 'age', + segment: 'Joker age is 45', + sharedId: 'shared4', + state: { + error: true, + }, + suggestedValue: null, + }); + const alfred = response.body.suggestions.find( + (suggestion: any) => suggestion.segment === 'Alfred 67 years old processing' ); + expect(alfred).toBeUndefined(); }); describe('pagination', () => { @@ -155,10 +175,12 @@ describe('suggestions routes', () => { page: { number: 2, size: 2 }, }) .expect(200); + expect(response.body.suggestions).toMatchObject([ { entityTitle: 'Alfred' }, { entityTitle: 'Robin' }, ]); + expect(response.body.totalPages).toBe(3); }); @@ -186,146 +208,104 @@ describe('suggestions routes', () => { .query({ filter: { extractorId: factory.id('enemy_extractor').toString(), - states: [SuggestionState.empty], + customFilter: { + labeled: { + match: false, + mismatch: false, + }, + nonLabeled: { + noSuggestion: true, + noContext: false, + obsolete: false, + others: false, + }, + }, }, }) .expect(200); expect(response.body.suggestions).toEqual([ expect.objectContaining({ entityTitle: 'Catwoman', - state: SuggestionState.empty, + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, suggestedValue: '', currentValue: '', }), ]); }); - - it('should filter by entity template', async () => { - const response = await request(app) - .get('/api/suggestions/') - .query({ - filter: { - extractorId: factory.id('title_extractor').toString(), - entityTemplates: [personTemplateId.toString()], - }, - }) - .expect(200); - expect(response.body.suggestions).toMatchObject([ - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared4', - language: 'en', - }, - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared3', - language: 'en', - }, - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared1', - language: 'en', - }, - { - propertyName: 'title', - entityTemplateId: personTemplateId.toString(), - sharedId: 'shared1', - language: 'es', - }, - ]); - }); }); - describe('aggregations', () => { - it('should return aggregations', async () => { + describe('sorting', () => { + it('should sort by entity title', async () => { const response = await request(app) - .get('/api/suggestions/') + .get('/api/suggestions') .query({ filter: { - extractorId: factory.id('title_extractor').toString(), + extractorId: factory.id('super_powers_extractor').toString(), }, + sort: { property: 'entityTitle', order: 'desc' }, }) .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 2 }, - { _id: personTemplateId.toString(), count: 4 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 2 }, - { _id: SuggestionState.valueMismatch, count: 4 }, - ], + + expect(response.body.suggestions[0]).toMatchObject({ + sharedId: 'shared2', + entityTitle: 'Batman es', + language: 'es', }); - }); - it('should return aggregations for a specific template', async () => { - const response = await request(app) - .get('/api/suggestions/') - .query({ - filter: { - extractorId: factory.id('title_extractor').toString(), - entityTemplates: [heroTemplateId.toString()], - }, - }) - .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 2 }, - { _id: personTemplateId.toString(), count: 4 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 1 }, - { _id: SuggestionState.valueMismatch, count: 1 }, - ], + expect(response.body.suggestions[1]).toMatchObject({ + sharedId: 'shared2', + entityTitle: 'Batman en', + language: 'en', }); - }); - it('should return aggregations for a specific state', async () => { - const response = await request(app) - .get('/api/suggestions/') - .query({ - filter: { - extractorId: factory.id('title_extractor').toString(), - states: [SuggestionState.valueMatch], - }, - }) - .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 1 }, - { _id: personTemplateId.toString(), count: 1 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 2 }, - { _id: SuggestionState.valueMismatch, count: 4 }, - ], + expect(response.body.suggestions[2]).toMatchObject({ + sharedId: 'shared3', + entityTitle: 'Alfred', + language: 'en', }); + + expect(response.body.totalPages).toBe(1); }); - it('should return aggregations for a specific template and state', async () => { + it('should sort by current value', async () => { const response = await request(app) - .get('/api/suggestions/') + .get('/api/suggestions') .query({ filter: { - extractorId: factory.id('title_extractor').toString(), - entityTemplates: [heroTemplateId.toString()], - states: [SuggestionState.valueMatch], + extractorId: factory.id('super_powers_extractor').toString(), }, + sort: { property: 'currentValue' }, }) .expect(200); - expect(response.body.aggregations).toMatchObject({ - template: sortAggregateById([ - { _id: heroTemplateId.toString(), count: 1 }, - { _id: personTemplateId.toString(), count: 1 }, - ]), - state: [ - { _id: SuggestionState.valueMatch, count: 1 }, - { _id: SuggestionState.valueMismatch, count: 1 }, - ], + + expect(response.body.suggestions[0]).toMatchObject({ + currentValue: 'conocimiento científico', + entityTitle: 'Batman es', + sharedId: 'shared2', + }); + + expect(response.body.suggestions[1]).toMatchObject({ + currentValue: 'no super powers', + entityTitle: 'Alfred', + sharedId: 'shared3', + }); + + expect(response.body.suggestions[2]).toMatchObject({ + currentValue: 'scientific knowledge', + entityTitle: 'Batman en', + sharedId: 'shared2', }); + + expect(response.body.totalPages).toBe(1); }); }); @@ -391,12 +371,13 @@ describe('suggestions routes', () => { await request(app) .post('/api/suggestions/accept') .send({ - suggestion: { - _id: suggestionSharedId6Title, - sharedId: 'shared6', - entityId: shared6enId, - }, - allLanguages: false, + suggestions: [ + { + _id: suggestionSharedId6Title, + sharedId: 'shared6', + entityId: shared6enId, + }, + ], }) .expect(200); @@ -417,37 +398,7 @@ describe('suggestions routes', () => { '+fullText' ); }); - it('should update the suggestion for all the languages', async () => { - await request(app) - .post('/api/suggestions/accept') - .send({ - allLanguages: true, - suggestion: { - _id: suggestionSharedId6Enemy, - sharedId: 'shared6', - entityId: shared6enId, - }, - }) - .expect(200); - const actualEntities = await entities.get({ sharedId: 'shared6' }); - expect(actualEntities).toMatchObject([ - { - metadata: { enemy: [{ value: 'Batman' }], age: [{ value: 40 }] }, - }, - { - metadata: { enemy: [{ value: 'Batman' }], age: [{ value: 40 }] }, - }, - { - metadata: { enemy: [{ value: 'Batman' }], age: [{ value: 40 }] }, - }, - ]); - const entityIds = actualEntities.map((e: WithId) => e._id); - expect(search.indexEntities).toHaveBeenCalledWith( - { _id: { $in: expect.arrayContaining(entityIds) } }, - '+fullText' - ); - }); it('should reject with unauthorized when user has not admin role', async () => { user = { username: 'user 1', role: 'editor' }; const response = await request(app) @@ -465,3 +416,59 @@ describe('suggestions routes', () => { }); }); }); + +describe('aggregation routes', () => { + describe('GET /api/suggestions/aggregation', () => { + beforeAll(async () => { + await testingEnvironment.setUp(stateFilterFixtures); + await Suggestions.updateStates({}); + }); + + describe('validation', () => { + it('should return a validation error if params are not valid', async () => { + const invalidQuery = { additionParam: true }; + const response = await request(app).get('/api/suggestions/aggregation').query(invalidQuery); + expect(response.status).toBe(400); + + const emptyQuery = {}; + const response2 = await request(app).get('/api/suggestions/aggregation').query(emptyQuery); + expect(response2.status).toBe(400); + }); + }); + + describe('authentication', () => { + it('should reject with unauthorized when the user does not have the admin role', async () => { + user = { username: 'user 1', role: 'editor' }; + const response = await request(app) + .get('/api/suggestions/aggregation') + .query({}) + .expect(401); + expect(response.unauthorized).toBe(true); + }); + }); + + it('should return the aggregation of suggestions', async () => { + const response = await request(app) + .get('/api/suggestions/aggregation') + .query({ + extractorId: factory.id('test_extractor').toString(), + }) + .expect(200); + expect(response.body).toEqual({ + total: 12, + labeled: { + _count: 4, + match: 2, + mismatch: 2, + }, + nonLabeled: { + _count: 8, + noSuggestion: 2, + noContext: 4, + obsolete: 2, + others: 2, + }, + }); + }); + }); +}); diff --git a/app/api/suggestions/specs/stats.spec.ts b/app/api/suggestions/specs/stats.spec.ts deleted file mode 100644 index 4f6b11fdcf..0000000000 --- a/app/api/suggestions/specs/stats.spec.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { getFixturesFactory } from 'api/utils/fixturesFactory'; -import { testingEnvironment } from 'api/utils/testingEnvironment'; -import { DBFixture, testingDB } from 'api/utils/testing_db'; -import { SuggestionState } from 'shared/types/suggestionSchema'; -import { getStats } from '../stats'; - -const fixturesFactory = getFixturesFactory(); - -const suggestionBase = { - entityId: '', - propertyName: 'age', - entityTemplate: fixturesFactory.id('template').toString(), - extractorId: fixturesFactory.id('age_extractor'), - suggestedValue: '', - segment: '', - language: '', -}; - -const fixtures: DBFixture = { - ixsuggestions: [ - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.labelEmpty, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.labelMatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.labelMismatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.valueEmpty, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.valueMatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.valueMismatch, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.obsolete, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.empty, - }, - { - _id: testingDB.id(), - ...suggestionBase, - state: SuggestionState.emptyMismatch, - }, - ], -}; - -beforeAll(async () => { - await testingEnvironment.setUp(fixtures); -}); - -afterAll(async () => { - await testingEnvironment.tearDown(); -}); - -describe('when the property exists', () => { - it('should return the training counts', async () => { - expect(await getStats(fixturesFactory.id('age_extractor').toString())).toMatchObject({ - counts: { - labeled: 3, - nonLabeledMatching: 1, - nonLabeledNotMatching: 1, - emptyOrObsolete: 4, - all: fixtures.ixsuggestions!.length, - }, - }); - }); - - it.each([ - { - state: SuggestionState.labelMatch, - action: 'count as correct', - result: 1, - }, - { - state: SuggestionState.labelMismatch, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.valueMatch, - action: 'count as correct', - result: 1, - }, - { - state: SuggestionState.valueMismatch, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.empty, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.obsolete, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.labelEmpty, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.valueEmpty, - action: 'count as incorrect', - result: 0, - }, - { - state: SuggestionState.error, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.processing, - action: 'not count', - result: 0, - }, - { - state: SuggestionState.emptyMismatch, - action: 'not count', - result: 0, - }, - ])('$state state should $action in accuracy', async ({ state, result }) => { - const input = { - _id: testingDB.id(), - ...suggestionBase, - state, - }; - await testingEnvironment.setUp({ ixsuggestions: [input] }); - const stats = await getStats(fixturesFactory.id('age_extractor').toString()); - expect(stats.accuracy).toEqual(result); - }); - - it.each([ - { - states: [SuggestionState.labelMatch, SuggestionState.labelMismatch, SuggestionState.empty], - }, - { - states: [ - SuggestionState.valueMatch, - SuggestionState.valueMismatch, - SuggestionState.processing, - ], - }, - { - states: [ - SuggestionState.labelMatch, - SuggestionState.labelEmpty, - SuggestionState.emptyMismatch, - ], - }, - { - states: [SuggestionState.valueMatch, SuggestionState.valueEmpty, SuggestionState.error], - }, - ])('should return accuracy correctly', async ({ states }) => { - const inputs = states.map(state => ({ - _id: testingDB.id(), - ...suggestionBase, - state, - })); - await testingEnvironment.setUp({ ixsuggestions: inputs }); - const stats = await getStats(fixturesFactory.id('age_extractor').toString()); - expect(stats.accuracy).toEqual(0.5); - }); -}); - -describe('when the property does not exists', () => { - it('should not fail', async () => { - await getStats(fixturesFactory.id('non_existing_extractor').toString()); - }); -}); diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index 063090b6ea..d340696ee7 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -1,7 +1,11 @@ import db from 'api/utils/testing_db'; -import { EntitySuggestionType, IXSuggestionType } from 'shared/types/suggestionType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; +import { + EntitySuggestionType, + IXSuggestionStateType, + IXSuggestionType, + IXSuggestionsFilter, +} from 'shared/types/suggestionType'; import { Suggestions } from '../suggestions'; import { factory, @@ -15,28 +19,48 @@ import { shared2AgeSuggestionId, } from './fixtures'; -const getSuggestions = async (extractorId: string, size = 5) => - Suggestions.get({ extractorId }, { page: { size, number: 1 } }); +const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => + Suggestions.get(filter, { page: { size, number: 1 } }); const findOneSuggestion = async (query: any): Promise => db.mongodb ?.collection('ixsuggestions') .findOne({ ...query }) as unknown as Promise; -const stateUpdateCases = [ +const stateUpdateCases: { + state: Partial; + reason: string; + suggestionQuery: any; +}[] = [ { - state: SuggestionState.obsolete, - reason: 'the suggestion is older than the model', + state: { obsolete: true }, + reason: 'obsolete, if the suggestion is older than the model', suggestionQuery: { entityId: 'shared5', propertyName: 'age' }, }, { - state: SuggestionState.valueEmpty, - reason: 'entity value exists, file label is empty, suggestion is empty', + state: { + withValue: true, + withSuggestion: false, + labeled: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if entity value exists, file label is empty, suggestion is empty', suggestionQuery: { entityId: 'shared3', propertyName: 'age' }, }, { - state: SuggestionState.labelMatch, - reason: 'file label exists, suggestion and entity value exist and match', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label exists, suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared2', propertyName: 'super_powers', @@ -45,8 +69,17 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelMatch, - reason: 'property is a date, file label exists, suggestion and entity value exist and match', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, + reason: + 'when property is a date, and if file label exists, suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared7', propertyName: 'first_encountered', @@ -54,8 +87,16 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.empty, - reason: 'entity value, file label, suggestion are all empty', + state: { + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if entity value, file label, suggestion are all empty', suggestionQuery: { entityId: 'shared8', propertyName: 'enemy', @@ -63,8 +104,16 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelEmpty, - reason: 'entity value and file label exists, suggestion is empty', + state: { + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if entity value and file label exists, suggestion is empty', suggestionQuery: { entityId: 'shared6', propertyName: 'enemy', @@ -73,8 +122,17 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelEmpty, - reason: 'property is a date, entity value and file label exists, suggestion is empty', + state: { + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: + 'when property is a date, and if entity value and file label exists, suggestion is empty', suggestionQuery: { entityId: 'shared7', propertyName: 'first_encountered', @@ -82,17 +140,33 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.labelMismatch, - reason: 'file label exists, suggestion and entity value exist but do not match', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label exists, suggestion and entity value exist but do not match', suggestionQuery: { propertyName: 'super_powers', language: 'es', }, }, { - state: SuggestionState.labelMismatch, + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + obsolete: false, + processing: false, + error: false, + }, reason: - 'property is a date, file label exists, suggestion and entity value exist but do not match', + 'when property is a date, if file label exists, suggestion and entity value exist but do not match', suggestionQuery: { entityId: 'shared7', propertyName: 'first_encountered', @@ -100,17 +174,33 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.valueMatch, - reason: 'file label is empty, but suggestion and entity value exist and match', + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label is empty, but suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared1', propertyName: 'enemy', }, }, { - state: SuggestionState.valueMatch, + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: true, + obsolete: false, + processing: false, + error: false, + }, reason: - 'property is a date, file label is empty, but suggestion and entity value exist and match', + 'when property is a date, and if file label is empty, but suggestion and entity value exist and match', suggestionQuery: { entityId: 'shared8', propertyName: 'first_encountered', @@ -118,8 +208,16 @@ const stateUpdateCases = [ }, }, { - state: SuggestionState.valueMismatch, - reason: 'file label is empty, suggestion and entity value exist but do not match', + state: { + labeled: false, + withValue: true, + withSuggestion: true, + match: false, + obsolete: false, + processing: false, + error: false, + }, + reason: 'if file label is empty, suggestion and entity value exist but do not match', suggestionQuery: { entityId: 'shared6', propertyName: 'enemy', @@ -201,7 +299,7 @@ describe('suggestions', () => { }, { page: { size: 50, number: 1 } } ); - expect(suggestions.length).toBe(2); + expect(suggestions.length).toBe(3); }); it('should return suggestion and extra entity information', async () => { @@ -210,6 +308,31 @@ describe('suggestions', () => { { page: { size: 50, number: 1 } } ); expect(suggestions).toMatchObject([ + { + fileId: file2Id, + propertyName: 'super_powers', + extractorId: factory.id('super_powers_extractor'), + suggestedValue: 'scientific knowledge', + segment: 'he relies on his own scientific knowledge', + language: 'en', + date: 4, + page: 5, + currentValue: 'scientific knowledge', + labeledValue: 'scientific knowledge', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, + entityId: shared2enId, + sharedId: 'shared2', + entityTitle: 'Batman en', + }, { fileId: file3Id, propertyName: 'super_powers', @@ -221,41 +344,70 @@ describe('suggestions', () => { page: 5, currentValue: 'conocimiento científico', labeledValue: 'conocimiento científico', - state: 'Mismatch / Label', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }, entityId: shared2esId, sharedId: 'shared2', entityTitle: 'Batman es', }, { - fileId: file2Id, + fileId: factory.id('F7'), propertyName: 'super_powers', extractorId: factory.id('super_powers_extractor'), - suggestedValue: 'scientific knowledge', - segment: 'he relies on his own scientific knowledge', + segment: 'he puts up with Bruce Wayne', + currentValue: 'no super powers', + date: 4000, + page: 3, + entityId: factory.id('Alfred-english-entity'), + entityTemplateId: personTemplateId, + entityTitle: 'Alfred', + error: '', + labeledValue: 'no super powers', language: 'en', - date: 4, - page: 5, - currentValue: 'scientific knowledge', - labeledValue: 'scientific knowledge', - state: 'Match / Label', - entityId: shared2enId, - sharedId: 'shared2', - entityTitle: 'Batman en', + sharedId: 'shared3', + state: { + error: false, + hasContext: true, + labeled: true, + match: false, + obsolete: false, + processing: false, + withSuggestion: true, + withValue: true, + }, + suggestedValue: 'puts up with Bruce Wayne', }, ]); }); it('should return match status', async () => { - const { suggestions: superPowersSuggestions } = await getSuggestions( - factory.id('super_powers_extractor').toString() - ); + const { suggestions: superPowersSuggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); expect( superPowersSuggestions.find((s: EntitySuggestionType) => s.language === 'en').state - ).toBe(SuggestionState.labelMatch); + ).toEqual({ + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); const { suggestions: enemySuggestions } = await getSuggestions( - factory.id('enemy_extractor').toString(), + { extractorId: factory.id('enemy_extractor').toString() }, 6 ); @@ -263,61 +415,126 @@ describe('suggestions', () => { enemySuggestions.filter( (s: EntitySuggestionType) => s.sharedId === 'shared6' && s.language === 'en' )[1].state - ).toBe(SuggestionState.labelEmpty); + ).toEqual({ + labeled: false, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); expect( enemySuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared1').state - ).toBe(SuggestionState.valueMatch); + ).toEqual({ + labeled: false, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); expect( enemySuggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared8' && s.language === 'en' ).state - ).toBe(SuggestionState.empty); + ).toEqual({ + labeled: false, + withValue: false, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); - const { suggestions: ageSuggestions } = await getSuggestions( - factory.id('age_extractor').toString() - ); + const { suggestions: ageSuggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); - expect(ageSuggestions.length).toBe(4); - expect(ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared5').state).toBe( - SuggestionState.obsolete - ); + expect(ageSuggestions.length).toBe(5); + expect( + ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared5').state.obsolete + ).toEqual(true); - expect(ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared3').state).toBe( - SuggestionState.valueEmpty - ); + expect( + ageSuggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared3').state + ).toEqual({ + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); }); it('should return mismatch status', async () => { - const { suggestions: superPowersSuggestions } = await getSuggestions( - factory.id('super_powers_extractor').toString() - ); + const { suggestions: superPowersSuggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); expect( superPowersSuggestions.find((s: EntitySuggestionType) => s.language === 'es').state - ).toBe(SuggestionState.labelMismatch); + ).toEqual({ + labeled: true, + withValue: true, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); - const { suggestions: enemySuggestions } = await getSuggestions( - factory.id('enemy_extractor').toString() - ); + const { suggestions: enemySuggestions } = await getSuggestions({ + extractorId: factory.id('enemy_extractor').toString(), + }); expect( enemySuggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared6' && s.language === 'en' ).state - ).toBe(SuggestionState.valueMismatch); + ).toEqual({ + labeled: true, + withValue: true, + withSuggestion: false, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); expect( enemySuggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared9' && s.language === 'en' ).state - ).toBe(SuggestionState.emptyMismatch); + ).toEqual({ + labeled: false, + withValue: false, + withSuggestion: true, + match: false, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }); }); it('should return error status', async () => { - const { suggestions } = await getSuggestions(factory.id('age_extractor').toString()); - expect(suggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared4').state).toBe( - SuggestionState.error - ); + const { suggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); + expect( + suggestions.find((s: EntitySuggestionType) => s.sharedId === 'shared4').state.error + ).toBe(true); }); }); @@ -326,70 +543,128 @@ describe('suggestions', () => { await Suggestions.updateStates({}); }); - it('should accept a suggestion', async () => { - const { suggestions } = await getSuggestions(factory.id('super_powers_extractor').toString()); - const labelMismatchedSuggestion = suggestions.find( - (sug: any) => sug.state === SuggestionState.labelMismatch + it('should accept suggestions', async () => { + const { suggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); + const labelMismatchedSuggestions = suggestions.filter( + (sug: any) => sug.state.labeled && !sug.state.match ); + const ids = new Set(labelMismatchedSuggestions.map((sug: any) => sug._id.toString())); await Suggestions.accept( + labelMismatchedSuggestions.map((sug: any) => ({ + _id: sug._id, + sharedId: sug.sharedId, + entityId: sug.entityId, + })) + ); + const { suggestions: newSuggestions } = await getSuggestions({ + extractorId: factory.id('super_powers_extractor').toString(), + }); + const changedSuggestions = newSuggestions.filter((sug: any) => ids.has(sug._id.toString())); + const matchState = { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + obsolete: false, + processing: false, + error: false, + }; + expect(changedSuggestions).toMatchObject([ { - _id: suggestionId, - sharedId: labelMismatchedSuggestion.sharedId, - entityId: labelMismatchedSuggestion.entityId, + _id: labelMismatchedSuggestions[0]._id, + state: matchState, + suggestedValue: labelMismatchedSuggestions[0].suggestedValue, + labeledValue: labelMismatchedSuggestions[0].suggestedValue, }, - false - ); - const { suggestions: newSuggestions } = await getSuggestions( - factory.id('super_powers_extractor').toString() - ); - const changedSuggestion = newSuggestions.find( - (sg: any) => sg._id.toString() === suggestionId.toString() - ); + { + _id: labelMismatchedSuggestions[1]._id, + state: matchState, + suggestedValue: labelMismatchedSuggestions[1].suggestedValue, + labeledValue: labelMismatchedSuggestions[1].suggestedValue, + }, + ]); + }); - expect(changedSuggestion.state).toBe(SuggestionState.labelMatch); - expect(changedSuggestion.suggestedValue).toEqual(changedSuggestion.labeledValue); + it('should require all suggestions to come from the same extractor', async () => { + const [ageSuggestion] = (await getSuggestions({ extractorId: factory.id('age_extractor') })) + .suggestions; + const [superPowersSuggestion] = ( + await getSuggestions({ + extractorId: factory.id('super_powers_extractor'), + }) + ).suggestions; + await expect( + Suggestions.accept([ + { + _id: ageSuggestion._id, + sharedId: ageSuggestion.sharedId, + entityId: ageSuggestion.entityId, + }, + { + _id: superPowersSuggestion._id, + sharedId: superPowersSuggestion.sharedId, + entityId: superPowersSuggestion.entityId, + }, + ]) + ).rejects.toThrow('All suggestions must come from the same extractor'); }); + it('should not accept a suggestion with an error', async () => { - const { suggestions } = await getSuggestions(factory.id('age_extractor').toString()); + const { suggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); const errorSuggestion = suggestions.find( (s: EntitySuggestionType) => s.sharedId === 'shared4' ); try { - await Suggestions.accept( + await Suggestions.accept([ { _id: errorSuggestion._id, sharedId: errorSuggestion.sharedId, entityId: errorSuggestion.entityId, }, - true - ); + ]); } catch (e: any) { - expect(e?.message).toBe('Suggestion has an error'); + expect(e?.message).toBe('Some Suggestions have an error.'); } }); + it('should update entities of all languages if property name is numeric or date', async () => { - const { suggestions } = await getSuggestions(factory.id('age_extractor').toString()); - const shared2Suggestion = suggestions.find(sug => sug.sharedId === 'shared2'); - await Suggestions.accept( + const { suggestions } = await getSuggestions({ + extractorId: factory.id('age_extractor').toString(), + }); + const suggestionsToAccept = suggestions.filter( + sug => sug.sharedId === 'shared2' || sug.sharedId === 'shared1' + ); + await Suggestions.accept([ { - _id: shared2AgeSuggestionId, - sharedId: shared2Suggestion.sharedId, - entityId: shared2Suggestion.entityId, + _id: suggestionsToAccept[0]._id, + sharedId: suggestionsToAccept[0].sharedId, + entityId: suggestionsToAccept[0].entityId, }, - false - ); + { + _id: suggestionsToAccept[1]._id, + sharedId: suggestionsToAccept[1].sharedId, + entityId: suggestionsToAccept[1].entityId, + }, + ]); - const entities = await db.mongodb + const entities1 = await db.mongodb ?.collection('entities') - .find({ sharedId: shared2Suggestion.sharedId }) + .find({ sharedId: 'shared1' }) .toArray(); + const ages1 = entities1?.map(entity => entity.metadata.age[0].value); + expect(ages1).toEqual(['17', '17']); - const propertyValues = entities?.map(entity => entity.metadata.age); - expect(propertyValues).not.toBe(undefined); - const ages = propertyValues?.map(value => value[0].value) as string[]; - expect(ages[0]).toEqual('20'); - expect(ages[1]).toEqual('20'); - expect(ages[2]).toEqual('20'); + const entities2 = await db.mongodb + ?.collection('entities') + .find({ sharedId: 'shared2' }) + .toArray(); + const ages2 = entities2?.map(entity => entity.metadata.age[0].value); + expect(ages2).toEqual(['20', '20', '20']); }); }); @@ -399,14 +674,18 @@ describe('suggestions', () => { await Suggestions.save(newErroringSuggestion); expect(await findOneSuggestion({ entityId: 'new_erroring_suggestion' })).toMatchObject({ ...newErroringSuggestion, - state: SuggestionState.error, + state: { + error: true, + }, }); const original = await findOneSuggestion({}); const changed: IXSuggestionType = { ...original, status: 'failed' }; await Suggestions.save(changed); expect(await findOneSuggestion({ _id: original._id })).toMatchObject({ ...changed, - state: SuggestionState.error, + state: { + error: true, + }, }); }); }); @@ -416,33 +695,34 @@ describe('suggestions', () => { await Suggestions.save(newProcessingSuggestion); expect(await findOneSuggestion({ entityId: 'new_processing_suggestion' })).toMatchObject({ ...newProcessingSuggestion, - state: 'Processing', + state: { + processing: true, + }, }); const original = await findOneSuggestion({}); const changed: IXSuggestionType = { ...original, status: 'processing' }; await Suggestions.save(changed); expect(await findOneSuggestion({ _id: original._id })).toMatchObject({ ...changed, - state: 'Processing', + state: { + processing: true, + }, }); }); }); }); describe('updateStates()', () => { - it.each(stateUpdateCases)( - 'should mark $state in state if $reason', - async ({ state, suggestionQuery }) => { - const original = await findOneSuggestion(suggestionQuery); - const idQuery = { _id: original._id }; - await Suggestions.updateStates(idQuery); - const changed = await findOneSuggestion(idQuery); - expect(changed).toMatchObject({ - ...original, - state, - }); - } - ); + it.each(stateUpdateCases)('should mark $reason', async ({ state, suggestionQuery }) => { + const original = await findOneSuggestion(suggestionQuery); + const idQuery = { _id: original._id }; + await Suggestions.updateStates(idQuery); + const changed = await findOneSuggestion(idQuery); + expect(changed).toMatchObject({ + ...original, + state, + }); + }); }); describe('setObsolete()', () => { @@ -450,8 +730,8 @@ describe('suggestions', () => { const query = { entityId: 'shared1' }; await Suggestions.setObsolete(query); const obsoletes = await db.mongodb?.collection('ixsuggestions').find(query).toArray(); - expect(obsoletes?.every(s => s.state === SuggestionState.obsolete)).toBe(true); - expect(obsoletes?.length).toBe(3); + expect(obsoletes?.every(s => s.state.obsolete)).toBe(true); + expect(obsoletes?.length).toBe(4); }); }); @@ -460,7 +740,7 @@ describe('suggestions', () => { const query = { entityId: 'shared1' }; await Suggestions.markSuggestionsWithoutSegmentation(query); const notSegmented = await db.mongodb?.collection('ixsuggestions').find(query).toArray(); - expect(notSegmented?.every(s => s.state === SuggestionState.error)).toBe(true); + expect(notSegmented?.every(s => s.state.error)).toBe(true); }); it('should not mark suggestions when segmentations are correct', async () => { @@ -475,9 +755,9 @@ describe('suggestions', () => { .find({ _id: shared2AgeSuggestionId }) .toArray(); expect(segmented?.length).toBe(1); - expect(segmented?.every(s => s.state === SuggestionState.error)).toBe(false); + expect(segmented?.every(s => s.state?.error)).toBe(false); expect(notSegmented?.length).toBe(1); - expect(notSegmented?.every(s => s.state === SuggestionState.error)).toBe(true); + expect(notSegmented?.every(s => s.state.error)).toBe(true); }); }); @@ -507,12 +787,16 @@ describe('suggestions', () => { expect(await findOneSuggestion({ entityId: newErroringSuggestion.entityId })).toMatchObject({ ...newErroringSuggestion, - state: SuggestionState.error, + state: { + error: true, + }, }); expect(await findOneSuggestion({ entityId: newProcessingSuggestion.entityId })).toMatchObject( { ...newProcessingSuggestion, - state: SuggestionState.processing, + state: { + processing: true, + }, } ); }); diff --git a/app/api/suggestions/stats.ts b/app/api/suggestions/stats.ts deleted file mode 100644 index 1ff5cca661..0000000000 --- a/app/api/suggestions/stats.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { ObjectId } from 'mongodb'; -import { ObjectIdSchema } from 'shared/types/commonTypes'; -import { SuggestionState } from 'shared/types/suggestionSchema'; -import { SuggestionsStats } from 'shared/types/suggestionStats'; -import { IXSuggestionsModel } from './IXSuggestionsModel'; - -interface StateGroup { - _id: T; - count: number; -} - -interface Groups { - buckets: StateGroup[]; - all: StateGroup<'all'>[]; -} - -const addCount = (sum: number, group: StateGroup) => sum + group.count; - -const addCountsOf = (groups: Groups, _states: SuggestionState[]) => { - const states = new Set(_states); - return groups.buckets.filter(g => states.has(g._id)).reduce(addCount, 0); -}; - -const getGroups = async (extractorId: ObjectIdSchema): Promise => - IXSuggestionsModel.db - .aggregate([ - { $match: { extractorId } }, - { - $facet: { - buckets: [ - { - $group: { - _id: '$state', - count: { - $sum: 1, - }, - }, - }, - ], - all: [ - { - $count: 'count', - }, - ], - }, - }, - ]) - .then(([result]) => result); - -const calcAccuracy = (groups: Groups) => { - const correct = addCountsOf(groups, [SuggestionState.labelMatch, SuggestionState.valueMatch]); - const incorect = addCountsOf(groups, [ - SuggestionState.labelMismatch, - SuggestionState.valueMismatch, - SuggestionState.labelEmpty, - SuggestionState.valueEmpty, - ]); - const total = correct + incorect; - return total ? correct / total : 0; -}; - -const getStats = async (_extractorId: string): Promise => { - const extractorId = new ObjectId(_extractorId); - const groups = await getGroups(extractorId); - - const labeled = addCountsOf(groups, [ - SuggestionState.labelMatch, - SuggestionState.labelMismatch, - SuggestionState.labelEmpty, - ]); - const nonLabeledMatching = addCountsOf(groups, [SuggestionState.valueMatch]); - const nonLabeledNotMatching = addCountsOf(groups, [SuggestionState.valueMismatch]); - const emptyOrObsolete = addCountsOf(groups, [ - SuggestionState.empty, - SuggestionState.obsolete, - SuggestionState.valueEmpty, - SuggestionState.emptyMismatch, - ]); - const all = groups.all[0]?.count || 0; - - const accuracy = calcAccuracy(groups); - - return { - counts: { - labeled, - nonLabeledMatching, - nonLabeledNotMatching, - emptyOrObsolete, - all, - }, - accuracy, - }; -}; - -export { getStats }; diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 05dd28870f..85474c0fff 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -1,30 +1,38 @@ -import { FilterQuery } from 'mongoose'; - +import { ObjectId } from 'mongodb'; import entities from 'api/entities/entities'; import { files } from 'api/files/files'; +import { EnforcedWithId } from 'api/odm'; import settings from 'api/settings/settings'; import { IXSuggestionsModel } from 'api/suggestions/IXSuggestionsModel'; import templates from 'api/templates'; +import { syncedPromiseLoop } from 'shared/data_utils/promiseUtils'; import { ExtractedMetadataSchema, LanguagesListSchema, ObjectIdSchema, } from 'shared/types/commonTypes'; import { EntitySchema } from 'shared/types/entityType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; -import { IXSuggestionsFilter, IXSuggestionType } from 'shared/types/suggestionType'; -import { ObjectId } from 'mongodb'; +import { FileType } from 'shared/types/fileType'; +import { + IXSuggestionAggregation, + IXSuggestionsFilter, + IXSuggestionsQuery, + IXSuggestionType, + SuggestionCustomFilter, +} from 'shared/types/suggestionType'; +import { objectIndex } from 'shared/data_utils/objectIndex'; import { getSegmentedFilesIds } from 'api/services/informationextraction/getFiles'; import { registerEventListeners } from './eventListeners'; import { + baseQueryFragment, + filterFragments, getCurrentValueStage, getEntityStage, getFileStage, getLabeledValueStage, getMatchStage, - groupByAndSort, + groupByAndCount, } from './pipelineStages'; -import { getStats } from './stats'; import { updateStates } from './updateState'; interface AcceptedSuggestion { @@ -35,74 +43,114 @@ interface AcceptedSuggestion { const updateEntitiesWithSuggestion = async ( allLanguages: boolean, - acceptedSuggestion: AcceptedSuggestion, - suggestion: IXSuggestionType + acceptedSuggestions: AcceptedSuggestion[], + suggestions: IXSuggestionType[] ) => { + const sharedIds = acceptedSuggestions.map(s => s.sharedId); + const entityIds = acceptedSuggestions.map(s => s.entityId); + const { propertyName } = suggestions[0]; const query = allLanguages - ? { sharedId: acceptedSuggestion.sharedId } - : { sharedId: acceptedSuggestion.sharedId, _id: acceptedSuggestion.entityId }; + ? { sharedId: { $in: sharedIds } } + : { sharedId: { $in: sharedIds }, _id: { $in: entityIds } }; const storedEntities = await entities.get(query, '+permissions'); + + const acceptedSuggestionsBySharedId = objectIndex( + acceptedSuggestions, + as => as.sharedId, + as => as + ); + const suggestionsById = objectIndex( + suggestions, + s => s._id?.toString() || '', + s => s + ); + + const getValue = (entity: EntitySchema) => + suggestionsById[acceptedSuggestionsBySharedId[entity.sharedId?.toString() || '']._id.toString()] + .suggestedValue; + const entitiesToUpdate = - suggestion.propertyName !== 'title' + propertyName !== 'title' ? storedEntities.map((entity: EntitySchema) => ({ ...entity, metadata: { ...entity.metadata, - [suggestion.propertyName]: [{ value: suggestion.suggestedValue }], + [propertyName]: [ + { + value: getValue(entity), + }, + ], }, permissions: entity.permissions || [], })) : storedEntities.map((entity: EntitySchema) => ({ ...entity, - title: suggestion.suggestedValue, + title: getValue(entity), })); await entities.saveMultiple(entitiesToUpdate); }; -const updateExtractedMetadata = async (suggestion: IXSuggestionType) => { - const fetchedFiles = await files.get({ _id: suggestion.fileId }); +const updateExtractedMetadata = async (suggestions: IXSuggestionType[]) => { + const fetchedFiles = await files.get({ _id: { $in: suggestions.map(s => s.fileId) } }); + const suggestionsByFileId = objectIndex( + suggestions, + s => s.fileId?.toString() || '', + s => s + ); - if (!fetchedFiles?.length) return Promise.resolve(); - const file = fetchedFiles[0]; + await syncedPromiseLoop(fetchedFiles, async (file: EnforcedWithId) => { + const suggestion = suggestionsByFileId[file._id.toString()]; + file.extractedMetadata = file.extractedMetadata ? file.extractedMetadata : []; - file.extractedMetadata = file.extractedMetadata ? file.extractedMetadata : []; - const extractedMetadata = file.extractedMetadata.find( - (em: any) => em.name === suggestion.propertyName - ) as ExtractedMetadataSchema; + const extractedMetadata = file.extractedMetadata.find( + (em: any) => em.name === suggestion.propertyName + ) as ExtractedMetadataSchema; - if (!extractedMetadata) { - file.extractedMetadata.push({ - name: suggestion.propertyName, - timestamp: Date(), - selection: { + if (!extractedMetadata) { + file.extractedMetadata.push({ + name: suggestion.propertyName, + timestamp: Date(), + selection: { + text: suggestion.suggestedText || suggestion.suggestedValue?.toString(), + selectionRectangles: suggestion.selectionRectangles, + }, + }); + } else { + extractedMetadata.timestamp = Date(); + extractedMetadata.selection = { text: suggestion.suggestedText || suggestion.suggestedValue?.toString(), selectionRectangles: suggestion.selectionRectangles, - }, - }); - } else { - extractedMetadata.timestamp = Date(); - extractedMetadata.selection = { - text: suggestion.suggestedText || suggestion.suggestedValue?.toString(), - selectionRectangles: suggestion.selectionRectangles, - }; - } - return files.save(file); + }; + } + + return files.save(file); + }); }; const buildListQuery = ( - filters: FilterQuery, + extractorId: ObjectId, + customFilter: SuggestionCustomFilter | undefined, setLanguages: LanguagesListSchema | undefined, offset: number, - limit: number + limit: number, + sort?: IXSuggestionsQuery['sort'] ) => { + const sortOrder = sort?.order === 'desc' ? -1 : 1; + const sorting = sort?.property ? { [sort.property]: sortOrder } : { date: 1, state: -1 }; + const pipeline = [ - ...getMatchStage(filters), - { $sort: { date: 1, state: -1 } }, - { $skip: offset }, - { $limit: limit }, + ...getMatchStage(extractorId, customFilter), ...getEntityStage(setLanguages!), ...getCurrentValueStage(), + { + $addFields: { + entityTitle: '$entity.title', + }, + }, + { $sort: sorting }, + { $skip: offset }, + { $limit: limit }, ...getFileStage(), ...getLabeledValueStage(), { @@ -110,7 +158,7 @@ const buildListQuery = ( entityId: '$entity._id', entityTemplateId: '$entity.template', sharedId: '$entity.sharedId', - entityTitle: '$entity.title', + entityTitle: 1, fileId: 1, language: 1, propertyName: 1, @@ -130,59 +178,56 @@ const buildListQuery = ( return pipeline; }; -const buildTemplateAggregationsQuery = (_filters: FilterQuery) => { - const { entityTemplate, ...filters } = _filters; - const pipeline = [...getMatchStage(filters), ...groupByAndSort('$entityTemplate')]; - return pipeline; -}; +async function getLabeledCounts(extractorId: ObjectId) { + const labeledAggregationQuery = [ + { + $match: { + ...baseQueryFragment(extractorId), + ...filterFragments.labeled._fragment, + }, + }, + ...groupByAndCount('$state.match'), + ]; + const labeledAggregation: { _id: boolean; count: number }[] = + await IXSuggestionsModel.db.aggregate(labeledAggregationQuery); + const matchCount = + labeledAggregation.find((aggregation: any) => aggregation._id === true)?.count || 0; + const mismatchCount = + labeledAggregation.find((aggregation: any) => aggregation._id === false)?.count || 0; + const labeledCount = matchCount + mismatchCount; + return { labeledCount, matchCount, mismatchCount }; +} -const buildStateAggregationsQuery = (_filters: FilterQuery) => { - const { state, ...filters } = _filters; - const pipeline = [...getMatchStage(filters), ...groupByAndSort('$state')]; - return pipeline; +const getNonLabeledCounts = async (_extractorId: ObjectId) => { + const extractorId = new ObjectId(_extractorId); + const unlabeledMatch = { + ...baseQueryFragment(extractorId), + ...filterFragments.nonLabeled._fragment, + }; + const nonLabeledCount = await IXSuggestionsModel.count(unlabeledMatch); + const noContextCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.noContext, + }); + const noSuggestionCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.noSuggestion, + }); + const obsoleteCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.obsolete, + }); + const othersCount = await IXSuggestionsModel.count({ + ...unlabeledMatch, + ...filterFragments.nonLabeled.others, + }); + return { nonLabeledCount, noContextCount, noSuggestionCount, obsoleteCount, othersCount }; }; -const fetchAndAggregateSuggestions = async ( - _filters: Omit, - setLanguages: LanguagesListSchema | undefined, - offset: number, - limit: number -) => { - const { - states, - entityTemplates, - ...filters - }: { - states?: string[]; - entityTemplates?: string[]; - extractorId?: ObjectIdSchema; - state?: { $in: string[] }; - entityTemplate?: { $in: string[] }; - } = _filters; - if (states) filters.state = { $in: _filters.states || [] }; - if (entityTemplates) filters.entityTemplate = { $in: _filters.entityTemplates || [] }; - - const count = await IXSuggestionsModel.db - .aggregate([{ $match: { ...filters, status: { $ne: 'processing' } } }, { $count: 'count' }]) - .then(result => (result?.length ? result[0].count : 0)); - - const suggestions = await IXSuggestionsModel.db.aggregate( - buildListQuery(filters, setLanguages, offset, limit) - ); - - const templateAggregations = await IXSuggestionsModel.db.aggregate( - buildTemplateAggregationsQuery(filters) - ); - - const stateAggregations = await IXSuggestionsModel.db.aggregate( - buildStateAggregationsQuery(filters) - ); - - return { - suggestions, - aggregations: { template: templateAggregations, state: stateAggregations }, - totalPages: Math.ceil(count / limit), - }; +const readFilter = (filter: IXSuggestionsFilter) => { + const { customFilter, extractorId: _extractorId } = filter; + const extractorId = new ObjectId(_extractorId); + return { customFilter, extractorId }; }; const Suggestions = { @@ -190,23 +235,60 @@ const Suggestions = { getByEntityId: async (sharedId: string) => IXSuggestionsModel.get({ entityId: sharedId }), getByExtractor: async (extractorId: ObjectIdSchema) => IXSuggestionsModel.get({ extractorId }), - get: async (filter: IXSuggestionsFilter, options: { page: { size: number; number: number } }) => { + get: async ( + filter: IXSuggestionsFilter, + options: { + page?: IXSuggestionsQuery['page']; + sort?: IXSuggestionsQuery['sort']; + } + ) => { const offset = options && options.page ? options.page.size * (options.page.number - 1) : 0; const DEFAULT_LIMIT = 30; const limit = options.page?.size || DEFAULT_LIMIT; const { languages: setLanguages } = await settings.get(); - const { language, ...filters } = filter; - filters.extractorId = new ObjectId(filter.extractorId); + const { customFilter, extractorId } = readFilter(filter); + + const count = await IXSuggestionsModel.db + .aggregate(getMatchStage(extractorId, customFilter, true)) + .then(result => (result?.length ? result[0].count : 0)); + + const suggestions = await IXSuggestionsModel.db.aggregate( + buildListQuery(extractorId, customFilter, setLanguages, offset, limit, options.sort) + ); - return fetchAndAggregateSuggestions(filters, setLanguages, offset, limit); + return { + suggestions, + totalPages: Math.ceil(count / limit), + }; }, - getStats, + aggregate: async (_extractorId: ObjectIdSchema): Promise => { + const extractorId = new ObjectId(_extractorId); + const { labeledCount, matchCount, mismatchCount } = await getLabeledCounts(extractorId); + const { nonLabeledCount, noContextCount, noSuggestionCount, obsoleteCount, othersCount } = + await getNonLabeledCounts(extractorId); + const totalCount = labeledCount + nonLabeledCount; + return { + total: totalCount, + labeled: { + _count: labeledCount, + match: matchCount, + mismatch: mismatchCount, + }, + nonLabeled: { + _count: nonLabeledCount, + noContext: noContextCount, + noSuggestion: noSuggestionCount, + obsolete: obsoleteCount, + others: othersCount, + }, + }; + }, updateStates, setObsolete: async (query: any) => - IXSuggestionsModel.updateMany(query, { $set: { state: SuggestionState.obsolete } }), + IXSuggestionsModel.updateMany(query, { $set: { 'state.obsolete': true } }), markSuggestionsWithoutSegmentation: async (query: any) => { const segmentedFilesIds = await getSegmentedFilesIds(); @@ -215,45 +297,39 @@ const Suggestions = { ...query, fileId: { $nin: segmentedFilesIds }, }, - { $set: { state: SuggestionState.error } } + { $set: { 'state.error': true } } ); }, save: async (suggestion: IXSuggestionType) => Suggestions.saveMultiple([suggestion]), saveMultiple: async (_suggestions: IXSuggestionType[]) => { - const toSave: IXSuggestionType[] = []; - const toSaveAndUpdate: IXSuggestionType[] = []; - _suggestions.forEach(s => { - if (s.status === 'failed') { - toSave.push({ ...s, state: SuggestionState.error }); - } else if (s.status === 'processing') { - toSave.push({ ...s, state: SuggestionState.processing }); - } else { - toSaveAndUpdate.push(s); - } - }); - await IXSuggestionsModel.saveMultiple(toSave); - const toUpdate = await IXSuggestionsModel.saveMultiple(toSaveAndUpdate); - if (toUpdate.length) await updateStates({ _id: { $in: toUpdate.map(s => s._id) } }); + const toUpdate = await IXSuggestionsModel.saveMultiple(_suggestions); + if (toUpdate.length > 0) await updateStates({ _id: { $in: toUpdate.map(s => s._id) } }); }, - accept: async (acceptedSuggestion: AcceptedSuggestion, allLanguages: boolean) => { - const suggestion = await IXSuggestionsModel.getById(acceptedSuggestion._id); - if (!suggestion) { - throw new Error('Suggestion not found'); + accept: async (acceptedSuggestions: AcceptedSuggestion[]) => { + const acceptedIds = Array.from(new Set(acceptedSuggestions.map(s => s._id.toString()))); + const suggestions = await IXSuggestionsModel.get({ _id: { $in: acceptedIds } }); + const extractors = new Set(suggestions.map(s => s.extractorId.toString())); + if (extractors.size > 1) { + throw new Error('All suggestions must come from the same extractor'); } - if (suggestion.error !== '') { - throw new Error('Suggestion has an error'); + const foundIds = new Set(suggestions.map(s => s._id.toString())); + if (!acceptedIds.every(id => foundIds.has(id))) { + throw new Error('Suggestion(s) not found.'); } - let shouldUpdateAllLanguages = allLanguages; - const property = await templates.getPropertyByName(suggestion.propertyName); - if (property && ['numeric', 'date'].includes(property.type)) { - shouldUpdateAllLanguages = true; + if (suggestions.some(s => s.error !== '')) { + throw new Error('Some Suggestions have an error.'); } - await updateEntitiesWithSuggestion(shouldUpdateAllLanguages, acceptedSuggestion, suggestion); - await updateExtractedMetadata(suggestion); - await Suggestions.updateStates({ _id: acceptedSuggestion._id }); + + const { propertyName } = suggestions[0]; + const property = await templates.getPropertyByName(propertyName); + const allLanguage = property.type === 'numeric' || property.type === 'date'; + + await updateEntitiesWithSuggestion(allLanguage, acceptedSuggestions, suggestions); + await updateExtractedMetadata(suggestions); + await Suggestions.updateStates({ _id: { $in: acceptedIds.map(id => new ObjectId(id)) } }); }, deleteByEntityId: async (sharedId: string) => { diff --git a/app/api/suggestions/updateState.ts b/app/api/suggestions/updateState.ts index d7fd6baf30..1c1e4edc32 100644 --- a/app/api/suggestions/updateState.ts +++ b/app/api/suggestions/updateState.ts @@ -48,7 +48,7 @@ const getModelCreationDateStage = () => [ const findSuggestions = (query: any, languages: LanguagesListSchema) => IXSuggestionsModel.db .aggregateCursor([ - { $match: { ...query, status: { $ne: 'processing' } } }, + { $match: { ...query } }, ...getEntityStage(languages), ...getCurrentValueStage(), { @@ -72,6 +72,9 @@ const findSuggestions = (query: any, languages: LanguagesListSchema) => date: 1, propertyName: 1, extractorId: 1, + status: 1, + state: 1, + segment: 1, }, }, ]) diff --git a/app/api/templates/specs/templates.spec.js b/app/api/templates/specs/templates.spec.js index 79d55b8efe..bce97c9c96 100644 --- a/app/api/templates/specs/templates.spec.js +++ b/app/api/templates/specs/templates.spec.js @@ -1,6 +1,3 @@ -/* eslint-disable max-lines */ -/* eslint-disable max-statements */ - import Ajv from 'ajv'; import db from 'api/utils/testing_db'; import documents from 'api/documents/documents.js'; @@ -587,7 +584,7 @@ describe('templates', () => { describe('getPropertyByName()', () => { it('should get properties with the name provided', async () => { const newTemplate = { - name: 'created template 2', + name: 'created template 2', commonProperties: [{ name: 'title', label: 'Title', type: 'text' }], properties: [ { label: 'label', type: 'text' }, @@ -600,11 +597,46 @@ describe('templates', () => { expect(property.type).toEqual('date'); }); - it('should throw an error when no template is found', async () => { + it('should throw an error when the property is not found', async () => { try { await templates.getPropertyByName('nonexistent property name'); } catch (e) { - expect(e.message).toEqual('No template with the given property name'); + expect(e.message).toEqual('Properties not found: nonexistent property name'); + } + }); + }); + + describe('getPropertiesByName()', () => { + it('should get properties with the name provided', async () => { + const newTemplate = { + name: 'created template 3', + commonProperties: [{ name: 'title', label: 'Title', type: 'text' }], + properties: [ + { label: 'label', type: 'text' }, + { label: 'Date', type: 'date' }, + ], + }; + const newTemplate2 = { + name: 'created template 4', + commonProperties: [{ name: 'title', label: 'Title', type: 'text' }], + properties: [{ label: 'number', type: 'numeric' }], + }; + await templates.save(newTemplate); + await templates.save(newTemplate2); + const properties = await templates.getPropertiesByName(['date', 'label', 'number', 'title']); + expect(properties).toMatchObject([ + { name: 'title', type: 'text' }, + { name: 'label', type: 'text' }, + { name: 'date', type: 'date' }, + { name: 'number', type: 'numeric' }, + ]); + }); + + it('should throw an error when a property is not found', async () => { + try { + await templates.getPropertiesByName(['nonexistent property name']); + } catch (e) { + expect(e.message).toEqual('Properties not found: nonexistent property name'); } }); }); diff --git a/app/api/templates/templates.ts b/app/api/templates/templates.ts index 4a48e056f2..4b0de3b0a5 100644 --- a/app/api/templates/templates.ts +++ b/app/api/templates/templates.ts @@ -1,3 +1,5 @@ +import { ObjectId } from 'mongodb'; + import entities from 'api/entities'; import { populateGeneratedIdByTemplate } from 'api/entities/generatedIdPropertyAutoFiller'; import { applicationEventsBus } from 'api/eventsbus'; @@ -7,7 +9,7 @@ import { updateMapping } from 'api/search/entitiesIndex'; import settings from 'api/settings/settings'; import dictionariesModel from 'api/thesauri/dictionariesModel'; import createError from 'api/utils/Error'; -import { ObjectId } from 'mongodb'; +import { objectIndex } from 'shared/data_utils/objectIndex'; import { propertyTypes } from 'shared/propertyTypes'; import { ContextType } from 'shared/translationSchema'; import { ensure } from 'shared/tsUtils'; @@ -247,14 +249,34 @@ export default { return model.get(query); }, - async getPropertyByName(propertyName: string): Promise { + async getPropertyByName(propertyName: string): Promise { + const [property] = await this.getPropertiesByName([propertyName]); + return property; + }, + + async getPropertiesByName(propertyNames: string[]): Promise { + const nameSet = new Set(propertyNames); const templates = await this.get({ - $or: [{ 'properties.name': propertyName }, { 'commonProperties.name': propertyName }], + $or: [ + { 'properties.name': { $in: propertyNames } }, + { 'commonProperties.name': { $in: propertyNames } }, + ], }); - if (!templates.length) { - throw createError('No template with the given property name'); + const allProperties = templates + .map(template => [template.properties || [], template.commonProperties || []]) + .flat() + .flat() + .filter(t => nameSet.has(t.name)); + const propertiesByName = objectIndex( + allProperties, + p => p.name, + p => p + ); + const missingProperties = propertyNames.filter(name => !propertiesByName[name]); + if (missingProperties.length > 0) { + throw createError(`Properties not found: ${missingProperties.join(', ')}`); } - return templates[0].properties?.find(property => property.name === propertyName); + return Array.from(Object.values(propertiesByName)); }, async setAsDefault(_id: string) { diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 1c478a162d..6d08a15a5d 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -17,10 +17,10 @@ import { import { UpdateLog } from 'api/updatelogs'; import { IXExtractorType } from 'shared/types/extractorType'; import { IXSuggestionType } from 'shared/types/suggestionType'; -import { SuggestionState } from 'shared/types/suggestionSchema'; import { WithId } from 'api/odm/model'; import { TemplateSchema } from 'shared/types/templateType'; import { getV2FixturesFactoryElements } from 'api/common.v2/testing/fixturesFactory'; +import { IXModelType } from 'shared/types/IXModelType'; import { PermissionSchema } from 'shared/types/permissionType'; function getIdMapper() { @@ -141,6 +141,18 @@ function getFixturesFactory() { }); }, + fileExtractedMetadata: ( + propertyName: string, + text: string, + rectangles = [{ top: 0, left: 0, width: 0, height: 0, page: '1' }] + ): ExtractedMetadataSchema => ({ + name: propertyName, + selection: { + text, + selectionRectangles: rectangles, + }, + }), + file: ( id: string, entity: string | undefined, @@ -248,6 +260,18 @@ function getFixturesFactory() { templates: templates.map(idMapper), }), + ixModel: ( + name: string, + extractor: string, + creationDate = 1, + status: IXModelType['status'] = 'ready' + ): IXModelType => ({ + _id: idMapper(name), + status, + creationDate, + extractorId: idMapper(extractor), + }), + ixSuggestion: ( suggestionId: string, extractor: string, @@ -269,7 +293,16 @@ function getFixturesFactory() { segment: '', suggestedValue: '', date: 1, - state: SuggestionState.valueEmpty, + state: { + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, + }, ...otherProps, }), diff --git a/app/react/App/App.js b/app/react/App/App.js index e40a7b469c..bdc795f74c 100644 --- a/app/react/App/App.js +++ b/app/react/App/App.js @@ -67,7 +67,7 @@ const App = ({ customParams }) => {
-