Merge branch 'development' into update-tables

huridocs · Jul 18, 2024 · cd4d7c2 · cd4d7c2
2 parents b363cfe + 894f418
commit cd4d7c2
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 54 deletions.
diff --git a/SELF_HOSTED_INSTRUCTIONS.md b/SELF_HOSTED_INSTRUCTIONS.md
@@ -64,7 +64,7 @@ mv prod latest && rm uwazi.tgz
 
 You should now be able to run an Uwazi instance in production mode:
 
-`DATABASE_NAME=my_db_name INDEX_NAME=my_db_name NODE_ENV=production FILES_ROOT_PATH=/xxxx/yyyy/uwazi/ node server.js`
+`DATABASE_NAME=my_db_name INDEX_NAME=my_db_name NODE_ENV=production FILES_ROOT_PATH=/xxxx/yyyy/uwazi/ node server.js --no-experimental-fetch`
 
 By default, Uwazi runs on `localhost` port 3000, so point your browser to http://localhost:3000 and authenticate yourself with the default username "admin" and password "change this password now".
 

diff --git a/app/api/entities/specs/validateEntity.spec.ts b/app/api/entities/specs/validateEntity.spec.ts
@@ -204,11 +204,6 @@ describe('validateEntity', () => {
           entity = createEntity({ metadata: { name: [{ value: 10 }] } });
           await expectError(entity, customErrorMessages[propertyTypes.text], ".metadata['name']");
         });
-
-        it('should fail if value is a string that exceeds the lucene term byte-length limit', async () => {
-          const entity = createEntity({ metadata: { name: [{ value: largeField }] } });
-          await expectError(entity, 'maximum field length exceeded', ".metadata['name']");
-        });
       });
 
       describe('markdown property', () => {

diff --git a/app/api/entities/validation/validateMetadataField.ts b/app/api/entities/validation/validateMetadataField.ts
@@ -1,17 +1,16 @@
+import { ObjectId } from 'mongodb';
 import Ajv, { ErrorObject } from 'ajv';
 import { isUndefined, isNull } from 'util';
 import { ensure } from 'shared/tsUtils';
 import { propertyTypes } from 'shared/propertyTypes';
-import entities from 'api/entities';
-import thesauris from 'api/thesauri';
 import { PropertySchema, MetadataObjectSchema } from 'shared/types/commonTypes';
 import { EntitySchema, EntityWithFilesSchema } from 'shared/types/entityType';
-
 import { TemplateSchema } from 'shared/types/templateType';
+import { arrayBidirectionalDiff } from 'shared/data_utils/arrayBidirectionalDiff';
+import entities from 'api/entities';
+import thesauris from 'api/thesauri';
 import { flatThesaurusValues } from 'api/thesauri/thesauri';
 import { validators, customErrorMessages } from './metadataValidators';
-import { ObjectId } from 'mongodb';
-import { arrayBidirectionalDiff } from 'shared/data_utils/arrayBidirectionalDiff';
 
 const hasValue = (value: any) => !isUndefined(value) && !isNull(value);
 
@@ -219,21 +218,6 @@ const validateSameRelationshipsMatch = (
       ];
 };
 
-const validateFieldSize = (
-  property: PropertySchema,
-  entity: EntitySchema,
-  value: MetadataObjectSchema[] = []
-) => {
-  if (
-    property.type !== propertyTypes.markdown &&
-    hasValue(value) &&
-    !validators.validateLuceneBytesLimit(value)
-  ) {
-    return [validationError({ message: customErrorMessages.length_exceeded }, property, entity)];
-  }
-  return [];
-};
-
 export const validateMetadataField = async (
   property: PropertySchema,
   entity: EntitySchema,
@@ -244,7 +228,6 @@ export const validateMetadataField = async (
   const errors: ErrorObject[] = [
     ...validateRequired(property, entity, value),
     ...validateType(property, entity, value),
-    ...validateFieldSize(property, entity, value),
     ...validateSameRelationshipsMatch(property, entity, template, value),
     ...(await validateRelationshipForeignIds(property, entity, value)),
     ...(await validateDictionariesForeignIds(property, entity, value)),

diff --git a/app/api/services/informationextraction/ixextractors.ts b/app/api/services/informationextraction/ixextractors.ts
@@ -151,6 +151,7 @@ export const Extractors = {
     await model.updateMany({ _id: { $in: extractorIds } }, { $pull: { templates: templateId } });
 
     await Suggestions.delete({ entityTemplate: templateId, extractorId: { $in: extractorIds } });
+    await model.delete({ _id: { $in: extractorIds }, templates: { $size: 0 } });
   },
 };
 

diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts
@@ -873,7 +873,10 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
       })
     );
 
-    const extractors = await testingDB.mongodb?.collection('ixextractors').find({}).toArray();
+    const extractors = await testingDB.mongodb
+      ?.collection('ixextractors')
+      .find({ templates: { $not: { $size: 0 } } })
+      .toArray();
 
     expect(extractors).toEqual([
       fixturesFactory.ixExtractor('title_extractor', 'title', [
@@ -884,7 +887,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
         extractedTemplateName,
         otherExtractedTemplateName,
       ]),
-      fixturesFactory.ixExtractor('extractor2', 'extracted_property_2', []),
       fixturesFactory.ixExtractor('extractor3', 'some_property', ['some_other_template']),
       fixturesFactory.ixExtractor('extractor4', 'extracted_property_2_1', [
         otherExtractedTemplateName,
@@ -894,7 +896,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
       ]),
       fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]),
       fixturesFactory.ixExtractor('extractor7', 'multiselect_property', [extractedTemplateName]),
-      fixturesFactory.ixExtractor('extractor8', 'relationship_property', []),
     ]);
 
     const suggestions = await testingDB.mongodb?.collection('ixsuggestions').find({}).toArray();
@@ -959,7 +960,10 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
       })
     );
 
-    const extractors = await testingDB.mongodb?.collection('ixextractors').find({}).toArray();
+    const extractors = await testingDB.mongodb
+      ?.collection('ixextractors')
+      .find({ templates: { $not: { $size: 0 } } })
+      .toArray();
 
     expect(extractors).toEqual([
       fixturesFactory.ixExtractor('title_extractor', 'title', [
@@ -970,7 +974,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
         extractedTemplateName,
         otherExtractedTemplateName,
       ]),
-      fixturesFactory.ixExtractor('extractor2', 'extracted_property_2', []),
       fixturesFactory.ixExtractor('extractor3', 'some_property', ['some_other_template']),
       fixturesFactory.ixExtractor('extractor4', 'extracted_property_2_1', [
         otherExtractedTemplateName,
@@ -979,7 +982,6 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
         otherExtractedTemplateName,
       ]),
       fixturesFactory.ixExtractor('extractor6', 'select_property', [extractedTemplateName]),
-      fixturesFactory.ixExtractor('extractor7', 'multiselect_property', []),
       fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]),
     ]);
 
@@ -1012,6 +1014,45 @@ describe(`On ${TemplateUpdatedEvent.name}`, () => {
       },
     ]);
   });
+
+  it('should delete the extractor itself if it does not contain any templates', async () => {
+    await applicationEventsBus.emit(
+      new TemplateUpdatedEvent({
+        before: {
+          _id: fixturesFactory.id(extractedTemplateName),
+          name: extractedTemplateName,
+          properties: [
+            fixturesFactory.property('not_extracted_property_1', propertyTypes.text),
+            fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric),
+            fixturesFactory.property('extracted_property_1', propertyTypes.text),
+            fixturesFactory.property('extracted_property_2', propertyTypes.numeric),
+            fixturesFactory.property('select_property', propertyTypes.select),
+            fixturesFactory.property('multiselect_property', propertyTypes.multiselect),
+            fixturesFactory.property('relationship_property', propertyTypes.relationship),
+          ],
+        },
+        after: {
+          _id: fixturesFactory.id(extractedTemplateName),
+          name: extractedTemplateName,
+          properties: [
+            fixturesFactory.property('not_extracted_property_1', propertyTypes.text),
+            fixturesFactory.property('not_extracted_property_2', propertyTypes.numeric),
+            fixturesFactory.property('extracted_property_1', propertyTypes.text),
+            fixturesFactory.property('extracted_property_2_renamed', propertyTypes.numeric),
+            fixturesFactory.property('select_property', propertyTypes.select),
+            fixturesFactory.property('multiselect_property_renamed', propertyTypes.multiselect),
+            fixturesFactory.property('relationship_property', propertyTypes.relationship),
+          ],
+        },
+      })
+    );
+
+    const extractorsWithoutTemplates = await testingDB.mongodb
+      ?.collection('ixextractors')
+      .countDocuments({ templates: { $size: 0 } });
+
+    expect(extractorsWithoutTemplates).toEqual(0);
+  });
 });
 
 describe(`On ${TemplateDeletedEvent.name}`, () => {
@@ -1022,27 +1063,40 @@ describe(`On ${TemplateDeletedEvent.name}`, () => {
       })
     );
 
-    const extractors = await testingDB.mongodb?.collection('ixextractors').find({}).toArray();
+    const extractors = await testingDB.mongodb
+      ?.collection('ixextractors')
+      .find({ templates: { $not: { $size: 0 } } })
+      .toArray();
 
     expect(extractors).toEqual([
       fixturesFactory.ixExtractor('title_extractor', 'title', [otherExtractedTemplateName]),
       fixturesFactory.ixExtractor('extractor1', 'extracted_property_1', [
         otherExtractedTemplateName,
       ]),
-      fixturesFactory.ixExtractor('extractor2', 'extracted_property_2', []),
       fixturesFactory.ixExtractor('extractor3', 'some_property', ['some_other_template']),
       fixturesFactory.ixExtractor('extractor4', 'extracted_property_2_1', [
         otherExtractedTemplateName,
       ]),
       fixturesFactory.ixExtractor('extractor5', 'extracted_property_2_2', [
         otherExtractedTemplateName,
       ]),
-      fixturesFactory.ixExtractor('extractor6', 'select_property', []),
-      fixturesFactory.ixExtractor('extractor7', 'multiselect_property', []),
-      fixturesFactory.ixExtractor('extractor8', 'relationship_property', []),
     ]);
   });
 
+  it('should delete the extractor itself if it does not contain any templates', async () => {
+    await applicationEventsBus.emit(
+      new TemplateDeletedEvent({
+        templateId: fixturesFactory.id(extractedTemplateName).toString(),
+      })
+    );
+
+    const extractorsWithoutTemplates = await testingDB.mongodb
+      ?.collection('ixextractors')
+      .countDocuments({ templates: { $size: 0 } });
+
+    expect(extractorsWithoutTemplates).toEqual(0);
+  });
+
   it('should delete the suggestions related to the template', async () => {
     await applicationEventsBus.emit(
       new TemplateDeletedEvent({

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "uwazi",
-  "version": "1.176.0-rc2",
+  "version": "1.176.0-rc4",
   "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.",
   "keywords": [
     "react"
@@ -151,7 +151,7 @@
     "filesize": "^10.1.0",
     "flag-icon-css": "^4.1.7",
     "flowbite": "^2.3.0",
-    "flowbite-datepicker": "^1.2.6",
+    "flowbite-datepicker": "^1.2.7",
     "flowbite-react": "^0.10.1",
     "formatcoords": "^1.1.3",
     "franc": "5.0.0",
@@ -193,7 +193,7 @@
     "prom-client": "^15.1.3",
     "prop-types": "^15.8.1",
     "qrcode.react": "^3.1.0",
-    "qs": "^6.12.1",
+    "qs": "^6.12.3",
     "react": "^18.3.1",
     "react-color": "^2.19.3",
     "react-datepicker": "6.9.0",
@@ -309,7 +309,7 @@
     "@types/node-uuid": "^0.0.28",
     "@types/nodemailer": "^6.4.15",
     "@types/prop-types": "^15.7.3",
-    "@types/qs": "^6.9.14",
+    "@types/qs": "^6.9.15",
     "@types/react": "^18.3.1",
     "@types/react-dnd": "^3.0.2",
     "@types/react-dom": "^18.0.9",

diff --git a/yarn.lock b/yarn.lock
@@ -5418,10 +5418,10 @@
   dependencies:
     "@types/node" "*"
 
-"@types/qs@*", "@types/qs@^6.9.14", "@types/qs@^6.9.5":
-  version "6.9.14"
-  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.14.tgz#169e142bfe493895287bee382af6039795e9b75b"
-  integrity sha512-5khscbd3SwWMhFqylJBLQ0zIu7c1K6Vz0uBIt915BI3zV0q1nfjRQD3RqSBcPaO6PHEF4ov/t9y89fSiyThlPA==
+"@types/qs@*", "@types/qs@^6.9.15", "@types/qs@^6.9.5":
+  version "6.9.15"
+  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.15.tgz#adde8a060ec9c305a82de1babc1056e73bd64dce"
+  integrity sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==
 
 "@types/range-parser@*":
   version "1.2.3"
@@ -10171,10 +10171,10 @@ flow-parser@0.*:
   resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.206.0.tgz#f4f794f8026535278393308e01ea72f31000bfef"
   integrity sha512-HVzoK3r6Vsg+lKvlIZzaWNBVai+FXTX1wdYhz/wVlH13tb/gOdLXmlTqy6odmTBhT5UoWUbq0k8263Qhr9d88w==
 
-flowbite-datepicker@^1.2.6:
-  version "1.2.6"
-  resolved "https://registry.yarnpkg.com/flowbite-datepicker/-/flowbite-datepicker-1.2.6.tgz#97fbd46496ec72f0322e5ea4d3d2462437ec1c0e"
-  integrity sha512-UbU/xXs9HFiwWfL4M1vpwIo8EpS0NUQSOvYnp0Z9u3N118nU7lPFGoUOq7su9d0aOJy9FssXzx1SZwN8MXhE1g==
+flowbite-datepicker@^1.2.7:
+  version "1.2.7"
+  resolved "https://registry.yarnpkg.com/flowbite-datepicker/-/flowbite-datepicker-1.2.7.tgz#274cc882a6a71ecffdca39b20409e986d40ebbdf"
+  integrity sha512-lT0SOrux9Hu9TMO1EG03r9B8VMppRr0wmz+M4YC9qVR0BCG83Qaky4HGHcAehcoBSXh13DfIg0jFKwSxWZuXBA==
   dependencies:
     flowbite "^2.0.0"
 
@@ -15072,10 +15072,10 @@ qs@6.11.0:
   dependencies:
     side-channel "^1.0.4"
 
-qs@^6.10.0, qs@^6.11.0, qs@^6.11.2, qs@^6.12.1:
-  version "6.12.1"
-  resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.1.tgz#39422111ca7cbdb70425541cba20c7d7b216599a"
-  integrity sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==
+qs@^6.10.0, qs@^6.11.0, qs@^6.11.2, qs@^6.12.3:
+  version "6.12.3"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.3.tgz#e43ce03c8521b9c7fd7f1f13e514e5ca37727754"
+  integrity sha512-AWJm14H1vVaO/iNZ4/hO+HyaTehuy9nRqVdkTqlJt0HWvBiBIEXFmb4C0DGeYo3Xes9rrEW+TxHsaigCbN5ICQ==
   dependencies:
     side-channel "^1.0.6"