v1.139.0

huridocs · Oct 23, 2023 · d8aa4cb · d8aa4cb
2 parents 5b40a72 + 1407107
commit d8aa4cb
Show file tree

Hide file tree

Showing 209 changed files with 6,756 additions and 4,202 deletions.
diff --git a/app/api/config.ts b/app/api/config.ts
@@ -25,7 +25,7 @@ const filesRootPath = FILES_ROOT_PATH || rootPath;
 // when using multiple node processes
 const CLUSTER_MODE = process.env.CLUSTER_MODE || false;
 
-const onlyDBHOST = () => (DBHOST ? `mongodb://${DBHOST}/` : 'mongodb://localhost/');
+const onlyDBHOST = () => (DBHOST ? `mongodb://${DBHOST}/` : 'mongodb://127.0.0.1/');
 
 export const config = {
   VERSION: ENVIRONMENT ? version : `development-${version}`,

diff --git a/app/api/csv/entityRow.ts b/app/api/csv/entityRow.ts
@@ -5,7 +5,7 @@ type Languages = string[];
 
 export type RawEntity = {
   language: string;
-  [k: string]: string;
+  propertiesFromColumns: CSVRow;
 };
 
 const toSafeName = (row: CSVRow, newNameGeneration: boolean = false): CSVRow =>
@@ -57,16 +57,16 @@ const extractEntity = (
             const propName = key.split(`__${languageCode}`)[0];
             const selectedKey =
               propName in propNameToThesauriId ? `${propName}__${defaultLanguage}` : key;
-            entity[propName] = safeNamed[selectedKey]; //eslint-disable-line no-param-reassign
+            entity.propertiesFromColumns[propName] = safeNamed[selectedKey]; //eslint-disable-line no-param-reassign
             return entity;
           },
-          { ...baseEntity, language: languageCode }
+          { propertiesFromColumns: { ...baseEntity }, language: languageCode }
         )
   );
 
   return {
-    rawEntity: rawEntities.find((e: CSVRow) => e.language === currentLanguage),
-    rawTranslations: rawEntities.filter((e: CSVRow) => e.language !== currentLanguage),
+    rawEntity: rawEntities.find((e: RawEntity) => e.language === currentLanguage),
+    rawTranslations: rawEntities.filter((e: RawEntity) => e.language !== currentLanguage),
   };
 };
 

diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts
@@ -22,7 +22,9 @@ const parse = async (toImportEntity: RawEntity, prop: PropertySchema, dateFormat
     : typeParsers.text(toImportEntity, prop);
 
 const hasValidValue = (prop: PropertySchema, toImportEntity: RawEntity) =>
-  prop.name ? toImportEntity[prop.name] || prop.type === propertyTypes.generatedid : false;
+  prop.name
+    ? toImportEntity.propertiesFromColumns[prop.name] || prop.type === propertyTypes.generatedid
+    : false;
 
 const toMetadata = async (
   template: TemplateSchema,
@@ -40,17 +42,18 @@ const toMetadata = async (
       Promise.resolve({})
     );
 
-const currentEntityIdentifiers = async (sharedId: string, language: string) =>
+const currentEntityIdentifiers = async (sharedId: string | undefined, language: string) =>
   sharedId ? entities.get({ sharedId, language }, '_id sharedId').then(([e]) => e) : {};
 
 const titleByTemplate = (template: TemplateSchema, entity: RawEntity) => {
+  const { propertiesFromColumns: data } = entity;
   const generatedTitle =
-    !entity.title &&
+    !data.title &&
     template.commonProperties?.find(property => property.name === 'title' && property.generatedId);
   if (generatedTitle) {
     return generateID(3, 4, 4);
   }
-  return entity.title;
+  return data.title;
 };
 
 const entityObject = async (
@@ -61,7 +64,7 @@ const entityObject = async (
   title: titleByTemplate(template, toImportEntity),
   template: template._id,
   metadata: await toMetadata(template, toImportEntity, dateFormat),
-  ...(await currentEntityIdentifiers(toImportEntity.id, language)),
+  ...(await currentEntityIdentifiers(toImportEntity.propertiesFromColumns.id, language)),
 });
 
 type Options = {
@@ -76,17 +79,18 @@ const importEntity = async (
   importFile: ImportFile,
   { user = {}, language, dateFormat }: Options
 ) => {
-  const { attachments } = toImportEntity;
-  delete toImportEntity.attachments;
+  const { propertiesFromColumns } = toImportEntity;
+  const { attachments } = propertiesFromColumns;
+  delete propertiesFromColumns.attachments;
   const eo = await entityObject(toImportEntity, template, { language, dateFormat });
   const entity = await entities.save(
     eo,
     { user, language },
     { updateRelationships: true, index: false }
   );
 
-  if (toImportEntity.file && entity.sharedId) {
-    const file = await importFile.extractFile(toImportEntity.file);
+  if (propertiesFromColumns.file && entity.sharedId) {
+    const file = await importFile.extractFile(propertiesFromColumns.file);
     await processDocument(entity.sharedId, file);
     await storage.storeFile(file.filename, createReadStream(file.path), 'document');
   }
@@ -148,7 +152,13 @@ const translateEntity = async (
     await Promise.all(
       translations.map(async translatedEntity => {
         const translatedEntityObject = await entityObject(
-          { ...translatedEntity, id: ensure(entity.sharedId) },
+          {
+            ...translatedEntity,
+            propertiesFromColumns: {
+              ...translatedEntity.propertiesFromColumns,
+              id: ensure(entity.sharedId),
+            },
+          },
           template,
           {
             language: translatedEntity.language,
@@ -167,8 +177,8 @@ const translateEntity = async (
 
   await Promise.all(
     translations.map(async translatedEntity => {
-      if (translatedEntity.file) {
-        const file = await importFile.extractFile(translatedEntity.file);
+      if (translatedEntity.propertiesFromColumns.file) {
+        const file = await importFile.extractFile(translatedEntity.propertiesFromColumns.file);
         await processDocument(ensure(entity.sharedId), file);
       }
     })

diff --git a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap
@@ -1,11 +1,11 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`importFile readStream should return a readable stream for the csv file 1`] = `
-"Title , text label  , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label
+"Title , text label  , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label, Language
 
-title1, text value 1, 1977, ______________, thesauri1   , notType1                 , 1|1,,tag1, multivalue1,03/01/2022
-title2, text value 2, 2019, ______________, thesauri2   , notType2                 ,    ,,tag2, multivalue2,01/03/2022
-title3, text value 3, 2020, ______________, thesauri2   , notType3                 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022
+title1, text value 1, 1977, ______________, thesauri1   , notType1                 , 1|1,,tag1, multivalue1,03/01/2022,English
+title2, text value 2, 2019, ______________, thesauri2   , notType2                 ,    ,,tag2, multivalue2,01/03/2022,Spanish
+title3, text value 3, 2020, ______________, thesauri2   , notType3                 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022,AnyStringIsGood
 "
 `;
 

diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js
@@ -168,6 +168,7 @@ describe('csvLoader', () => {
         'additional_tag(s)',
         'multi_select_label',
         'date_label',
+        'language',
       ]);
     });
 
@@ -177,6 +178,11 @@ describe('csvLoader', () => {
       expect(textValues.length).toEqual(0);
     });
 
+    it('should import properties named "Language" properly', () => {
+      const textValues = imported.map(i => i.metadata.language[0].value);
+      expect(textValues).toEqual(['English', 'Spanish', 'AnyStringIsGood']);
+    });
+
     describe('metadata parsing', () => {
       it('should parse metadata properties by type using typeParsers', () => {
         const textValues = imported.map(i => i.metadata.text_label[0].value);
@@ -287,8 +293,8 @@ describe('csvLoader', () => {
     it('should fail when parsing throws an error', async () => {
       jest.spyOn(entities, 'save').mockImplementation(() => Promise.resolve({}));
       jest.spyOn(typeParsers, 'text').mockImplementation(entity => {
-        if (entity.title === 'title2') {
-          throw new Error(`error-${entity.title}`);
+        if (entity.propertiesFromColumns.title === 'title2') {
+          throw new Error(`error-${entity.propertiesFromColumns.title}`);
         }
       });
 

diff --git a/app/api/csv/specs/csvLoaderFixtures.ts b/app/api/csv/specs/csvLoaderFixtures.ts
@@ -110,6 +110,11 @@ export default {
           label: 'Date label',
           name: templateUtils.safeName('Date label'),
         },
+        {
+          type: propertyTypes.text,
+          label: 'Language',
+          name: templateUtils.safeName('Language'),
+        },
       ],
     },
     {

diff --git a/app/api/csv/specs/entityRow.spec.js b/app/api/csv/specs/entityRow.spec.js
@@ -19,8 +19,10 @@ describe('entityRow', () => {
     );
 
     expect(rawEntity).toEqual({
-      title: 'test_en',
-      not_portuguese_ptescaped_property: 'not portuguese',
+      propertiesFromColumns: {
+        title: 'test_en',
+        not_portuguese_ptescaped_property: 'not portuguese',
+      },
       language: 'en',
     });
   });
@@ -37,7 +39,9 @@ describe('entityRow', () => {
       {}
     );
 
-    expect(rawTranslations).toEqual([{ title: 'test_es', language: 'es' }]);
+    expect(rawTranslations).toEqual([
+      { propertiesFromColumns: { title: 'test_es' }, language: 'es' },
+    ]);
   });
 
   it('should return translations for languages that have values not blank', () => {
@@ -53,7 +57,9 @@ describe('entityRow', () => {
       {}
     );
 
-    expect(rawTranslations).toEqual([{ title: 'test_es', language: 'es' }]);
+    expect(rawTranslations).toEqual([
+      { propertiesFromColumns: { title: 'test_es' }, language: 'es' },
+    ]);
   });
 
   it('should return all translations when everything is translated', () => {
@@ -70,8 +76,8 @@ describe('entityRow', () => {
     );
 
     expect(rawTranslations).toEqual([
-      { title: 'test_es', language: 'es' },
-      { text: 'text_pt', language: 'pt' },
+      { propertiesFromColumns: { title: 'test_es' }, language: 'es' },
+      { propertiesFromColumns: { text: 'text_pt' }, language: 'pt' },
     ]);
   });
 
@@ -90,17 +96,21 @@ describe('entityRow', () => {
     );
 
     expect(rawEntity).toEqual({
-      title: 'test_es',
-      some__entirely_new_property: 'has __en in name, but is not english',
-      a__pt_property: 'not portugese',
+      propertiesFromColumns: {
+        title: 'test_es',
+        some__entirely_new_property: 'has __en in name, but is not english',
+        a__pt_property: 'not portugese',
+      },
       language: 'es',
     });
 
     expect(rawTranslations).toEqual([
       {
-        title: 'test_en',
-        some__entirely_new_property: 'has __en in name, but is not english',
-        a__pt_property: 'not portugese',
+        propertiesFromColumns: {
+          title: 'test_en',
+          some__entirely_new_property: 'has __en in name, but is not english',
+          a__pt_property: 'not portugese',
+        },
         language: 'en',
       },
     ]);

diff --git a/app/api/csv/specs/test.csv b/app/api/csv/specs/test.csv
@@ -1,5 +1,5 @@
-Title , text label  , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label
+Title , text label  , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), Multi Select Label, Date label, Language
 
-title1, text value 1, 1977, ______________, thesauri1   , notType1                 , 1|1,,tag1, multivalue1,03/01/2022
-title2, text value 2, 2019, ______________, thesauri2   , notType2                 ,    ,,tag2, multivalue2,01/03/2022
-title3, text value 3, 2020, ______________, thesauri2   , notType3                 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022
+title1, text value 1, 1977, ______________, thesauri1   , notType1                 , 1|1,,tag1, multivalue1,03/01/2022,English
+title2, text value 2, 2019, ______________, thesauri2   , notType2                 ,    ,,tag2, multivalue2,01/03/2022,Spanish
+title3, text value 3, 2020, ______________, thesauri2   , notType3                 , 0|0,,tag3, multivalue1|multivalue3,01/01/2022,AnyStringIsGood
diff --git a/app/api/csv/specs/typeParsers.spec.js b/app/api/csv/specs/typeParsers.spec.js
@@ -1,11 +1,15 @@
 import moment from 'moment';
 import typeParsers from '../typeParsers';
 
+const rawEntityWithProps = props => ({
+  propertiesFromColumns: props,
+});
+
 describe('csvLoader typeParsers', () => {
   describe('text', () => {
     it('should return the value', async () => {
       const templateProp = { name: 'text_prop' };
-      const rawEntity = { text_prop: 'text' };
+      const rawEntity = rawEntityWithProps({ text_prop: 'text' });
 
       expect(await typeParsers.text(rawEntity, templateProp)).toEqual([{ value: 'text' }]);
     });
@@ -14,14 +18,14 @@ describe('csvLoader typeParsers', () => {
   describe('numeric', () => {
     it('should return numeric value', async () => {
       const templateProp = { name: 'numeric_prop' };
-      const rawEntity = { numeric_prop: '2019' };
+      const rawEntity = rawEntityWithProps({ numeric_prop: '2019' });
 
       expect(await typeParsers.numeric(rawEntity, templateProp)).toEqual([{ value: 2019 }]);
     });
 
     it('should return original value if value is NaN (will be catched by the entitiy validator)', async () => {
       const templateProp = { name: 'numeric_prop' };
-      const rawEntity = { numeric_prop: 'Not a number' };
+      const rawEntity = rawEntityWithProps({ numeric_prop: 'Not a number' });
 
       expect(await typeParsers.numeric(rawEntity, templateProp)).toEqual([
         { value: 'Not a number' },
@@ -32,7 +36,7 @@ describe('csvLoader typeParsers', () => {
   describe('link', () => {
     it('should use the text as url and label', async () => {
       const templateProp = { name: 'link_prop' };
-      const rawEntity = { link_prop: 'http://www.url.com' };
+      const rawEntity = rawEntityWithProps({ link_prop: 'http://www.url.com' });
 
       expect(await typeParsers.link(rawEntity, templateProp)).toEqual([
         {
@@ -46,14 +50,14 @@ describe('csvLoader typeParsers', () => {
 
     it('should return null if url is not valid', async () => {
       const templateProp = { name: 'link_prop' };
-      const rawEntity = { link_prop: 'url' };
+      const rawEntity = rawEntityWithProps({ link_prop: 'url' });
 
       expect(await typeParsers.link(rawEntity, templateProp)).toBe(null);
     });
 
     it('should use "|" as separator for label and url', async () => {
       const templateProp = { name: 'link_prop' };
-      const rawEntity = { link_prop: 'label|http://www.url.com' };
+      const rawEntity = rawEntityWithProps({ link_prop: 'label|http://www.url.com' });
 
       expect(await typeParsers.link(rawEntity, templateProp)).toEqual([
         {
@@ -104,7 +108,11 @@ describe('csvLoader typeParsers', () => {
       async ({ dateProp, dateFormat, expectedDate }) => {
         const templateProp = { name: 'date_prop' };
 
-        const expected = await typeParsers.date({ date_prop: dateProp }, templateProp, dateFormat);
+        const expected = await typeParsers.date(
+          rawEntityWithProps({ date_prop: dateProp }),
+          templateProp,
+          dateFormat
+        );
 
         expect(moment.utc(expected[0].value, 'X').format('DD-MM-YYYY')).toEqual(expectedDate);
       }
@@ -129,7 +137,7 @@ describe('csvLoader typeParsers', () => {
         const templateProp = { name: 'date_prop' };
 
         const expected = await typeParsers.multidate(
-          { date_prop: dateProp },
+          rawEntityWithProps({ date_prop: dateProp }),
           templateProp,
           dateFormat
         );
@@ -160,7 +168,11 @@ describe('csvLoader typeParsers', () => {
           {
             value: { from, to },
           },
-        ] = await typeParsers.daterange({ date_prop: dateProp }, templateProp, dateFormat);
+        ] = await typeParsers.daterange(
+          rawEntityWithProps({ date_prop: dateProp }),
+          templateProp,
+          dateFormat
+        );
 
         expect({
           from: from && moment.utc(from, 'X').format('DD-MM-YYYY'),
@@ -249,7 +261,7 @@ describe('csvLoader typeParsers', () => {
         const templateProp = { name: 'date_prop' };
 
         const expected = await typeParsers.multidaterange(
-          { date_prop: dateProp },
+          rawEntityWithProps({ date_prop: dateProp }),
           templateProp,
           dateFormat
         );