diff --git a/app/api/entities.v2/EntityInputDataSchema.ts b/app/api/entities.v2/EntityInputDataSchema.ts new file mode 100644 index 0000000000..818c4296f8 --- /dev/null +++ b/app/api/entities.v2/EntityInputDataSchema.ts @@ -0,0 +1,88 @@ +import { availableLanguagesISO6391 } from 'shared/languagesList'; + +const linkSchema = { + type: 'object', + additionalProperties: false, + required: ['label', 'url'], + properties: { + label: { oneOf: [{ type: 'string' }, { type: 'null' }] }, + url: { oneOf: [{ type: 'string' }, { type: 'null' }] }, + }, +}; + +const dateRangeSchema = { + type: 'object', + additionalProperties: false, + required: ['from', 'to'], + properties: { + from: { oneOf: [{ type: 'number' }, { type: 'null' }] }, + to: { oneOf: [{ type: 'number' }, { type: 'null' }] }, + }, +}; + +const latLonSchema = { + type: 'object', + required: ['lon', 'lat'], + additionalProperties: false, + properties: { + label: { type: 'string' }, + lat: { type: 'number', minimum: -90, maximum: 90 }, + lon: { type: 'number', minimum: -180, maximum: 180 }, + }, +}; + +const geolocationSchema = { + type: 'array', + items: latLonSchema, +}; + +const propertyValueSchema = { + definitions: { linkSchema, dateRangeSchema, latLonSchema }, + oneOf: [ + { type: 'null' }, + { type: 'string' }, + { type: 'number' }, + { type: 'boolean' }, + linkSchema, + dateRangeSchema, + latLonSchema, + geolocationSchema, + ], +}; + +const metadataObjectSchema = { + type: 'object', + definitions: { propertyValueSchema }, + required: ['value'], + properties: { + value: propertyValueSchema, + }, +}; + +const metadataSchema = { + type: 'object', + definitions: { metadataObjectSchema }, + additionalProperties: { + anyOf: [{ type: 'array', items: metadataObjectSchema }], + }, + patternProperties: { + '^.*_nested$': { type: 'array', items: { type: 'object' } }, + }, +}; + +export const entityInputDataSchema = { + title: 'EntityInputData', + $schema: 'http://json-schema.org/schema#', + type: 'object', + required: ['_id', 'sharedId', 'language', 'title', 'template', 'metadata'], + properties: { + _id: { type: 'string' }, + sharedId: { type: 'string', minLength: 1 }, + language: { enum: availableLanguagesISO6391 }, + title: { type: 'string', minLength: 1 }, + template: { type: 'string' }, + metadata: metadataSchema, + }, +}; + +export const emitSchemaTypes = true; diff --git a/app/api/entities.v2/EntityInputDataType.d.ts b/app/api/entities.v2/EntityInputDataType.d.ts new file mode 100644 index 0000000000..beef2170b9 --- /dev/null +++ b/app/api/entities.v2/EntityInputDataType.d.ts @@ -0,0 +1,227 @@ +/* eslint-disable */ +/**AUTO-GENERATED. RUN yarn emit-types to update.*/ + +export interface EntityInputData { + _id: string; + sharedId: string; + language: + | 'ab' + | 'aa' + | 'af' + | 'ak' + | 'sq' + | 'am' + | 'ar' + | 'an' + | 'hy' + | 'as' + | 'av' + | 'ae' + | 'ay' + | 'az' + | 'bm' + | 'ba' + | 'eu' + | 'be' + | 'bn' + | 'bh' + | 'bi' + | 'bs' + | 'br' + | 'bg' + | 'my' + | 'ca' + | 'ch' + | 'ce' + | 'ny' + | 'zh' + | 'zh-Hans' + | 'zh-Hant' + | 'cv' + | 'kw' + | 'co' + | 'cr' + | 'hr' + | 'cs' + | 'da' + | 'dv' + | 'nl' + | 'dz' + | 'en' + | 'eo' + | 'et' + | 'ee' + | 'fo' + | 'fj' + | 'fi' + | 'fr' + | 'ff' + | 'gl' + | 'gd' + | 'gv' + | 'ka' + | 'de' + | 'el' + | 'gn' + | 'gu' + | 'ht' + | 'ha' + | 'he' + | 'hz' + | 'hi' + | 'ho' + | 'hu' + | 'is' + | 'io' + | 'ig' + | 'in' + | 'ia' + | 'ie' + | 'iu' + | 'ik' + | 'ga' + | 'it' + | 'ja' + | 'jv' + | 'kl' + | 'kn' + | 'kr' + | 'ks' + | 'kk' + | 'km' + | 'ki' + | 'rw' + | 'rn' + | 'ky' + | 'kv' + | 'kg' + | 'ko' + | 'ku' + | 'kj' + | 'lo' + | 'la' + | 'lv' + | 'li' + | 'ln' + | 'lt' + | 'lu' + | 'lg' + | 'lb' + | 'mk' + | 'mg' + | 'ms' + | 'ml' + | 'mt' + | 'mi' + | 'mr' + | 'mh' + | 'mn' + | 'na' + | 'nv' + | 'ng' + | 'nd' + | 'ne' + | 'no' + | 'nb' + | 'nn' + | 'oc' + | 'oj' + | 'cu' + | 'or' + | 'om' + | 'os' + | 'pi' + | 'ps' + | 'fa' + | 'pl' + | 'pt' + | 'pa' + | 'qu' + | 'rm' + | 'ro' + | 'ru' + | 'se' + | 'sm' + | 'sg' + | 'sa' + | 'sr' + | 'sh' + | 'st' + | 'tn' + | 'sn' + | 'ii' + | 'sd' + | 'si' + | 'ss' + | 'sk' + | 'sl' + | 'so' + | 'nr' + | 'es' + | 'su' + | 'sw' + | 'sv' + | 'tl' + | 'ty' + | 'tg' + | 'ta' + | 'tt' + | 'te' + | 'th' + | 'bo' + | 'ti' + | 'to' + | 'ts' + | 'tr' + | 'tk' + | 'tw' + | 'ug' + | 'uk' + | 'ur' + | 'uz' + | 've' + | 'vi' + | 'vo' + | 'wa' + | 'cy' + | 'wo' + | 'fy' + | 'xh' + | 'yi' + | 'yo' + | 'za' + | 'zu'; + title: string; + template: string; + metadata: { + [k: string]: + | { + value: + | null + | string + | number + | boolean + | { + label: string | null; + url: string | null; + } + | { + from: number | null; + to: number | null; + } + | { + label?: string; + lat: number; + lon: number; + } + | { + label?: string; + lat: number; + lon: number; + }[]; + [k: string]: unknown | undefined; + }[] + | undefined; + }; + [k: string]: unknown | undefined; +} diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 73099ce7b4..fd3f35e94d 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -17,6 +17,7 @@ import ID from 'shared/uniqueID'; import { denormalizeMetadata, denormalizeRelated } from './denormalize'; import model from './entitiesModel'; +import { EntityCreatedEvent } from './events/EntityCreatedEvent'; import { EntityUpdatedEvent } from './events/EntityUpdatedEvent'; import { EntityDeletedEvent } from './events/EntityDeletedEvent'; import { saveSelections } from './metadataExtraction/saveSelections'; @@ -149,7 +150,7 @@ async function updateEntity(entity, _template, unrestricted = false) { await applicationEventsBus.emit( new EntityUpdatedEvent({ before: docLanguages, - after: result, + after: await model.get({ sharedId: entity.sharedId }), targetLanguageKey: entity.language, }) ); @@ -157,7 +158,7 @@ async function updateEntity(entity, _template, unrestricted = false) { return result; } -async function createEntity(doc, languages, sharedId, docTemplate) { +async function createEntity(doc, [currentLanguage, languages], sharedId, docTemplate) { if (!docTemplate) docTemplate = await templates.getById(doc.template); const thesauriByKey = await templates.getRelatedThesauri(docTemplate); @@ -203,6 +204,14 @@ async function createEntity(doc, languages, sharedId, docTemplate) { await updateNewRelationships(v2RelationshipsUpdates); await Promise.all(result.map(r => denormalizeAfterEntityCreation(r))); + + await applicationEventsBus.emit( + new EntityCreatedEvent({ + entities: await model.get({ sharedId }), + targetLanguageKey: currentLanguage, + }) + ); + return result; } @@ -410,7 +419,12 @@ export default { docTemplate = defaultTemplate; } doc.metadata = doc.metadata || {}; - await this.createEntity(this.sanitize(doc, docTemplate), languages, sharedId, docTemplate); + await this.createEntity( + this.sanitize(doc, docTemplate), + [language, languages], + sharedId, + docTemplate + ); } const [entity] = includeDocuments diff --git a/app/api/entities/events/EntityCreatedEvent.ts b/app/api/entities/events/EntityCreatedEvent.ts new file mode 100644 index 0000000000..12d561a6ec --- /dev/null +++ b/app/api/entities/events/EntityCreatedEvent.ts @@ -0,0 +1,11 @@ +import { AbstractEvent } from 'api/eventsbus'; +import { EntitySchema } from 'shared/types/entityType'; + +interface EntityCreatedData { + entities: EntitySchema[]; + targetLanguageKey: string; +} + +class EntityCreatedEvent extends AbstractEvent {} + +export { EntityCreatedEvent }; diff --git a/app/api/entities/specs/entities.spec.js b/app/api/entities/specs/entities.spec.js index 5ea4daa2fb..6e28939018 100644 --- a/app/api/entities/specs/entities.spec.js +++ b/app/api/entities/specs/entities.spec.js @@ -2,6 +2,8 @@ /* eslint-disable max-nested-callbacks,max-statements */ import Ajv from 'ajv'; +// eslint-disable-next-line node/no-restricted-import +import fs from 'fs/promises'; import entitiesModel from 'api/entities/entitiesModel'; import { spyOnEmit } from 'api/eventsbus/eventTesting'; @@ -11,10 +13,9 @@ import { search } from 'api/search'; import date from 'api/utils/date.js'; import db from 'api/utils/testing_db'; import { UserInContextMockFactory } from 'api/utils/testingUserInContext'; -// eslint-disable-next-line node/no-restricted-import -import fs from 'fs/promises'; import { UserRole } from 'shared/types/userSchema'; +import { applicationEventsBus } from 'api/eventsbus'; import fixtures, { adminId, batmanFinishesId, @@ -32,6 +33,7 @@ import fixtures, { import entities from '../entities.js'; import { EntityUpdatedEvent } from '../events/EntityUpdatedEvent'; import { EntityDeletedEvent } from '../events/EntityDeletedEvent'; +import { EntityCreatedEvent } from '../events/EntityCreatedEvent'; describe('entities', () => { const userFactory = new UserInContextMockFactory(); @@ -509,22 +511,61 @@ describe('entities', () => { }); describe('events', () => { + let emitSpy; + + beforeEach(() => { + emitSpy = jest.spyOn(applicationEventsBus, 'emit'); + emitSpy.mockClear(); + }); + + it('should emit an event when an entity is created', async () => { + const newEntity = { + template: templateId, + title: 'New Super Hero', + metadata: { + text: [{ value: 'New Text' }], + property1: [{ value: 'value1' }], + property2: [{ value: 'value2' }], + description: [{ value: 'ew Description' }], + friends: [{ icon: null, label: 'shared2title', type: 'entity', value: 'shared2' }], + enemies: [{ icon: null, label: 'shared2title', type: 'entity', value: 'shared2' }], + select: [], + }, + }; + + const savedEntity = await entities.save(newEntity, { + user: { _id: adminId }, + language: 'en', + }); + + const afterAllLanguages = await entities.getAllLanguages(savedEntity.sharedId); + + expect(emitSpy.mock.calls[0][0]).toBeInstanceOf(EntityCreatedEvent); + expect(emitSpy).toHaveBeenCalledWith( + new EntityCreatedEvent({ + entities: afterAllLanguages, + targetLanguageKey: 'en', + }) + ); + }); + it('should emit an event when an entity is updated', async () => { - const emitSpy = spyOnEmit(); const before = fixtures.entities.find(e => e._id === batmanFinishesId); - const beforeAllLanguages = fixtures.entities.filter(e => e.sharedId === before.sharedId); + const beforeAllLanguages = await entities.getAllLanguages(before.sharedId); const after = { ...before, title: 'new title' }; await entities.save(after, { language: 'en' }); const afterAllLanguages = await entities.getAllLanguages(before.sharedId); - emitSpy.expectToEmitEvent(EntityUpdatedEvent, { - before: beforeAllLanguages, - after: afterAllLanguages, - targetLanguageKey: 'en', - }); - emitSpy.restore(); + expect(emitSpy.mock.calls[0][0]).toBeInstanceOf(EntityUpdatedEvent); + expect(emitSpy).toHaveBeenCalledWith( + new EntityUpdatedEvent({ + before: beforeAllLanguages, + after: afterAllLanguages, + targetLanguageKey: 'en', + }) + ); }); }); }); diff --git a/app/api/externalIntegrations.v2/automaticTranslation/RequestEntityTranslation.ts b/app/api/externalIntegrations.v2/automaticTranslation/RequestEntityTranslation.ts new file mode 100644 index 0000000000..f0d25f9aef --- /dev/null +++ b/app/api/externalIntegrations.v2/automaticTranslation/RequestEntityTranslation.ts @@ -0,0 +1,107 @@ +import { getTenant } from 'api/common.v2/database/getConnectionForCurrentTenant'; +import { TaskManager } from 'api/services/tasksmanager/TaskManager'; +import { TemplatesDataSource } from 'api/templates.v2/contracts/TemplatesDataSource'; +import { EntityInputData } from 'api/entities.v2/EntityInputDataType'; +import { EntityInputValidator } from './contracts/EntityInputValidator'; +import { ATConfigService } from './services/GetAutomaticTranslationConfig'; +import { InvalidInputDataFormat } from './errors/generateATErrors'; + +export type ATTaskMessage = { + params: { + key: string[]; + text: string; + language_from: string; + languages_to: string[]; + }; +}; + +export class RequestEntityTranslation { + static SERVICE_NAME = 'AutomaticTranslation'; + + private taskManager: TaskManager; + + private templatesDS: TemplatesDataSource; + + private aTConfigService: ATConfigService; + + private inputValidator: EntityInputValidator; + + constructor( + taskManager: TaskManager, + templatesDS: TemplatesDataSource, + aTConfigService: ATConfigService, + inputValidator: EntityInputValidator + ) { + this.taskManager = taskManager; + this.templatesDS = templatesDS; + this.aTConfigService = aTConfigService; + this.inputValidator = inputValidator; + } + + // eslint-disable-next-line max-statements + async execute(entity: EntityInputData | unknown) { + if (!this.inputValidator.validate(entity)) { + throw new InvalidInputDataFormat(this.inputValidator.getErrors()[0]); + } + const atConfig = await this.aTConfigService.get(); + const atTemplateConfig = atConfig.templates.find( + t => t.template === entity.template?.toString() + ); + + if (!atTemplateConfig) { + return; + } + + const template = await this.templatesDS.getById(atTemplateConfig?.template); + + const languageFrom = entity.language; + if (!atConfig.languages.includes(languageFrom)) { + return; + } + + const languagesTo = atConfig.languages.filter(language => language !== languageFrom); + atTemplateConfig?.commonProperties.forEach(async commonPropId => { + const commonPropName = template?.commonProperties.find( + prop => prop.id === commonPropId + )?.name; + + if (!commonPropName) { + throw new Error('Common property not found'); + } + + if (!(typeof entity[commonPropName] === 'string')) { + throw new Error('Common property is not a string'); + } + + await this.taskManager.startTask({ + params: { + key: [getTenant().name, entity.sharedId, commonPropName], + text: entity[commonPropName], + language_from: languageFrom, + languages_to: languagesTo, + }, + }); + }); + atTemplateConfig?.properties.forEach(async propId => { + const propName = template?.properties.find(prop => prop.id === propId)?.name; + if (!propName) { + throw new Error('Property not found'); + } + + if (!(typeof entity.metadata[propName]?.[0].value === 'string')) { + throw new Error('Property is not a string'); + } + + if (entity.metadata[propName]?.[0].value) { + await this.taskManager.startTask({ + params: { + key: [getTenant().name, entity.sharedId, propName], + text: entity.metadata[propName][0].value, + language_from: languageFrom, + languages_to: languagesTo, + }, + }); + } + }); + } +} diff --git a/app/api/externalIntegrations.v2/automaticTranslation/contracts/EntityInputValidator.ts b/app/api/externalIntegrations.v2/automaticTranslation/contracts/EntityInputValidator.ts new file mode 100644 index 0000000000..91692e571c --- /dev/null +++ b/app/api/externalIntegrations.v2/automaticTranslation/contracts/EntityInputValidator.ts @@ -0,0 +1,6 @@ +import { EntityInputData } from 'api/entities.v2/EntityInputDataType'; + +export interface EntityInputValidator { + getErrors(): string[]; + validate(data: unknown): data is EntityInputData; +} diff --git a/app/api/externalIntegrations.v2/automaticTranslation/infrastructure/EntityInputValidator.ts b/app/api/externalIntegrations.v2/automaticTranslation/infrastructure/EntityInputValidator.ts new file mode 100644 index 0000000000..f0deedbfcb --- /dev/null +++ b/app/api/externalIntegrations.v2/automaticTranslation/infrastructure/EntityInputValidator.ts @@ -0,0 +1,20 @@ +import { Ajv } from 'ajv'; +import { entityInputDataSchema } from 'api/entities.v2/EntityInputDataSchema'; +import { EntityInputData } from 'api/entities.v2/EntityInputDataType'; +import { EntityInputValidator } from '../contracts/EntityInputValidator'; + +export class AJVEntityInputValidator implements EntityInputValidator { + private errors: string[] = []; + + getErrors() { + return this.errors; + } + + validate(data: unknown) { + const ajv = new Ajv({ strict: false }); + const validate = ajv.compile(entityInputDataSchema); + const result = validate(data); + this.errors = validate.errors ? validate.errors?.map(e => JSON.stringify(e.params)) : []; + return result; + } +} diff --git a/app/api/externalIntegrations.v2/automaticTranslation/services/GetAutomaticTranslationConfig.ts b/app/api/externalIntegrations.v2/automaticTranslation/services/GetAutomaticTranslationConfig.ts index c5caaa1437..dec59ce500 100644 --- a/app/api/externalIntegrations.v2/automaticTranslation/services/GetAutomaticTranslationConfig.ts +++ b/app/api/externalIntegrations.v2/automaticTranslation/services/GetAutomaticTranslationConfig.ts @@ -3,7 +3,7 @@ import { TemplatesDataSource } from 'api/templates.v2/contracts/TemplatesDataSou import { ATGateway } from '../contracts/ATGateway'; import { ATConfigDataSource } from '../contracts/ATConfigDataSource'; -export class GetAutomaticTranslationConfig { +export class ATConfigService { private settings: SettingsDataSource; private config: ATConfigDataSource; @@ -24,7 +24,7 @@ export class GetAutomaticTranslationConfig { this.automaticTranslation = automaticTranslation; } - async execute() { + async get() { const config = await this.config.get(); const validProperties = await this.templates.getAllTextProperties().all(); diff --git a/app/api/externalIntegrations.v2/automaticTranslation/services/specs/GetAutomaticTranslationConfig.spec.ts b/app/api/externalIntegrations.v2/automaticTranslation/services/specs/GetAutomaticTranslationConfig.spec.ts index 2181008369..b622a0ff8d 100644 --- a/app/api/externalIntegrations.v2/automaticTranslation/services/specs/GetAutomaticTranslationConfig.spec.ts +++ b/app/api/externalIntegrations.v2/automaticTranslation/services/specs/GetAutomaticTranslationConfig.spec.ts @@ -6,11 +6,11 @@ import { getFixturesFactory } from 'api/utils/fixturesFactory'; import { testingEnvironment } from 'api/utils/testingEnvironment'; import { MongoATConfigDataSource } from '../../infrastructure/MongoATConfigDataSource'; import { ATExternalAPI } from '../../infrastructure/ATExternalAPI'; -import { GetAutomaticTranslationConfig } from '../GetAutomaticTranslationConfig'; +import { ATConfigService } from '../GetAutomaticTranslationConfig'; const createService = () => { const transactionManager = DefaultTransactionManager(); - return new GetAutomaticTranslationConfig( + return new ATConfigService( DefaultSettingsDataSource(transactionManager), new MongoATConfigDataSource(getConnection(), transactionManager), DefaultTemplatesDataSource(transactionManager), @@ -89,7 +89,7 @@ afterAll(async () => { describe('GetAutomaticTranslationConfig', () => { it('should return only title, text and markdown properties', async () => { - const config = await createService().execute(); + const config = await createService().get(); expect(config.templates[0]).toEqual({ template: fixtures.id('template 1').toString(), commonProperties: [fixtures.commonPropertiesTitleId('template 1')], @@ -98,7 +98,7 @@ describe('GetAutomaticTranslationConfig', () => { }); it('should not include properties that no longer exist', async () => { - const config = await createService().execute(); + const config = await createService().get(); expect(config.templates[0].properties).toEqual([ fixtures.id('text property').toString(), fixtures.id('rich text').toString(), @@ -106,7 +106,7 @@ describe('GetAutomaticTranslationConfig', () => { }); it('should not include properties belonging to other templates', async () => { - const config = await createService().execute(); + const config = await createService().get(); expect(config.templates[1]).toEqual({ template: fixtures.id('template 2').toString(), commonProperties: [], @@ -115,12 +115,12 @@ describe('GetAutomaticTranslationConfig', () => { }); it('should return languages available filtered by the supported languages of automatic translation', async () => { - const config = await createService().execute(); + const config = await createService().get(); expect(config.languages).toEqual(['en', 'es']); }); it('should allow configuring only title without any properties', async () => { - const config = await createService().execute(); + const config = await createService().get(); expect(config.templates[2]).toEqual({ template: fixtures.id('template 3').toString(), commonProperties: [fixtures.commonPropertiesTitleId('template 3')], @@ -129,7 +129,7 @@ describe('GetAutomaticTranslationConfig', () => { }); it('should not include properties configurations belonging to an unexistent template', async () => { - const config = await createService().execute(); + const config = await createService().get(); expect(config.templates[3]).toBeUndefined(); }); }); diff --git a/app/api/externalIntegrations.v2/automaticTranslation/specs/RequestEntityTranslation.spec.ts b/app/api/externalIntegrations.v2/automaticTranslation/specs/RequestEntityTranslation.spec.ts new file mode 100644 index 0000000000..2b762cb80d --- /dev/null +++ b/app/api/externalIntegrations.v2/automaticTranslation/specs/RequestEntityTranslation.spec.ts @@ -0,0 +1,144 @@ +import { DefaultTransactionManager } from 'api/common.v2/database/data_source_defaults'; +import { + ATConfig, + ATTemplateConfig, +} from 'api/externalIntegrations.v2/automaticTranslation/model/ATConfig'; +import { TaskManager } from 'api/services/tasksmanager/TaskManager'; +import { DefaultTemplatesDataSource } from 'api/templates.v2/database/data_source_defaults'; +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { LanguageISO6391 } from 'shared/types/commonTypes'; +import { EntitySchema } from 'shared/types/entityType'; +import { ATTaskMessage, RequestEntityTranslation } from '../RequestEntityTranslation'; +import { ATConfigService } from '../services/GetAutomaticTranslationConfig'; +import { AJVEntityInputValidator } from '../infrastructure/EntityInputValidator'; +import { InvalidInputDataFormat } from '../errors/generateATErrors'; + +const factory = getFixturesFactory(); +const fixtures = { + templates: [ + factory.template('template1', [factory.property('text1'), factory.property('empty_text')]), + ], + entities: [ + ...factory.entityInMultipleLanguages(['en', 'es'], 'entity1', 'template1', { + text1: [{ value: 'original text1' }], + empty_text: [{ value: '' }], + }), + ], + settings: [ + { + languages: [ + { label: 'en', key: 'en' as LanguageISO6391, default: true }, + { label: 'es', key: 'es' as LanguageISO6391 }, + ], + }, + ], +}; + +// @ts-ignore +class TestATConfigService implements ATConfigService { + // eslint-disable-next-line class-methods-use-this + async get() { + return new ATConfig( + true, + ['es', 'en'], + [ + new ATTemplateConfig( + factory.idString('template1'), + [factory.idString('text1'), factory.idString('empty_text')], + [factory.commonPropertiesTitleId('template1')] + ), + ] + ); + } +} + +let taskManager: TaskManager; + +beforeEach(async () => { + await testingEnvironment.setUp(fixtures); + await testingEnvironment.setTenant('tenant'); + taskManager = new TaskManager({ + serviceName: RequestEntityTranslation.SERVICE_NAME, + }); + jest.spyOn(taskManager, 'startTask').mockImplementation(async () => ''); +}); + +afterAll(async () => { + await testingEnvironment.tearDown(); +}); + +describe('RequestEntityTranslation', () => { + it('should send a task in the automatic translation service queue', async () => { + const languageFromEntity = fixtures.entities.find(e => e.language === 'en') as EntitySchema; + languageFromEntity._id = languageFromEntity?._id?.toString(); + languageFromEntity.template = languageFromEntity?.template?.toString(); + + const requestEntityTranslation = new RequestEntityTranslation( + taskManager, + DefaultTemplatesDataSource(DefaultTransactionManager()), + // @ts-ignore + new TestATConfigService(), + new AJVEntityInputValidator() + ); + + await requestEntityTranslation.execute(languageFromEntity!); + + expect(taskManager.startTask).toHaveBeenCalledTimes(2); + + expect(taskManager.startTask).toHaveBeenCalledWith({ + params: { + key: ['tenant', 'entity1', 'title'], + text: 'entity1', + language_from: 'en', + languages_to: ['es'], + }, + }); + + expect(taskManager.startTask).toHaveBeenCalledWith({ + params: { + key: ['tenant', 'entity1', 'text1'], + text: 'original text1', + language_from: 'en', + languages_to: ['es'], + }, + }); + }); + + it('should do nothing if entity.language is not supported', async () => { + const entityWithNotSupportedLanguage = factory.entity( + 'entity2', + 'template1', + {}, + { language: 'pt' } + ); + entityWithNotSupportedLanguage._id = entityWithNotSupportedLanguage?._id?.toString(); + entityWithNotSupportedLanguage.template = entityWithNotSupportedLanguage?.template?.toString(); + + const requestEntityTranslation = new RequestEntityTranslation( + taskManager, + DefaultTemplatesDataSource(DefaultTransactionManager()), + // @ts-ignore + new TestATConfigService(), + new AJVEntityInputValidator() + ); + + await requestEntityTranslation.execute(entityWithNotSupportedLanguage); + expect(taskManager.startTask).not.toHaveBeenCalled(); + }); + + it('should validate input has proper shape at runtime', async () => { + const requestEntityTranslation = new RequestEntityTranslation( + taskManager, + DefaultTemplatesDataSource(DefaultTransactionManager()), + // @ts-ignore + new TestATConfigService(), + new AJVEntityInputValidator() + ); + + const invalidEntity = { invalid_prop: true }; + await expect(requestEntityTranslation.execute(invalidEntity)).rejects.toEqual( + new InvalidInputDataFormat('{"missingProperty":"_id"}') + ); + }); +}); diff --git a/app/api/services/informationextraction/InformationExtraction.ts b/app/api/services/informationextraction/InformationExtraction.ts index a15993f494..98677dc320 100644 --- a/app/api/services/informationextraction/InformationExtraction.ts +++ b/app/api/services/informationextraction/InformationExtraction.ts @@ -7,7 +7,7 @@ import _ from 'lodash'; import { ObjectId } from 'mongodb'; import { storage } from 'api/files'; -import { ResultsMessage, TaskManager } from 'api/services/tasksmanager/TaskManager'; +import { TaskManager } from 'api/services/tasksmanager/TaskManager'; import { IXSuggestionsModel } from 'api/suggestions/IXSuggestionsModel'; import { SegmentationModel } from 'api/services/pdfsegmentation/segmentationModel'; import { EnforcedWithId } from 'api/odm'; @@ -58,15 +58,33 @@ interface TaskParameters { metadata?: { [key: string]: string }; } +interface TaskMessage { + tenant: string; + task: TaskTypes; + params?: TaskParameters; +} + type ResultParameters = TaskParameters; +/* eslint-disable camelcase */ +interface ResultMessage

{ + tenant: string; + task: TaskTypes; + params?: P; + data_url?: string; + file_url?: string; + success?: boolean; + error_message?: string; +} +/* eslint-enable camelcase */ + interface InternalResultParameters { id: ObjectId; } -type IXResultsMessage = ResultsMessage; +type IXResultsMessage = ResultMessage; -type InternalIXResultsMessage = ResultsMessage; +type InternalIXResultsMessage = ResultMessage; interface CommonMaterialsData { xml_file_name: string; @@ -110,7 +128,7 @@ async function fetchCandidates(property: PropertySchema) { class InformationExtraction { static SERVICE_NAME = 'information_extraction'; - public taskManager: TaskManager; + public taskManager: TaskManager; static mock: any; @@ -150,9 +168,9 @@ class InformationExtraction { file: FileWithAggregation, _data: CommonMaterialsData ): MaterialsData => { - const language_iso = languages.get(file.language!, 'ISO639_1') || defaultTrainingLanguage; + const languageIso = languages.get(file.language!, 'ISO639_1') || defaultTrainingLanguage; - let data: MaterialsData = { ..._data, language_iso }; + let data: MaterialsData = { ..._data, language_iso: languageIso }; const noExtractedData = propertyTypeIsWithoutExtractedMetadata(propertyType); diff --git a/app/api/services/tasksmanager/TaskManager.ts b/app/api/services/tasksmanager/TaskManager.ts index e6dcf9166a..c426a71b9e 100644 --- a/app/api/services/tasksmanager/TaskManager.ts +++ b/app/api/services/tasksmanager/TaskManager.ts @@ -9,17 +9,17 @@ type DefaultTaskType = string; type DefaultMessageParameters = Record; -export type TaskMessage = { +export type TaskMessage = { tenant: string; - task: T; - params?: P; + task: DefaultTaskType; + params?: DefaultMessageParameters; }; /* eslint-disable camelcase */ -export interface ResultsMessage { +export interface ResultsMessage { tenant: string; - task: T; - params?: P; + task: DefaultTaskType; + params?: DefaultMessageParameters; data_url?: string; file_url?: string; success?: boolean; @@ -27,20 +27,16 @@ export interface ResultsMessage { +export interface Service { serviceName: string; - processResults?: (results: ResultsMessage) => Promise; + processResults?: (results: R) => Promise; processResultsMessageHiddenTime?: number; } -export class TaskManager< - T = DefaultTaskType, - S = DefaultMessageParameters, - R = DefaultMessageParameters, -> { +export class TaskManager { redisSMQ: RedisSMQ; - readonly service: Service; + readonly service: Service; readonly taskQueue: string; @@ -50,7 +46,7 @@ export class TaskManager< redisClient: RedisClient; - constructor(service: Service) { + constructor(service: Service) { this.service = service; this.taskQueue = `${config.ENVIRONMENT}_${service.serviceName}_tasks`; this.resultsQueue = `${config.ENVIRONMENT}_${service.serviceName}_results`; @@ -121,7 +117,7 @@ export class TaskManager< } } - async startTask(taskMessage: TaskMessage) { + async startTask(taskMessage: T) { if (!this.redisClient.connected) { throw new Error('Redis is not connected'); } @@ -133,7 +129,9 @@ export class TaskManager< } async stop() { - await this.repeater!.stop(); + if (this.repeater) { + await this.repeater.stop(); + } await this.redisClient.end(true); } } diff --git a/app/api/utils/testingEnvironment.ts b/app/api/utils/testingEnvironment.ts index 0368e697eb..3a61d20077 100644 --- a/app/api/utils/testingEnvironment.ts +++ b/app/api/utils/testingEnvironment.ts @@ -17,10 +17,10 @@ const testingEnvironment = { await this.setElastic(elasticIndex); }, - async setTenant(name = 'defaultDB') { + async setTenant(name?: string) { testingTenants.mockCurrentTenant({ - name: testingDB.dbName || name, - dbName: testingDB.dbName || name, + name: name || testingDB.dbName || 'defaultDB', + dbName: testingDB.dbName || name || 'defaultDB', indexName: 'index', }); await setupTestUploadedPaths(); diff --git a/app/shared/languagesList.ts b/app/shared/languagesList.ts index 023968f977..8bdbac915c 100644 --- a/app/shared/languagesList.ts +++ b/app/shared/languagesList.ts @@ -1359,4 +1359,6 @@ const language = (key: string, purpose: keyof (typeof elasticLanguages)[number] return elasticLanguages[key] ? elasticLanguages[key][purpose] : defaultValue; }; -export { elasticLanguages, availableLanguages, language }; +const availableLanguagesISO6391 = availableLanguages.map(l => l.key); + +export { elasticLanguages, availableLanguages, language, availableLanguagesISO6391 }; diff --git a/package.json b/package.json index f423f8f647..3fca7b341b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.186.0-rc3", + "version": "1.186.0-rc5", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react" @@ -336,7 +336,7 @@ "copy-webpack-plugin": "12.0.2", "css-loader": "^7.1.2", "css-minimizer-webpack-plugin": "^7.0.0", - "cypress": "13.14.2", + "cypress": "13.15.0", "cypress-axe": "^1.5.0", "cypress-plugin-snapshots": "^1.4.4", "cypress-real-events": "^1.13.0", diff --git a/yarn.lock b/yarn.lock index 51bff6df8b..e1a54de3ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1839,10 +1839,10 @@ resolved "https://registry.yarnpkg.com/@cypress/react18/-/react18-2.0.1.tgz#7bc6d9b557fd6a516a690996946092b12abbd3f9" integrity sha512-T/bhFEvVDIu0lDOKXbEQqVEmmANKWc/pyFDyDoJw3OndRYv9QVEJSsE/VNXIaOQLDjWvQkKBOwd0lLe1hWF/Zg== -"@cypress/request@^3.0.1": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@cypress/request/-/request-3.0.1.tgz#72d7d5425236a2413bd3d8bb66d02d9dc3168960" - integrity sha512-TWivJlJi8ZDx2wGOw1dbLuHJKUYX7bWySw377nlnGOW3hP9/MUKIsEdXT/YngWxVdgNCHRBmFlBipE+5/2ZZlQ== +"@cypress/request@^3.0.4": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@cypress/request/-/request-3.0.5.tgz#d893a6e68ce2636c085fcd8d7283c3186499ba63" + integrity sha512-v+XHd9XmWbufxF1/bTaVm2yhbxY+TB4YtWRqF2zaXBlDNMkls34KiATz0AVDLavL3iB6bQk9/7n3oY1EoLSWGA== dependencies: aws-sign2 "~0.7.0" aws4 "^1.8.0" @@ -1850,14 +1850,14 @@ combined-stream "~1.0.6" extend "~3.0.2" forever-agent "~0.6.1" - form-data "~2.3.2" - http-signature "~1.3.6" + form-data "~4.0.0" + http-signature "~1.4.0" is-typedarray "~1.0.0" isstream "~0.1.2" json-stringify-safe "~5.0.1" mime-types "~2.1.19" performance-now "^2.1.0" - qs "6.10.4" + qs "6.13.0" safe-buffer "^5.1.2" tough-cookie "^4.1.3" tunnel-agent "^0.6.0" @@ -7385,7 +7385,7 @@ colors@0.5.x: version "0.5.1" resolved "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz" -combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: +combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -8001,12 +8001,12 @@ cypress-real-events@^1.13.0: resolved "https://registry.yarnpkg.com/cypress-real-events/-/cypress-real-events-1.13.0.tgz#6b7cd32dcac172db1493608f97a2576c7d0bd5af" integrity sha512-LoejtK+dyZ1jaT8wGT5oASTPfsNV8/ClRp99ruN60oPj8cBJYod80iJDyNwfPAu4GCxTXOhhAv9FO65Hpwt6Hg== -cypress@13.14.2: - version "13.14.2" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.14.2.tgz#4237eb7b26de2baeaa1f01e585f965d88fca7f39" - integrity sha512-lsiQrN17vHMB2fnvxIrKLAjOr9bPwsNbPZNrWf99s4u+DVmCY6U+w7O3GGG9FvP4EUVYaDu+guWeNLiUzBrqvA== +cypress@13.15.0: + version "13.15.0" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-13.15.0.tgz#5eca5387ef34b2e611cfa291967c69c2cd39381d" + integrity sha512-53aO7PwOfi604qzOkCSzNlWquCynLlKE/rmmpSPcziRH6LNfaDUAklQT6WJIsD8ywxlIy+uVZsnTMCCQVd2kTw== dependencies: - "@cypress/request" "^3.0.1" + "@cypress/request" "^3.0.4" "@cypress/xvfb" "^1.2.4" "@types/sinonjs__fake-timers" "8.1.1" "@types/sizzle" "^2.3.2" @@ -10316,7 +10316,7 @@ fork-ts-checker-webpack-plugin@^8.0.0: semver "^7.3.5" tapable "^2.2.1" -form-data@^4.0.0: +form-data@^4.0.0, form-data@~4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== @@ -10325,15 +10325,6 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.6" - mime-types "^2.1.12" - formatcoords@^1.1.3: version "1.1.3" resolved "https://registry.npmjs.org/formatcoords/-/formatcoords-1.1.3.tgz" @@ -11149,14 +11140,14 @@ http-proxy-agent@^5.0.0: agent-base "6" debug "4" -http-signature@~1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.6.tgz#cb6fbfdf86d1c974f343be94e87f7fc128662cf9" - integrity sha512-3adrsD6zqo4GsTqtO7FyrejHNv+NgiIfAfv68+jVlFmSr9OGy7zrxONceFRLKvnnZA5jbxQBX1u9PpB6Wi32Gw== +http-signature@~1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.4.0.tgz#dee5a9ba2bf49416abc544abd6d967f6a94c8c3f" + integrity sha512-G5akfn7eKbpDN+8nPS/cb57YeA1jLTVxjpCj7tmm3QKPdyDy7T+qSC40e9ptydSWvkwjSXw1VbkpyEm39ukeAg== dependencies: assert-plus "^1.0.0" jsprim "^2.0.2" - sshpk "^1.14.1" + sshpk "^1.18.0" http2-wrapper@^1.0.0-beta.5.2: version "1.0.3" @@ -15116,13 +15107,6 @@ qrcode.react@^4.0.1: resolved "https://registry.yarnpkg.com/qrcode.react/-/qrcode.react-4.0.1.tgz#1caf1d3f45bf1b6d9cf800cb0f0d671f6a89e68f" integrity sha512-Lpj0tPBn561WiQ3QQWXbkx8xTtB8BZkJeMZWLJIL8iaPBCoWzW1IpCeU3gY5MDqsb0+efCvEGkl9O0naP64crA== -qs@6.10.4: - version "6.10.4" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.4.tgz#6a3003755add91c0ec9eacdc5f878b034e73f9e7" - integrity sha512-OQiU+C+Ds5qiH91qh/mg0w+8nwQuLjM4F4M/PbmhDOoYehPh+Fb0bDjtR1sOvy7YKxvj28Y/M0PhP5uVX0kB+g== - dependencies: - side-channel "^1.0.4" - qs@6.11.0: version "6.11.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" @@ -16712,10 +16696,10 @@ spy-on-component@^1.1.0: resolved "https://registry.npmjs.org/spy-on-component/-/spy-on-component-1.1.3.tgz" integrity sha512-a7jgnoBSdkcDWIQQwtEgUq4etajwG6+wGIjfC9ARUKwKOdHxJd+utgHTgLn81ETizpsw4xddUS3W8VePedtaIQ== -sshpk@^1.14.1: - version "1.17.0" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" - integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== +sshpk@^1.18.0: + version "1.18.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.18.0.tgz#1663e55cddf4d688b86a46b77f0d5fe363aba028" + integrity sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ== dependencies: asn1 "~0.2.3" assert-plus "^1.0.0"