WIP, changes to actually make it work on the dummy service (#7283)

* WIP, changes to actually make it work on the dummy service

* Added option to deactivate the service

* Removed unused includes

* Included ATConfig injection

* Fixed typing errors

* Removed access to async context before tenant

* Delete ATSemanticConfig.json

* Upped version.

* Added environmnent to test.

---------

Co-authored-by: RafaPolit <rafaelpolit@gmail.com>

app/api/eventListeners.ts

@@ -1,10 +1,12 @@
 import { EventsBus } from './eventsbus';
+import { AutomaticTranslationFactory } from './externalIntegrations.v2/automaticTranslation/AutomaticTranslationFactory';
 import { registerEventListeners as registerSegmentationListeners } from './services/pdfsegmentation/eventListeners';
 import { Suggestions } from './suggestions/suggestions';
 
 const registerEventListeners = (eventsBus: EventsBus) => {
   Suggestions.registerEventListeners(eventsBus);
   registerSegmentationListeners(eventsBus);
+  AutomaticTranslationFactory.defaultATEntityCreationListener(eventsBus).start();
 };
 
 export { registerEventListeners };

app/api/externalIntegrations.v2/automaticTranslation/AutomaticTranslationFactory.ts

@@ -1,10 +1,12 @@
 import { DefaultTransactionManager } from 'api/common.v2/database/data_source_defaults';
 import { getConnection } from 'api/common.v2/database/getConnectionForCurrentTenant';
 import { DefaultEntitiesDataSource } from 'api/entities.v2/database/data_source_defaults';
+import { EventsBus } from 'api/eventsbus';
 import { TaskManager } from 'api/services/tasksmanager/TaskManager';
 import { DefaultSettingsDataSource } from 'api/settings.v2/database/data_source_defaults';
 import { DefaultTemplatesDataSource } from 'api/templates.v2/database/data_source_defaults';
 import { MongoTemplatesDataSource } from 'api/templates.v2/database/MongoTemplatesDataSource';
+import { ATEntityCreationListener } from './adapters/driving/ATEntityCreationListener';
 import { GenerateAutomaticTranslationsCofig } from './GenerateAutomaticTranslationConfig';
 import { AJVATConfigValidator } from './infrastructure/AJVATConfigValidator';
 import { AJVTranslationResultValidator } from './infrastructure/AJVTranslationResultValidator';
@@ -53,6 +55,10 @@ const AutomaticTranslationFactory = {
       new AJVEntityInputValidator()
     );
   },
+
+  defaultATEntityCreationListener(eventsBus: EventsBus) {
+    return new ATEntityCreationListener(eventsBus);
+  },
 };
 
 export { AutomaticTranslationFactory };

app/api/externalIntegrations.v2/automaticTranslation/RequestEntityTranslation.ts

@@ -7,16 +7,14 @@ import { ATConfigService } from './services/GetAutomaticTranslationConfig';
 import { InvalidInputDataFormat } from './errors/generateATErrors';
 
 export type ATTaskMessage = {
-  params: {
-    key: string[];
-    text: string;
-    language_from: string;
-    languages_to: string[];
-  };
+  key: string[];
+  text: string;
+  language_from: string;
+  languages_to: string[];
 };
 
 export class RequestEntityTranslation {
-  static SERVICE_NAME = 'AutomaticTranslation';
+  static SERVICE_NAME = 'translations';
 
   private taskManager: TaskManager<ATTaskMessage>;
 
@@ -74,12 +72,10 @@ export class RequestEntityTranslation {
       }
 
       await this.taskManager.startTask({
-        params: {
-          key: [getTenant().name, entity.sharedId, commonPropName],
-          text: entity[commonPropName],
-          language_from: languageFrom,
-          languages_to: languagesTo,
-        },
+        key: [getTenant().name, entity.sharedId, commonPropId.toString()],
+        text: entity[commonPropName],
+        language_from: languageFrom,
+        languages_to: languagesTo,
       });
     });
     atTemplateConfig?.properties.forEach(async propId => {
@@ -94,12 +90,10 @@ export class RequestEntityTranslation {
 
       if (entity.metadata[propName]?.[0].value) {
         await this.taskManager.startTask({
-          params: {
-            key: [getTenant().name, entity.sharedId, propName],
-            text: entity.metadata[propName][0].value,
-            language_from: languageFrom,
-            languages_to: languagesTo,
-          },
+          key: [getTenant().name, entity.sharedId, propId],
+          text: entity.metadata[propName][0].value,
+          language_from: languageFrom,
+          languages_to: languagesTo,
         });
       }
     });

app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/ATEntityCreationListener.ts

@@ -1,25 +1,34 @@
 import { EntityCreatedEvent } from 'api/entities/events/EntityCreatedEvent';
 import { EventsBus } from 'api/eventsbus';
-import { RequestEntityTranslation } from '../../RequestEntityTranslation';
+import { permissionsContext } from 'api/permissions/permissionsContext';
+import { AutomaticTranslationFactory } from '../../AutomaticTranslationFactory';
 
 export class ATEntityCreationListener {
-  private requestEntityTranslation: RequestEntityTranslation;
-
   private eventBus: EventsBus;
 
-  constructor(requestEntityTranslation: RequestEntityTranslation, eventBus: EventsBus) {
-    this.requestEntityTranslation = requestEntityTranslation;
+  private ATFactory: typeof AutomaticTranslationFactory;
+
+  constructor(
+    eventBus: EventsBus,
+    ATFactory: typeof AutomaticTranslationFactory = AutomaticTranslationFactory
+  ) {
     this.eventBus = eventBus;
+    this.ATFactory = ATFactory;
   }
 
   start() {
     this.eventBus.on(EntityCreatedEvent, async event => {
-      const entityFrom = event.entities.find(e => e.language === event.targetLanguageKey) || {};
+      const { active } = await this.ATFactory.defaultATConfigService().get();
+
+      if (active) {
+        permissionsContext.setCommandContext();
+        const entityFrom = event.entities.find(e => e.language === event.targetLanguageKey) || {};
 
-      entityFrom._id = entityFrom._id?.toString();
-      entityFrom.template = entityFrom.template?.toString();
+        entityFrom._id = entityFrom._id?.toString();
+        entityFrom.template = entityFrom.template?.toString();
 
-      await this.requestEntityTranslation.execute(entityFrom);
+        await this.ATFactory.defaultRequestEntityTranslation().execute(entityFrom);
+      }
     });
   }
 }

app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/ATServiceListener.ts

@@ -1,12 +1,13 @@
 import { tenants } from 'api/tenants';
 import { TaskManager } from 'api/services/tasksmanager/TaskManager';
+import { permissionsContext } from 'api/permissions/permissionsContext';
 import { ATTranslationResultValidator } from '../../contracts/ATTranslationResultValidator';
 import { AJVTranslationResultValidator } from '../../infrastructure/AJVTranslationResultValidator';
 import { InvalidATServerResponse } from '../../errors/generateATErrors';
 import { AutomaticTranslationFactory } from '../../AutomaticTranslationFactory';
 
 export class ATServiceListener {
-  static SERVICE_NAME = 'AutomaticTranslation';
+  static SERVICE_NAME = 'translations';
 
   private taskManager: TaskManager;
 
@@ -20,6 +21,7 @@ export class ATServiceListener {
         }
 
         await tenants.run(async () => {
+          permissionsContext.setCommandContext();
           await ATFactory.defaultSaveEntityTranslations().execute(result);
         }, result.key[0]);
       },

app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/specs/ATEntityCreatedListener.spec.ts

@@ -4,25 +4,52 @@ import { RequestEntityTranslation } from 'api/externalIntegrations.v2/automaticT
 import { testingEnvironment } from 'api/utils/testingEnvironment';
 import { AutomaticTranslationFactory } from 'api/externalIntegrations.v2/automaticTranslation/AutomaticTranslationFactory';
 import { getFixturesFactory } from 'api/utils/fixturesFactory';
+import { tenants } from 'api/tenants';
+import { DefaultTransactionManager } from 'api/common.v2/database/data_source_defaults';
+import { ATConfigService } from 'api/externalIntegrations.v2/automaticTranslation/services/GetAutomaticTranslationConfig';
+import { DefaultSettingsDataSource } from 'api/settings.v2/database/data_source_defaults';
+import { MongoATConfigDataSource } from 'api/externalIntegrations.v2/automaticTranslation/infrastructure/MongoATConfigDataSource';
+import { getConnection } from 'api/common.v2/database/getConnectionForCurrentTenant';
+import { DefaultTemplatesDataSource } from 'api/templates.v2/database/data_source_defaults';
+import { ATExternalAPI } from 'api/externalIntegrations.v2/automaticTranslation/infrastructure/ATExternalAPI';
 import { ATEntityCreationListener } from '../ATEntityCreationListener';
 
 const factory = getFixturesFactory();
 
+const prepareATFactory = (executeSpy: jest.Mock<any, any, any>) => {
+  // @ts-ignore
+  const ATFactory: typeof AutomaticTranslationFactory = {
+    defaultATConfigService() {
+      const transactionManager = DefaultTransactionManager();
+      return new ATConfigService(
+        DefaultSettingsDataSource(transactionManager),
+        new MongoATConfigDataSource(getConnection(), transactionManager),
+        DefaultTemplatesDataSource(transactionManager),
+        new ATExternalAPI()
+      );
+    },
+    defaultRequestEntityTranslation() {
+      return { execute: executeSpy } as unknown as RequestEntityTranslation;
+    },
+  };
+
+  return ATFactory;
+};
+
 describe('ATEntityCreationListener', () => {
   let listener: ATEntityCreationListener;
   const eventBus: EventsBus = new EventsBus();
-  let requestEntityTranslation: RequestEntityTranslation;
+  let executeSpy: jest.Mock<any, any, any>;
 
   beforeEach(async () => {
     await testingEnvironment.setUp({
-      settings: [{ features: { automaticTranslation: { active: true } } }],
+      settings: [{ features: { automaticTranslation: { active: false } } }],
     });
     await testingEnvironment.setTenant('tenant');
 
-    requestEntityTranslation = AutomaticTranslationFactory.defaultRequestEntityTranslation();
-    jest.spyOn(requestEntityTranslation, 'execute');
+    executeSpy = jest.fn();
 
-    listener = new ATEntityCreationListener(requestEntityTranslation, eventBus);
+    listener = new ATEntityCreationListener(eventBus, prepareATFactory(executeSpy));
     listener.start();
   });
 
@@ -31,16 +58,34 @@ describe('ATEntityCreationListener', () => {
   });
 
   describe('Request entity translation', () => {
+    it('should not request translations if feature flag is off', async () => {
+      const entityCreationEvent = new EntityCreatedEvent({
+        entities: [{ sharedId: 'entity 1' }],
+        targetLanguageKey: 'en',
+      });
+
+      await tenants.run(async () => {
+        await eventBus.emit(entityCreationEvent);
+      }, 'tenant');
+
+      expect(executeSpy).not.toHaveBeenCalled();
+    });
+
     it('should call RequestEntityTranslation on receiving entity creation event', async () => {
+      await testingEnvironment.setFixtures({
+        settings: [{ features: { automaticTranslation: { active: true } } }],
+      });
       const entityEn = factory.entity('entity1', 'template1', {}, { language: 'en' });
       const entityCreationEvent = new EntityCreatedEvent({
         entities: [factory.entity('entity1', 'template1', {}, { language: 'es' }), entityEn],
         targetLanguageKey: 'en',
       });
 
-      await eventBus.emit(entityCreationEvent);
+      await tenants.run(async () => {
+        await eventBus.emit(entityCreationEvent);
+      }, 'tenant');
 
-      expect(requestEntityTranslation.execute).toHaveBeenCalledWith(entityEn);
+      expect(executeSpy).toHaveBeenCalledWith(entityEn);
     });
   });
 });

app/api/externalIntegrations.v2/automaticTranslation/adapters/driving/specs/ATServiceListener.spec.ts

@@ -5,16 +5,12 @@ import RedisSMQ from 'rsmq';
 import waitForExpect from 'wait-for-expect';
 import { AutomaticTranslationFactory } from 'api/externalIntegrations.v2/automaticTranslation/AutomaticTranslationFactory';
 import { testingEnvironment } from 'api/utils/testingEnvironment';
-import { GenerateAutomaticTranslationsCofig } from 'api/externalIntegrations.v2/automaticTranslation/GenerateAutomaticTranslationConfig';
 import { SaveEntityTranslations } from 'api/externalIntegrations.v2/automaticTranslation/SaveEntityTranslations';
 import { ATServiceListener } from '../ATServiceListener';
 
 const prepareATFactory = (executeSpy: jest.Mock<any, any, any>) => {
   // @ts-ignore
   const ATFactory: typeof AutomaticTranslationFactory = {
-    defaultGenerateATConfig() {
-      return {} as GenerateAutomaticTranslationsCofig;
-    },
     defaultSaveEntityTranslations() {
       return { execute: executeSpy } as unknown as SaveEntityTranslations;
     },
@@ -57,7 +53,7 @@ describe('ATServiceListener', () => {
       await redisSMQ.createQueueAsync({ qname: queueName });
     };
 
-    await recreateQueue('AutomaticTranslation_results').catch(error => {
+    await recreateQueue(`${ATServiceListener.SERVICE_NAME}_results`).catch(error => {
       throw error;
     });
 
@@ -85,7 +81,7 @@ describe('ATServiceListener', () => {
       executeSpy.mockClear();
 
       await redisSMQ.sendMessageAsync({
-        qname: 'AutomaticTranslation_results',
+        qname: `${config.ENVIRONMENT}_${ATServiceListener.SERVICE_NAME}_results`,
         message: JSON.stringify(message),
       });
 

app/api/externalIntegrations.v2/automaticTranslation/specs/RequestEntityTranslation.spec.ts

@@ -87,21 +87,17 @@ describe('RequestEntityTranslation', () => {
     expect(taskManager.startTask).toHaveBeenCalledTimes(2);
 
     expect(taskManager.startTask).toHaveBeenCalledWith({
-      params: {
-        key: ['tenant', 'entity1', 'title'],
-        text: 'entity1',
-        language_from: 'en',
-        languages_to: ['es'],
-      },
+      key: ['tenant', 'entity1', factory.commonPropertiesTitleId('template1').toString()],
+      text: 'entity1',
+      language_from: 'en',
+      languages_to: ['es'],
     });
 
     expect(taskManager.startTask).toHaveBeenCalledWith({
-      params: {
-        key: ['tenant', 'entity1', 'text1'],
-        text: 'original text1',
-        language_from: 'en',
-        languages_to: ['es'],
-      },
+      key: ['tenant', 'entity1', factory.property('text1')._id?.toString()],
+      text: 'original text1',
+      language_from: 'en',
+      languages_to: ['es'],
     });
   });
 

app/api/utils/handleError.js

@@ -69,6 +69,7 @@ const prettifyError = (error, { req = {}, uncaught = false } = {}) => {
   }
 
   if (error instanceof Ajv.ValidationError) {
+    console.log(util.inspect(error));
     result = { code: 422, message: error.message, validations: error.errors, logLevel: 'debug' };
   }
 

app/worker.ts

@@ -13,6 +13,7 @@ import { InformationExtraction } from 'api/services/informationextraction/Inform
 import { setupWorkerSockets } from 'api/socketio/setupSockets';
 import { ConvertToPdfWorker } from 'api/services/convertToPDF/ConvertToPdfWorker';
 import { handleError } from './api/utils/handleError.js';
+import { ATServiceListener } from 'api/externalIntegrations.v2/automaticTranslation/adapters/driving/ATServiceListener';
 
 let dbAuth = {};
 
@@ -42,6 +43,7 @@ DB.connect(config.DBHOST, dbAuth)
 
       console.info('==> 📡 starting external services...');
       ocrManager.start();
+      new ATServiceListener().start();
       new InformationExtraction().start();
 
       new ConvertToPdfWorker().start();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "uwazi",
-  "version": "1.187.0-rc1",
+  "version": "1.187.0-rc2",
   "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.",
   "keywords": [
     "react"