Translate attributes of one produt at once. Refactor translation process

macopedia · Jun 26, 2023 · 1993328 · 1993328
1 parent 085a68b
commit 1993328
Show file tree

Hide file tree

Showing 11 changed files with 229 additions and 23 deletions.
diff --git a/...anslator/Connector/Job/JobParameters/ConstraintCollectionProvider/TranslateAttributes.php b/...anslator/Connector/Job/JobParameters/ConstraintCollectionProvider/TranslateAttributes.php
@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Macopedia\OpenAiTranslator\Connector\Job\JobParameters\ConstraintCollectionProvider;
+
+use Akeneo\Tool\Component\Batch\Job\JobInterface;
+use Akeneo\Tool\Component\Batch\Job\JobParameters\ConstraintCollectionProviderInterface;
+use Symfony\Component\Validator\Constraints\All;
+use Symfony\Component\Validator\Constraints\Collection;
+use Symfony\Component\Validator\Constraints\NotNull;
+use Symfony\Component\Validator\Constraints\Type;
+
+class TranslateAttributes implements ConstraintCollectionProviderInterface
+{
+    /**
+     * @param array<string> $supportedJobNames
+     */
+    public function __construct(
+        private array $supportedJobNames,
+    ) {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getConstraintCollection(): Collection
+    {
+        return new Collection(
+            [
+                'fields' => [
+                    'filters' => new NotNull(),
+                    'actions' => new NotNull(),
+                    'realTimeVersioning' => new Type('bool'),
+                    'users_to_notify' => [
+                        new Type('array'),
+                        new All(new Type('string')),
+                    ],
+                    'is_user_authenticated' => new Type('bool')
+                ]
+            ]
+        );
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function supports(JobInterface $job): bool
+    {
+        return in_array($job->getName(), $this->supportedJobNames);
+    }
+}
diff --git a/...OpenAiTranslator/Connector/Job/JobParameters/DefaultValueProvider/TranslateAttributes.php b/...OpenAiTranslator/Connector/Job/JobParameters/DefaultValueProvider/TranslateAttributes.php
@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Macopedia\OpenAiTranslator\Connector\Job\JobParameters\DefaultValueProvider;
+
+use Akeneo\Tool\Component\Batch\Job\JobInterface;
+use Akeneo\Tool\Component\Batch\Job\JobParameters\DefaultValuesProviderInterface;
+
+class TranslateAttributes implements DefaultValuesProviderInterface
+{
+    /**
+     * @param array<string> $supportedJobNames
+     */
+    public function __construct(
+        private array $supportedJobNames
+    ) {
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDefaultValues(): array
+    {
+        return [
+            'filters' => [],
+            'actions' => [],
+            'realTimeVersioning' => true,
+            'users_to_notify' => [],
+            'is_user_authenticated' => false
+        ];
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function supports(JobInterface $job): bool
+    {
+        return in_array($job->getName(), $this->supportedJobNames);
+    }
+}
diff --git a/src/Macopedia/OpenAiTranslator/Connector/Processor/MassEdit/TranslateAttributesProcessor.php b/src/Macopedia/OpenAiTranslator/Connector/Processor/MassEdit/TranslateAttributesProcessor.php
@@ -45,6 +45,8 @@ public function process(mixed $item): ProductInterface|ProductModelInterface
      */
     private function translateAttributes(mixed $product, array $action): ProductInterface|ProductModelInterface
     {
-        return $this->translateAttributesService->translateAttributes($product, $action);
+        return $this->translateAttributesService
+            ->setStepExecution($this->stepExecution)
+            ->translateAttributes($product, $action);
     }
 }
diff --git a/src/Macopedia/OpenAiTranslator/Connector/Tasklet/ValidateOpenAiKeyTasklet.php b/src/Macopedia/OpenAiTranslator/Connector/Tasklet/ValidateOpenAiKeyTasklet.php
@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Macopedia\OpenAiTranslator\Connector\Tasklet;
+
+use Akeneo\Tool\Component\Batch\Job\BatchStatus;
+use Akeneo\Tool\Component\Batch\Model\StepExecution;
+use Akeneo\Tool\Component\Connector\Step\TaskletInterface;
+use Exception;
+
+class ValidateOpenAiKeyTasklet implements TaskletInterface
+{
+    private StepExecution $stepExecution;
+
+    public function __construct(
+        private ?string $openAiKey
+    ) {
+    }
+
+    public function execute(): void
+    {
+        if (empty($this->openAiKey)) {
+            $this->stepExecution->addFailureException(new Exception('OpenAI key is not set'));
+            $this->stepExecution->setStatus(new BatchStatus(BatchStatus::FAILED));
+        }
+    }
+
+    public function setStepExecution(StepExecution $stepExecution): void
+    {
+        $this->stepExecution = $stepExecution;
+    }
+}
diff --git a/src/Macopedia/OpenAiTranslator/DependencyInjection/MacopediaTranslatorExtension.php b/src/Macopedia/OpenAiTranslator/DependencyInjection/MacopediaTranslatorExtension.php
@@ -22,7 +22,8 @@ final class MacopediaTranslatorExtension extends Extension
     public function load(array $configs, ContainerBuilder $container): void
     {
         $loader = new YamlFileLoader($container, new FileLocator(__DIR__ . '/../Resources/config'));
-        $loader->load('connector.yml');
         $loader->load('services.yml');
+        $loader->load('connector.yml');
+        $loader->load('repositories.yml');
     }
 }
diff --git a/src/Macopedia/OpenAiTranslator/Repository/AttributeRepository.php b/src/Macopedia/OpenAiTranslator/Repository/AttributeRepository.php
@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Macopedia\OpenAiTranslator\Repository;
+
+use Akeneo\Pim\Structure\Bundle\Doctrine\ORM\Repository\AttributeRepository as BaseAttributeRepository;
+
+class AttributeRepository extends BaseAttributeRepository
+{
+    public function getAttributesByCodes(array $codes): array
+    {
+        return $this->_em->createQueryBuilder()
+            ->select('att')
+            ->from($this->_entityName, 'att', 'att.code')
+            ->where('att.code IN (:codes)')->setParameter('codes', $codes)
+            ->getQuery()
+            ->getResult();
+    }
+}
diff --git a/src/Macopedia/OpenAiTranslator/Resources/config/connector.yml b/src/Macopedia/OpenAiTranslator/Resources/config/connector.yml
@@ -1,3 +1,5 @@
+parameters:
+  maco.open_ai_translator.job: 'update_product_translations'
 services:
   Macopedia\OpenAiTranslator\Connector\Processor\MassEdit\TranslateAttributesProcessor:
     arguments:
@@ -6,34 +8,48 @@ services:
   macopedia.job.update_product_translations:
     class: '%pim_connector.job.simple_job.class%'
     arguments:
-      - 'update_product_translations'
+      - '%maco.open_ai_translator.job%'
       - '@event_dispatcher'
       - '@akeneo_batch.job_repository'
-      - [ '@macopedia.step.update_product_translations.mass_edit' ]
+      -
+          - '@macopedia.step.validate_openai_key_tasklet'
+          - '@macopedia.step.update_product_translations.mass_edit'
     tags:
       - { name: akeneo_batch.job, connector: 'Macopedia OpenAi Connector', type: '%pim_enrich.job.mass_edit_type%' }
 
   macopedia.job.default_values_provider.translate_product:
     class: Akeneo\Pim\Enrichment\Component\Product\Connector\Job\JobParameters\DefaultValueProvider\ProductMassEdit
     arguments:
-      - [ 'update_product_translations' ]
+      - [ '%maco.open_ai_translator.job%' ]
     tags:
       - { name: akeneo_batch.job.job_parameters.default_values_provider }
 
   macopedia.job.constraint_collection_provider.translate_product:
     class: 'Akeneo\Pim\Enrichment\Component\Product\Connector\Job\JobParameters\ConstraintCollectionProvider\ProductMassEdit'
     arguments:
-      - [ 'update_product_translations' ]
+      - [ '%maco.open_ai_translator.job%' ]
     tags:
       - { name: akeneo_batch.job.job_parameters.constraint_collection_provider }
 
   macopedia.step.update_product_translations.mass_edit:
     class: '%pim_connector.step.item_step.class%'
     arguments:
-      - 'perform'
+      - 'translate'
       - '@event_dispatcher'
       - '@akeneo_batch.job_repository'
       - '@pim_enrich.reader.database.product_and_product_model'
       - '@Macopedia\OpenAiTranslator\Connector\Processor\MassEdit\TranslateAttributesProcessor'
       - '@pim_enrich.writer.database.product_and_product_model_writer'
-      - '%pim_job_product_batch_size%'
+      - '%pim_job_product_batch_size%'
+
+  Macopedia\OpenAiTranslator\Connector\Tasklet\ValidateOpenAiKeyTasklet:
+    arguments:
+      - '%open_ai_key%'
+
+  macopedia.step.validate_openai_key_tasklet:
+    class: '%pim_connector.step.tasklet.class%'
+    arguments:
+      - 'validate_openai_key'
+      - '@event_dispatcher'
+      - '@akeneo_batch.job_repository'
+      - '@Macopedia\OpenAiTranslator\Connector\Tasklet\ValidateOpenAiKeyTasklet'
diff --git a/src/Macopedia/OpenAiTranslator/Resources/config/repositories.yml b/src/Macopedia/OpenAiTranslator/Resources/config/repositories.yml
@@ -0,0 +1,6 @@
+services:
+  Macopedia\OpenAiTranslator\Repository\AttributeRepository:
+    factory: [ '@doctrine.orm.entity_manager', 'getRepository' ]
+    arguments: [ '%pim_catalog.entity.attribute.class%' ]
+    tags:
+      - { name: 'pim_repository' }
diff --git a/src/Macopedia/OpenAiTranslator/Resources/config/services.yml b/src/Macopedia/OpenAiTranslator/Resources/config/services.yml
@@ -14,6 +14,6 @@ services:
   Macopedia\OpenAiTranslator\Service\TranslateAttributesService:
     arguments:
       - '@Macopedia\OpenAiTranslator\Translator\OpenAiTranslator'
-      - '@pim_catalog.repository.attribute'
+      - '@Macopedia\OpenAiTranslator\Repository\AttributeRepository'
       - '@pim_catalog.entity_with_family_variant.check_attribute_editable'
       - '@pim_catalog.updater.property_setter'
diff --git a/src/Macopedia/OpenAiTranslator/Service/TranslateAttributesService.php b/src/Macopedia/OpenAiTranslator/Service/TranslateAttributesService.php
@@ -7,16 +7,22 @@
 use Akeneo\Pim\Enrichment\Component\Product\EntityWithFamilyVariant\CheckAttributeEditable;
 use Akeneo\Pim\Enrichment\Component\Product\Model\ProductInterface;
 use Akeneo\Pim\Enrichment\Component\Product\Model\ProductModelInterface;
-use Akeneo\Pim\Enrichment\Component\Product\Value\ScalarValue;
 use Akeneo\Pim\Structure\Component\AttributeTypes;
 use Akeneo\Pim\Structure\Component\Repository\AttributeRepositoryInterface;
+use Akeneo\Tool\Component\Batch\Item\DataInvalidItem;
+use Akeneo\Tool\Component\Batch\Job\BatchStatus;
+use Akeneo\Tool\Component\Batch\Job\ExitStatus;
+use Akeneo\Tool\Component\Batch\Model\StepExecution;
 use Akeneo\Tool\Component\StorageUtils\Updater\PropertySetterInterface;
 use Macopedia\OpenAiTranslator\Translator\Language;
 use Macopedia\OpenAiTranslator\Translator\TranslatorInterface;
+use Macopedia\OpenAiTranslator\Exception\InvalidOpenAiResponseException;
 use Webmozart\Assert\Assert;
 
 class TranslateAttributesService
 {
+    private StepExecution $stepExecution;
+
     public function __construct(
         private TranslatorInterface $translator,
         private AttributeRepositoryInterface $attributeRepository,
@@ -25,21 +31,38 @@ public function __construct(
     ) {
     }
 
+    public function setStepExecution(StepExecution $stepExecution): self
+    {
+        $this->stepExecution = $stepExecution;
+
+        return $this;
+    }
+
     /**
      * @param ProductInterface|ProductModelInterface $product
      * @param array<string, string|array<int, string>> $action
      */
     public function translateAttributes(mixed $product, array $action): ProductInterface|ProductModelInterface
     {
         [$sourceScope, $targetScope, $sourceLocaleAkeneo, $targetLocaleAkeneo, $targetLocale, $attributesToTranslate] = $this->extractVariables($action);
+        $translations = [];
+
+        $attributes = $this->attributeRepository->getAttributesByCodes($attributesToTranslate);
+        $summary = [];
+        $scopes = [];
 
-        foreach ($attributesToTranslate as $attributeCode) {
-            $attribute = $this->attributeRepository->findOneByIdentifier($attributeCode);
+        foreach ($attributes as $attribute) {
             if (!$this->checkAttributeEditable->isEditable($product, $attribute)) {
+                $this->stepExecution->addWarning('Attribute is not editable', [], new DataInvalidItem($attribute));
+                $this->stepExecution->incrementSummaryInfo('skip');
+                $this->stepExecution->incrementProcessedItems();
                 continue;
             }
 
             if (!($attribute->getType() === AttributeTypes::TEXT || $attribute->getType() === AttributeTypes::TEXTAREA)) {
+                $this->stepExecution->addWarning('Attribute is not text', [], new DataInvalidItem($attribute));
+                $this->stepExecution->incrementSummaryInfo('skip');
+                $this->stepExecution->incrementProcessedItems();
                 continue;
             }
 
@@ -48,26 +71,38 @@ public function translateAttributes(mixed $product, array $action): ProductInter
                 $targetScope = null;
             }
 
+            $attributeCode = $attribute->getCode();
             $attributeValue = $product->getValue($attributeCode, $sourceLocaleAkeneo, $sourceScope);
             if ($attributeValue === null) {
+                $this->stepExecution->addWarning('Attribute value is empty', [], new DataInvalidItem($attribute));
+                $this->stepExecution->incrementSummaryInfo('skip');
+                $this->stepExecution->incrementProcessedItems();
                 continue;
             }
 
-            $translatedText = $this->translator->translate(
-                $attributeValue->getData(),
-                $targetLocale
-            );
+            $translations[$attributeCode] = $attributeValue->getData();
+            $scopes[$attributeCode] = $targetScope;
+        }
 
-            if ($translatedText === null) {
-                continue;
-            }
+        $translatedText = $this->translator->translate(
+            json_encode($translations),
+            $targetLocale
+        );
 
-            $this->propertySetter->setData($product, $attributeCode, $translatedText, [
+        if ($translatedText === null) {
+            throw new InvalidOpenAiResponseException($translations);
+        }
+
+        foreach (json_decode($translatedText) as $key => $translation) {
+            $summary[$key] = [[$translations[$key] => $translation]];
+            $this->propertySetter->setData($product, $key, $translation, [
                 'locale' => $targetLocaleAkeneo,
-                'scope' => $targetScope,
+                'scope' => $scopes[$key],
             ]);
         }
 
+        $this->stepExecution->setExitStatus(new ExitStatus(ExitStatus::COMPLETED, json_encode($summary)));
+
         return $product;
     }
 

diff --git a/src/Macopedia/OpenAiTranslator/Translator/OpenAiTranslator.php b/src/Macopedia/OpenAiTranslator/Translator/OpenAiTranslator.php
@@ -8,7 +8,7 @@
 
 class OpenAiTranslator implements TranslatorInterface
 {
-    private const MESSAGE = 'Translate text betweeen <START> and <STOP> to %s. Keep HTMl unchanged. <START>%s<STOP>';
+    private const MESSAGE = 'Translate all values of given JSON betweeen <START> and <STOP> to %s. Keep HTML unchanged. Return valid JSON. <START>%s<STOP>';
 
     public function __construct(
         private OpenAiClient $openAiClient
@@ -22,7 +22,7 @@ public function translate(string $text, Language $targetLanguageCode): ?string
             ->ask('user', sprintf(self::MESSAGE, $targetLanguageCode->asText(), $text));
 
         if ($answer !== null) {
-            $answer = preg_replace('/(\(Note.*)/','',$answer);
+            $answer = preg_replace('/(\(Note.*)/', '', $answer);
         }
 
         return $answer;