From 1993328571bdf4c058ab0520ab8d5a12c35564ad Mon Sep 17 00:00:00 2001 From: drejmanMacopedia Date: Mon, 5 Jun 2023 11:52:20 +0200 Subject: [PATCH] Translate attributes of one produt at once. Refactor translation process --- .../TranslateAttributes.php | 52 ++++++++++++++++ .../TranslateAttributes.php | 41 +++++++++++++ .../MassEdit/TranslateAttributesProcessor.php | 4 +- .../Tasklet/ValidateOpenAiKeyTasklet.php | 33 +++++++++++ .../MacopediaTranslatorExtension.php | 3 +- .../Repository/AttributeRepository.php | 20 +++++++ .../Resources/config/connector.yml | 28 +++++++-- .../Resources/config/repositories.yml | 6 ++ .../Resources/config/services.yml | 2 +- .../Service/TranslateAttributesService.php | 59 +++++++++++++++---- .../Translator/OpenAiTranslator.php | 4 +- 11 files changed, 229 insertions(+), 23 deletions(-) create mode 100644 src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/ConstraintCollectionProvider/TranslateAttributes.php create mode 100644 src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/DefaultValueProvider/TranslateAttributes.php create mode 100644 src/Macopedia/OpenAiTranslator/Connector/Tasklet/ValidateOpenAiKeyTasklet.php create mode 100644 src/Macopedia/OpenAiTranslator/Repository/AttributeRepository.php create mode 100644 src/Macopedia/OpenAiTranslator/Resources/config/repositories.yml diff --git a/src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/ConstraintCollectionProvider/TranslateAttributes.php b/src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/ConstraintCollectionProvider/TranslateAttributes.php new file mode 100644 index 0000000..69840ae --- /dev/null +++ b/src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/ConstraintCollectionProvider/TranslateAttributes.php @@ -0,0 +1,52 @@ + $supportedJobNames + */ + public function __construct( + private array $supportedJobNames, + ) { + } + + /** + * {@inheritdoc} + */ + public function getConstraintCollection(): Collection + { + return new Collection( + [ + 'fields' => [ + 'filters' => new NotNull(), + 'actions' => new NotNull(), + 'realTimeVersioning' => new Type('bool'), + 'users_to_notify' => [ + new Type('array'), + new All(new Type('string')), + ], + 'is_user_authenticated' => new Type('bool') + ] + ] + ); + } + + /** + * {@inheritdoc} + */ + public function supports(JobInterface $job): bool + { + return in_array($job->getName(), $this->supportedJobNames); + } +} diff --git a/src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/DefaultValueProvider/TranslateAttributes.php b/src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/DefaultValueProvider/TranslateAttributes.php new file mode 100644 index 0000000..3408b56 --- /dev/null +++ b/src/Macopedia/OpenAiTranslator/Connector/Job/JobParameters/DefaultValueProvider/TranslateAttributes.php @@ -0,0 +1,41 @@ + $supportedJobNames + */ + public function __construct( + private array $supportedJobNames + ) { + } + + /** + * {@inheritdoc} + */ + public function getDefaultValues(): array + { + return [ + 'filters' => [], + 'actions' => [], + 'realTimeVersioning' => true, + 'users_to_notify' => [], + 'is_user_authenticated' => false + ]; + } + + /** + * {@inheritdoc} + */ + public function supports(JobInterface $job): bool + { + return in_array($job->getName(), $this->supportedJobNames); + } +} diff --git a/src/Macopedia/OpenAiTranslator/Connector/Processor/MassEdit/TranslateAttributesProcessor.php b/src/Macopedia/OpenAiTranslator/Connector/Processor/MassEdit/TranslateAttributesProcessor.php index 01eb9d2..83ff5c3 100644 --- a/src/Macopedia/OpenAiTranslator/Connector/Processor/MassEdit/TranslateAttributesProcessor.php +++ b/src/Macopedia/OpenAiTranslator/Connector/Processor/MassEdit/TranslateAttributesProcessor.php @@ -45,6 +45,8 @@ public function process(mixed $item): ProductInterface|ProductModelInterface */ private function translateAttributes(mixed $product, array $action): ProductInterface|ProductModelInterface { - return $this->translateAttributesService->translateAttributes($product, $action); + return $this->translateAttributesService + ->setStepExecution($this->stepExecution) + ->translateAttributes($product, $action); } } diff --git a/src/Macopedia/OpenAiTranslator/Connector/Tasklet/ValidateOpenAiKeyTasklet.php b/src/Macopedia/OpenAiTranslator/Connector/Tasklet/ValidateOpenAiKeyTasklet.php new file mode 100644 index 0000000..8b21f73 --- /dev/null +++ b/src/Macopedia/OpenAiTranslator/Connector/Tasklet/ValidateOpenAiKeyTasklet.php @@ -0,0 +1,33 @@ +openAiKey)) { + $this->stepExecution->addFailureException(new Exception('OpenAI key is not set')); + $this->stepExecution->setStatus(new BatchStatus(BatchStatus::FAILED)); + } + } + + public function setStepExecution(StepExecution $stepExecution): void + { + $this->stepExecution = $stepExecution; + } +} diff --git a/src/Macopedia/OpenAiTranslator/DependencyInjection/MacopediaTranslatorExtension.php b/src/Macopedia/OpenAiTranslator/DependencyInjection/MacopediaTranslatorExtension.php index ffdfca4..3ce0cc3 100644 --- a/src/Macopedia/OpenAiTranslator/DependencyInjection/MacopediaTranslatorExtension.php +++ b/src/Macopedia/OpenAiTranslator/DependencyInjection/MacopediaTranslatorExtension.php @@ -22,7 +22,8 @@ final class MacopediaTranslatorExtension extends Extension public function load(array $configs, ContainerBuilder $container): void { $loader = new YamlFileLoader($container, new FileLocator(__DIR__ . '/../Resources/config')); - $loader->load('connector.yml'); $loader->load('services.yml'); + $loader->load('connector.yml'); + $loader->load('repositories.yml'); } } diff --git a/src/Macopedia/OpenAiTranslator/Repository/AttributeRepository.php b/src/Macopedia/OpenAiTranslator/Repository/AttributeRepository.php new file mode 100644 index 0000000..a9ec30b --- /dev/null +++ b/src/Macopedia/OpenAiTranslator/Repository/AttributeRepository.php @@ -0,0 +1,20 @@ +_em->createQueryBuilder() + ->select('att') + ->from($this->_entityName, 'att', 'att.code') + ->where('att.code IN (:codes)')->setParameter('codes', $codes) + ->getQuery() + ->getResult(); + } +} diff --git a/src/Macopedia/OpenAiTranslator/Resources/config/connector.yml b/src/Macopedia/OpenAiTranslator/Resources/config/connector.yml index 8a73b34..a884fdd 100644 --- a/src/Macopedia/OpenAiTranslator/Resources/config/connector.yml +++ b/src/Macopedia/OpenAiTranslator/Resources/config/connector.yml @@ -1,3 +1,5 @@ +parameters: + maco.open_ai_translator.job: 'update_product_translations' services: Macopedia\OpenAiTranslator\Connector\Processor\MassEdit\TranslateAttributesProcessor: arguments: @@ -6,34 +8,48 @@ services: macopedia.job.update_product_translations: class: '%pim_connector.job.simple_job.class%' arguments: - - 'update_product_translations' + - '%maco.open_ai_translator.job%' - '@event_dispatcher' - '@akeneo_batch.job_repository' - - [ '@macopedia.step.update_product_translations.mass_edit' ] + - + - '@macopedia.step.validate_openai_key_tasklet' + - '@macopedia.step.update_product_translations.mass_edit' tags: - { name: akeneo_batch.job, connector: 'Macopedia OpenAi Connector', type: '%pim_enrich.job.mass_edit_type%' } macopedia.job.default_values_provider.translate_product: class: Akeneo\Pim\Enrichment\Component\Product\Connector\Job\JobParameters\DefaultValueProvider\ProductMassEdit arguments: - - [ 'update_product_translations' ] + - [ '%maco.open_ai_translator.job%' ] tags: - { name: akeneo_batch.job.job_parameters.default_values_provider } macopedia.job.constraint_collection_provider.translate_product: class: 'Akeneo\Pim\Enrichment\Component\Product\Connector\Job\JobParameters\ConstraintCollectionProvider\ProductMassEdit' arguments: - - [ 'update_product_translations' ] + - [ '%maco.open_ai_translator.job%' ] tags: - { name: akeneo_batch.job.job_parameters.constraint_collection_provider } macopedia.step.update_product_translations.mass_edit: class: '%pim_connector.step.item_step.class%' arguments: - - 'perform' + - 'translate' - '@event_dispatcher' - '@akeneo_batch.job_repository' - '@pim_enrich.reader.database.product_and_product_model' - '@Macopedia\OpenAiTranslator\Connector\Processor\MassEdit\TranslateAttributesProcessor' - '@pim_enrich.writer.database.product_and_product_model_writer' - - '%pim_job_product_batch_size%' \ No newline at end of file + - '%pim_job_product_batch_size%' + + Macopedia\OpenAiTranslator\Connector\Tasklet\ValidateOpenAiKeyTasklet: + arguments: + - '%open_ai_key%' + + macopedia.step.validate_openai_key_tasklet: + class: '%pim_connector.step.tasklet.class%' + arguments: + - 'validate_openai_key' + - '@event_dispatcher' + - '@akeneo_batch.job_repository' + - '@Macopedia\OpenAiTranslator\Connector\Tasklet\ValidateOpenAiKeyTasklet' \ No newline at end of file diff --git a/src/Macopedia/OpenAiTranslator/Resources/config/repositories.yml b/src/Macopedia/OpenAiTranslator/Resources/config/repositories.yml new file mode 100644 index 0000000..6c2cac2 --- /dev/null +++ b/src/Macopedia/OpenAiTranslator/Resources/config/repositories.yml @@ -0,0 +1,6 @@ +services: + Macopedia\OpenAiTranslator\Repository\AttributeRepository: + factory: [ '@doctrine.orm.entity_manager', 'getRepository' ] + arguments: [ '%pim_catalog.entity.attribute.class%' ] + tags: + - { name: 'pim_repository' } diff --git a/src/Macopedia/OpenAiTranslator/Resources/config/services.yml b/src/Macopedia/OpenAiTranslator/Resources/config/services.yml index cd9cd38..cbf70eb 100644 --- a/src/Macopedia/OpenAiTranslator/Resources/config/services.yml +++ b/src/Macopedia/OpenAiTranslator/Resources/config/services.yml @@ -14,6 +14,6 @@ services: Macopedia\OpenAiTranslator\Service\TranslateAttributesService: arguments: - '@Macopedia\OpenAiTranslator\Translator\OpenAiTranslator' - - '@pim_catalog.repository.attribute' + - '@Macopedia\OpenAiTranslator\Repository\AttributeRepository' - '@pim_catalog.entity_with_family_variant.check_attribute_editable' - '@pim_catalog.updater.property_setter' diff --git a/src/Macopedia/OpenAiTranslator/Service/TranslateAttributesService.php b/src/Macopedia/OpenAiTranslator/Service/TranslateAttributesService.php index 39dd98f..7b3b7d5 100644 --- a/src/Macopedia/OpenAiTranslator/Service/TranslateAttributesService.php +++ b/src/Macopedia/OpenAiTranslator/Service/TranslateAttributesService.php @@ -7,16 +7,22 @@ use Akeneo\Pim\Enrichment\Component\Product\EntityWithFamilyVariant\CheckAttributeEditable; use Akeneo\Pim\Enrichment\Component\Product\Model\ProductInterface; use Akeneo\Pim\Enrichment\Component\Product\Model\ProductModelInterface; -use Akeneo\Pim\Enrichment\Component\Product\Value\ScalarValue; use Akeneo\Pim\Structure\Component\AttributeTypes; use Akeneo\Pim\Structure\Component\Repository\AttributeRepositoryInterface; +use Akeneo\Tool\Component\Batch\Item\DataInvalidItem; +use Akeneo\Tool\Component\Batch\Job\BatchStatus; +use Akeneo\Tool\Component\Batch\Job\ExitStatus; +use Akeneo\Tool\Component\Batch\Model\StepExecution; use Akeneo\Tool\Component\StorageUtils\Updater\PropertySetterInterface; use Macopedia\OpenAiTranslator\Translator\Language; use Macopedia\OpenAiTranslator\Translator\TranslatorInterface; +use Macopedia\OpenAiTranslator\Exception\InvalidOpenAiResponseException; use Webmozart\Assert\Assert; class TranslateAttributesService { + private StepExecution $stepExecution; + public function __construct( private TranslatorInterface $translator, private AttributeRepositoryInterface $attributeRepository, @@ -25,6 +31,13 @@ public function __construct( ) { } + public function setStepExecution(StepExecution $stepExecution): self + { + $this->stepExecution = $stepExecution; + + return $this; + } + /** * @param ProductInterface|ProductModelInterface $product * @param array> $action @@ -32,14 +45,24 @@ public function __construct( public function translateAttributes(mixed $product, array $action): ProductInterface|ProductModelInterface { [$sourceScope, $targetScope, $sourceLocaleAkeneo, $targetLocaleAkeneo, $targetLocale, $attributesToTranslate] = $this->extractVariables($action); + $translations = []; + + $attributes = $this->attributeRepository->getAttributesByCodes($attributesToTranslate); + $summary = []; + $scopes = []; - foreach ($attributesToTranslate as $attributeCode) { - $attribute = $this->attributeRepository->findOneByIdentifier($attributeCode); + foreach ($attributes as $attribute) { if (!$this->checkAttributeEditable->isEditable($product, $attribute)) { + $this->stepExecution->addWarning('Attribute is not editable', [], new DataInvalidItem($attribute)); + $this->stepExecution->incrementSummaryInfo('skip'); + $this->stepExecution->incrementProcessedItems(); continue; } if (!($attribute->getType() === AttributeTypes::TEXT || $attribute->getType() === AttributeTypes::TEXTAREA)) { + $this->stepExecution->addWarning('Attribute is not text', [], new DataInvalidItem($attribute)); + $this->stepExecution->incrementSummaryInfo('skip'); + $this->stepExecution->incrementProcessedItems(); continue; } @@ -48,26 +71,38 @@ public function translateAttributes(mixed $product, array $action): ProductInter $targetScope = null; } + $attributeCode = $attribute->getCode(); $attributeValue = $product->getValue($attributeCode, $sourceLocaleAkeneo, $sourceScope); if ($attributeValue === null) { + $this->stepExecution->addWarning('Attribute value is empty', [], new DataInvalidItem($attribute)); + $this->stepExecution->incrementSummaryInfo('skip'); + $this->stepExecution->incrementProcessedItems(); continue; } - $translatedText = $this->translator->translate( - $attributeValue->getData(), - $targetLocale - ); + $translations[$attributeCode] = $attributeValue->getData(); + $scopes[$attributeCode] = $targetScope; + } - if ($translatedText === null) { - continue; - } + $translatedText = $this->translator->translate( + json_encode($translations), + $targetLocale + ); - $this->propertySetter->setData($product, $attributeCode, $translatedText, [ + if ($translatedText === null) { + throw new InvalidOpenAiResponseException($translations); + } + + foreach (json_decode($translatedText) as $key => $translation) { + $summary[$key] = [[$translations[$key] => $translation]]; + $this->propertySetter->setData($product, $key, $translation, [ 'locale' => $targetLocaleAkeneo, - 'scope' => $targetScope, + 'scope' => $scopes[$key], ]); } + $this->stepExecution->setExitStatus(new ExitStatus(ExitStatus::COMPLETED, json_encode($summary))); + return $product; } diff --git a/src/Macopedia/OpenAiTranslator/Translator/OpenAiTranslator.php b/src/Macopedia/OpenAiTranslator/Translator/OpenAiTranslator.php index 6ac423e..718aa19 100644 --- a/src/Macopedia/OpenAiTranslator/Translator/OpenAiTranslator.php +++ b/src/Macopedia/OpenAiTranslator/Translator/OpenAiTranslator.php @@ -8,7 +8,7 @@ class OpenAiTranslator implements TranslatorInterface { - private const MESSAGE = 'Translate text betweeen and to %s. Keep HTMl unchanged. %s'; + private const MESSAGE = 'Translate all values of given JSON betweeen and to %s. Keep HTML unchanged. Return valid JSON. %s'; public function __construct( private OpenAiClient $openAiClient @@ -22,7 +22,7 @@ public function translate(string $text, Language $targetLanguageCode): ?string ->ask('user', sprintf(self::MESSAGE, $targetLanguageCode->asText(), $text)); if ($answer !== null) { - $answer = preg_replace('/(\(Note.*)/','',$answer); + $answer = preg_replace('/(\(Note.*)/', '', $answer); } return $answer;