From ec4ed1ffd7a80e00f7b30a84c64937d702ac36fb Mon Sep 17 00:00:00 2001 From: Myddleware Date: Thu, 11 Jan 2024 09:53:55 +0100 Subject: [PATCH 01/88] Task parallelization (#1109) * Add column to lock a rule (send and read actions) Signed-off-by: myddleware * Manage read lock for rule Signed-off-by: myddleware * Manage lock on rule send action Signed-off-by: myddleware * Add jobLock field into document table Signed-off-by: myddleware * Fix bug Signed-off-by: myddleware * Add lock on the document Signed-off-by: myddleware * Add job lock index (and apply the other index changes) Signed-off-by: myddleware * Manage document lock Signed-off-by: myddleware * Manage lock on read and send functions Signed-off-by: myddleware * Manage unlock Signed-off-by: myddleware * Change crontab scheduler function Signed-off-by: myddleware * Remove parameter force job Remove lock on job (we can now run several job in the same time) Signed-off-by: myddleware * Remove unlock for rule and document on normal process as the locks should alredy be removed Signed-off-by: myddleware * Unlock rule in case of reading error Signed-off-by: myddleware * Fix lock on rule and document Signed-off-by: myddleware * Add indexes on rule table Signed-off-by: myddleware * Fix bug for mass rerun error Signed-off-by: myddleware * Add unset lock in case of fatal error Signed-off-by: myddleware * Manage unset for the synchro command Signed-off-by: myddleware * Change the place of the begin transaction to include the create document Signed-off-by: myddleware * Remove some transactions Signed-off-by: myddleware * Add parameter refreshDate to method updateDocumentData Signed-off-by: myddleware * Continue to run other documents when a document is locked. Signed-off-by: myddleware * Fix bug on Airtable : if data has been removed by a customcode, the process for this record has to be stopped Signed-off-by: myddleware * feat: reduce size of description, running and max * feat: working cron results * feat: edit running instances & max instance * feat: WIP: sort by id * feat import future crontabj.s * feat: edit crontab page * feat: adapt function to handle dates * feat: add order info, and sort table date modified * feat: order flag * feat: output * feat: working run at * feat: runtime, comment run at * feat: status code * feat: translation * feat: export js logic to dedicated file * Change the way to lock the send action, the lock isn't on the rule anymore but on the document Signed-off-by: myddleware * Change lock management using queries Signed-off-by: myddleware * Remove query on send lock Signed-off-by: myddleware * Fix close job Signed-off-by: myddleware * Fix bug on document selection with lock Signed-off-by: myddleware * SugarCRM : updatre connector Signed-off-by: myddleware * Fix bugs Signed-off-by: myddleware * BugFix : lock table document during selection Signed-off-by: myddleware * Fix : bug on transaction not closed Signed-off-by: myddleware * SugarCRM : manage team name field Solution : manage error incase of simulation (no rule in parameter) Bugfix : set status error sending if we couldn't connect to the target application Signed-off-by: myddleware --------- Signed-off-by: myddleware Co-authored-by: AlexMyddleware <106162060+AlexMyddleware@users.noreply.github.com> --- assets/app.js | 1 + assets/js/crontab.js | 96 +++ src/Command/CheckJobCommand.php | 2 +- src/Command/CronRunCommand.php | 32 +- src/Command/MassActionCommand.php | 4 +- src/Command/NotificationCommand.php | 2 +- src/Command/ReadRecordCommand.php | 7 +- src/Command/RerunErrorCommand.php | 7 +- src/Command/SynchroCommand.php | 18 +- src/Controller/ApiController.php | 6 +- src/Controller/JobSchedulerController.php | 8 +- src/Controller/TaskController.php | 13 + src/Entity/Document.php | 35 +- src/Entity/Rule.php | 25 +- src/Form/JobSchedulerCronType.php | 3 + src/Manager/DocumentManager.php | 340 +++++++--- src/Manager/JobManager.php | 50 +- src/Manager/RuleManager.php | 630 +++++++++++------- src/Repository/DocumentRepository.php | 13 + src/Repository/RuleRepository.php | 13 + src/Solutions/airtable.php | 44 +- src/Solutions/database.php | 3 +- src/Solutions/moodle.php | 2 +- src/Solutions/solution.php | 2 +- src/Solutions/sugarcrm.php | 94 ++- templates/JobScheduler/crontab_list.html.twig | 6 +- templates/JobScheduler/edit_crontab.html.twig | 14 + templates/JobScheduler/show_crontab.html.twig | 168 +++-- translations/messages.en.yml | 8 + webpack.config.js | 1 + yarn.lock | 3 +- 31 files changed, 1125 insertions(+), 525 deletions(-) create mode 100644 assets/js/crontab.js diff --git a/assets/app.js b/assets/app.js index 1edce9b14..3b5673b6a 100755 --- a/assets/app.js +++ b/assets/app.js @@ -41,6 +41,7 @@ require('./js/task.js') require('./js/connector.js') require('./js/smtp.js') require('./js/filter.js') +require('./js/crontab.js') require('./js/rule_relation_filter.js') diff --git a/assets/js/crontab.js b/assets/js/crontab.js new file mode 100644 index 000000000..4d29dc2a4 --- /dev/null +++ b/assets/js/crontab.js @@ -0,0 +1,96 @@ +// Creates a function that will be called when a button with the class crontab_class +// is clicked. The function will send a request to the server for the function with the following name crontab_show + +console.log('crontabbou.js'); + +// if an element with the class table-head-result is clicked, then call the function sortTable with the id of the element as an argument +$(document).on('click', '.table-head-result', function() { + console.log('click on table-head-result'); + var id = $(this).attr('id'); + console.log(id); + sortTable(id); +}); + +console.log('coucou camille'); +function sortTable(n) { + console.log('inside function sortTable'); + var table, rows, switching, i, x, y, shouldSwitch, dir, switchcount = 0; + table = document.getElementById("myTable"); + switching = true; + dir = "asc"; + + // Convert n to a number + var columnIndex = parseInt(n, 10); // Parse n as an integer + console.log("Column Index: ", columnIndex); + + console.log(n) + + // Update the h5 content to show the current sorting direction + var orderInfo = document.getElementById("crontab-order-info"); + orderInfo.innerText = "Sorting: " + (dir === "asc" ? "Ascending" : "Descending"); + + while (switching) { + switching = false; + rows = table.rows; + + for (i = 1; i < (rows.length - 1); i++) { + shouldSwitch = false; + x = rows[i].getElementsByTagName("TD")[n]; + y = rows[i + 1].getElementsByTagName("TD")[n]; + + // Check if we are dealing with dates + if (columnIndex === 1 || columnIndex === 2 || columnIndex === 4) { // assuming date columns are 1 (Date created) and 2 (Date update) and 4 (Run at) + var xDate = new Date(x.innerHTML); + var yDate = new Date(y.innerHTML); + } else { + var xContent = isNaN(parseInt(x.innerHTML)) ? x.innerHTML.toLowerCase() : parseInt(x.innerHTML); + var yContent = isNaN(parseInt(y.innerHTML)) ? y.innerHTML.toLowerCase() : parseInt(y.innerHTML); + } + + if (columnIndex === 1 || columnIndex === 2 || columnIndex === 4) { // date columns + if (dir == "asc") { + if (xDate > yDate) { + shouldSwitch = true; + break; + } + } else if (dir == "desc") { + if (xDate < yDate) { + shouldSwitch = true; + break; + } + } + } else { // other columns + if (dir == "asc") { + if (xContent > yContent) { + shouldSwitch = true; + break; + } + } else if (dir == "desc") { + if (xContent < yContent) { + shouldSwitch = true; + break; + } + } + } + } + + if (shouldSwitch) { + rows[i].parentNode.insertBefore(rows[i + 1], rows[i]); + switching = true; + switchcount++; + } else { + if (switchcount == 0 && dir == "asc") { + dir = "desc"; + switching = true; + // Update the h5 content when the direction changes + orderInfo.innerText = "Sorting: Descending"; + } + } + } + + // Final update in case no switching was done and the direction is already descending + if (!switching && dir === "desc") { + orderInfo.innerText = "Sorting: Descending"; + } + } + \ No newline at end of file diff --git a/src/Command/CheckJobCommand.php b/src/Command/CheckJobCommand.php index 665226cca..6d8bd6394 100644 --- a/src/Command/CheckJobCommand.php +++ b/src/Command/CheckJobCommand.php @@ -66,7 +66,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $period = $input->getArgument('period'); - $data = $this->jobManager->initJob('checkJob', 1); + $data = $this->jobManager->initJob('checkJob'); if (false === $data['success']) { $output->writeln('0;'.$data['message'].''); $this->logger->error($data['message']); diff --git a/src/Command/CronRunCommand.php b/src/Command/CronRunCommand.php index 0c7633184..3fd171cb2 100644 --- a/src/Command/CronRunCommand.php +++ b/src/Command/CronRunCommand.php @@ -56,10 +56,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (!($entity)) { throw new Exception("Couldn't fetch Cronjobs"); } - $valueCron = $entity->getValue(); + $valueCron = $entity->getValue(); if ( $valueCron == 1 - && !empty($valueCron)) + && !empty($valueCron)) { $jobsToRun = $jobRepo->findAll(); @@ -109,16 +109,25 @@ protected function execute(InputInterface $input, OutputInterface $output): int } else { $style->success('cronjob started successfully and is running in background'); } - - $style->info($job->getFullCommand().' : Begin '.date('Y-m-d h:i:s')); - $this->waitProcesses($processes); - $style->info($job->getFullCommand().' : End '.date('Y-m-d h:i:s')); - $processes = []; } - $style->success('All jobs are finished.'); + sleep(1); + + $style->section('Summary'); + + if (count($processes) > 0) { + $style->text('waiting for all running jobs ...'); + + // wait for all processes + $this->waitProcesses($processes); + + $style->success('All jobs are finished.'); + } else { + $style->info('No jobs were executed. See reasons below.'); + } return CronJobResult::EXIT_CODE_SUCCEEDED; - }else{ + + } else { $style->error('Your crontabs are disabled'); return CronJobResult::EXIT_CODE_FAILED; } @@ -138,7 +147,7 @@ public function waitProcesses(array $processes): void try { $process->checkTimeout(); - if (true === $process->isRunning()) { + if ($process->isRunning() === true) { break; } } catch (ProcessTimedOutException $e) { @@ -170,8 +179,7 @@ private function runJob(CronJob $job): Process $process->disableOutput(); $timeout = $this->commandHelper->getTimeout(); - - if (null !== $timeout && $timeout > 0) { + if ($timeout !== null && $timeout > 0) { $process->setTimeout($timeout); } diff --git a/src/Command/MassActionCommand.php b/src/Command/MassActionCommand.php index b89e8c032..12ec28b0d 100644 --- a/src/Command/MassActionCommand.php +++ b/src/Command/MassActionCommand.php @@ -99,7 +99,6 @@ protected function configure() ->addArgument('action', InputArgument::REQUIRED, 'Action (rerun, cancel, remove, restore or changeStatus)') ->addArgument('dataType', InputArgument::REQUIRED, 'Data type (rule or document)') ->addArgument('ids', InputArgument::REQUIRED, 'Rule or document ids') // id séparés par des ";" - ->addArgument('force', InputArgument::OPTIONAL, 'Force run even if another task is running.') ->addArgument('forceAll', InputArgument::OPTIONAL, 'Set Y to process action on all documents (not only open and error ones)') ->addArgument('fromStatus', InputArgument::OPTIONAL, 'Get all document with this status(Only with changeStatus action)') ->addArgument('toStatus', InputArgument::OPTIONAL, 'Set this status (Only with changeStatus action)') @@ -119,7 +118,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $fromStatus = $input->getArgument('fromStatus'); $toStatus = $input->getArgument('toStatus'); $api = $input->getArgument('api'); - $force = $input->getArgument('force') ? $input->getArgument('force') : false; // to avoid unwanted apostrophes in SQL queries $action = str_replace('\'', '', $action); $dataType = str_replace('\'', '', $dataType); @@ -134,7 +132,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $paramJobString = "Mass $action on data type $dataType"; - $data = $this->jobManager->initJob($paramJobString, $force); + $data = $this->jobManager->initJob($paramJobString); if (false === $data['success']) { $output->writeln('1;'.$data['message'].''); diff --git a/src/Command/NotificationCommand.php b/src/Command/NotificationCommand.php index 26b445f7e..bd56b8904 100644 --- a/src/Command/NotificationCommand.php +++ b/src/Command/NotificationCommand.php @@ -77,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Standard notification else { try { - $data = $this->jobManager->initJob('notification', '1'); + $data = $this->jobManager->initJob('notification'); if (false === $data['success']) { $output->writeln('0;'.$data['message'].''); diff --git a/src/Command/ReadRecordCommand.php b/src/Command/ReadRecordCommand.php index 783343c71..96ad0ee6f 100644 --- a/src/Command/ReadRecordCommand.php +++ b/src/Command/ReadRecordCommand.php @@ -60,7 +60,6 @@ protected function configure() ->addArgument('ruleId', InputArgument::REQUIRED, 'Rule used to read the records') ->addArgument('filterQuery', InputArgument::REQUIRED, 'Filter used to read data in the source application, eg : id') ->addArgument('filterValues', InputArgument::REQUIRED, 'Values corresponding to the fileter separated by comma, eg : 1256,4587') - ->addArgument('force', InputArgument::OPTIONAL, 'Force run even if another task is running.') ->addArgument('api', InputArgument::OPTIONAL, 'Call from API') ; } @@ -74,10 +73,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $ruleId = $input->getArgument('ruleId'); $filterQuery = $input->getArgument('filterQuery'); $filterValues = $input->getArgument('filterValues'); - $force = $input->getArgument('force'); - if (empty($force)) { - $force = false; - } $rule = $this->ruleRepository->findOneBy(['id' => $ruleId, 'deleted' => false]); if (null === $rule) { @@ -88,7 +83,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Set the API value $this->jobManager->setApi((bool) $api); - $data = $this->jobManager->initJob('read records with filter '.$filterQuery.' IN ('.$filterValues.')', $force); + $data = $this->jobManager->initJob('read records with filter '.$filterQuery.' IN ('.$filterValues.')'); if (false === $data['success']) { $output->writeln('0;'.$data['message'].''); $this->logger->error($data['message']); diff --git a/src/Command/RerunErrorCommand.php b/src/Command/RerunErrorCommand.php index 86e1e5a77..e3e08d5bb 100644 --- a/src/Command/RerunErrorCommand.php +++ b/src/Command/RerunErrorCommand.php @@ -58,7 +58,6 @@ protected function configure() ->setDescription('Synchronisation des données') ->addArgument('limit', InputArgument::REQUIRED, 'Nombre maximum de flux en erreur traité') ->addArgument('attempt', InputArgument::REQUIRED, 'Nombre maximum de tentative') - ->addArgument('force', InputArgument::OPTIONAL, 'Force run even if another task is running.') ->addArgument('api', InputArgument::OPTIONAL, 'Call from API') ; } @@ -71,15 +70,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int $limit = $input->getArgument('limit'); $attempt = $input->getArgument('attempt'); $api = $input->getArgument('api'); - $force = $input->getArgument('force'); - if (empty($force)) { - $force = false; - } // Set the API value $this->jobManager->setApi((bool) $api); - $data = $this->jobManager->initJob('Rerun error : limit '.$limit.', attempt '.$attempt, $force); + $data = $this->jobManager->initJob('Rerun error : limit '.$limit.', attempt '.$attempt); if (false === $data['success']) { $output->writeln('0;'.$data['message'].''); diff --git a/src/Command/SynchroCommand.php b/src/Command/SynchroCommand.php index f3a59701f..b141366a0 100644 --- a/src/Command/SynchroCommand.php +++ b/src/Command/SynchroCommand.php @@ -35,6 +35,7 @@ use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; +use Doctrine\Persistence\ManagerRegistry; class SynchroCommand extends Command { @@ -54,7 +55,8 @@ public function __construct( DocumentManager $documentManager, RuleManager $ruleManager, EntityManagerInterface $entityManager, - DocumentRepository $documentRepository + DocumentRepository $documentRepository, + ManagerRegistry $registry ) { parent::__construct(); $this->logger = $logger; @@ -63,6 +65,7 @@ public function __construct( $this->ruleManager = $ruleManager; $this->documentManager = $documentManager; $this->documentRepository = $documentRepository; + $this->registry = $registry; } protected function configure() @@ -71,7 +74,7 @@ protected function configure() ->setName('myddleware:synchro') ->setDescription('Execute all active Myddleware transfer rules') ->addArgument('rule', InputArgument::REQUIRED, 'Rule id, you can put several rule id separated by coma') - ->addArgument('force', InputArgument::OPTIONAL, 'Force run even if another task is running.') + ->addArgument('force', InputArgument::OPTIONAL, 'Force run even if the rule is inactive.') ->addArgument('api', InputArgument::OPTIONAL, 'Call from API') ; } @@ -93,7 +96,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Clear message in case this task is run by jobscheduler. In this case message has to be refreshed. $this->jobManager->message = ''; $this->jobManager->setApi($api); - $data = $this->jobManager->initJob('Synchro : '.$rule, $force); + $data = $this->jobManager->initJob('Synchro : '.$rule); if (true === $data['success']) { $output->writeln('1;'.$this->jobManager->getId()); // Not removed, user for manual job and webservices @@ -142,6 +145,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->jobManager->sendDocuments(); } catch (\Exception $e) { $this->jobManager->message .= 'Error rule '.$value.' '.$e->getMessage(); + // Reset entity manager in case it has been closed by the exception + if (!$this->entityManager->isOpen()) { + $this->entityManager = $this->registry->resetManager(); + } + // Unset all the read and send locks of the rule in case of fatal error (if the losk correspond to the current job) + if (!$this->jobManager->unsetRuleLock()) { + $this->jobManager->message .= 'Failed to unset the lock for the rule '.$value.'. '; + } } } } @@ -152,6 +163,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } catch (\Exception $e) { $this->jobManager->message .= $e->getMessage(); } + // Close job if it has been created if (true === $this->jobManager->createdJob) { $this->jobManager->closeJob(); diff --git a/src/Controller/ApiController.php b/src/Controller/ApiController.php index 39b577b53..280835fff 100644 --- a/src/Controller/ApiController.php +++ b/src/Controller/ApiController.php @@ -67,7 +67,6 @@ public function synchroAction(Request $request): JsonResponse // Get input data $data = json_decode($request->getContent(), true); - $force = !(('ALL' === $data['rule'])); // Check parameter if (empty($data['rule'])) { @@ -79,7 +78,7 @@ public function synchroAction(Request $request): JsonResponse $application->setAutoExit(false); $arguments = [ 'command' => 'myddleware:synchro', - 'force' => $force, + 'force' => (empty($data['force']) ? false : true), 'api' => 1, '--env' => $this->env, ]; @@ -148,7 +147,6 @@ public function readRecordAction(Request $request): JsonResponse $application->setAutoExit(false); $arguments = [ 'command' => 'myddleware:readrecord', - 'force' => 1, 'api' => 1, '--env' => $this->env, ]; @@ -331,7 +329,6 @@ public function massActionAction(Request $request): JsonResponse $application->setAutoExit(false); $arguments = [ 'command' => 'myddleware:massaction', - 'force' => 1, 'api' => 1, '--env' => $this->env, ]; @@ -401,7 +398,6 @@ public function rerunErrorAction(Request $request): JsonResponse $application->setAutoExit(false); $arguments = [ 'command' => 'myddleware:rerunerror', - 'force' => 1, 'api' => 1, '--env' => $this->env, ]; diff --git a/src/Controller/JobSchedulerController.php b/src/Controller/JobSchedulerController.php index 3ea951633..032c5a175 100644 --- a/src/Controller/JobSchedulerController.php +++ b/src/Controller/JobSchedulerController.php @@ -19,6 +19,7 @@ use Symfony\Contracts\Translation\TranslatorInterface; use Symfony\Component\Form\Extension\Core\Type\TextType; use Shapecode\Bundle\CronBundle\Entity\CronJob as CronJob; +use Shapecode\Bundle\CronBundle\Entity\CronJobResult; use Symfony\Component\Form\Extension\Core\Type\SubmitType; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; @@ -428,11 +429,14 @@ public function showCrontab($id): Response throw $this->createNotFoundException('Unable to find crontab entity.'); } - /** @var UserRepository $userRepository */ - $userRepository = $this->entityManager->getRepository(User::class); + // Fetch the CronJobResults and sort them by 'id' in ascending order + $entityResult = $this->entityManager->getRepository(CronJobResult::class)->findBy( + ['cronJob' => $id], + ); return $this->render('JobScheduler/show_crontab.html.twig', [ 'entity' => $entity, + 'entityResult' => $entityResult, ]); } diff --git a/src/Controller/TaskController.php b/src/Controller/TaskController.php index f51667d4f..57a259677 100644 --- a/src/Controller/TaskController.php +++ b/src/Controller/TaskController.php @@ -28,6 +28,8 @@ use App\Entity\Config; use App\Entity\Job; use App\Entity\Log; +use App\Entity\Document; +use App\Entity\Rule; use App\Manager\JobManager; use App\Repository\DocumentRepository; use App\Repository\JobRepository; @@ -74,6 +76,9 @@ public function __construct( $this->params[$config->getName()] = $config->getvalue(); } } + if (empty($this->DocumentRepository)) { + $this->documentRepository = $this->entityManager->getRepository(Document::class); + } } /** @@ -170,6 +175,14 @@ public function stopTask(Job $taskStop): RedirectResponse $log->setMessage('The task has been manually stopped. '); $log->setJob($taskStop); $em->persist($log); + + // Remove lock on document locked by this job + $this->documentRepository->removeLock($taskStop->getId()); + + // Remove lock (send and read) on rule locked by this job + $ruleRepository = $this->entityManager->getRepository(Rule::class); + $ruleRepository->removeLock($taskStop->getId()); + $em->flush(); return $this->redirect($this->generateURL('task_view', ['id' => $taskStop->getId()])); diff --git a/src/Entity/Document.php b/src/Entity/Document.php index c968cbb21..1586a15cd 100755 --- a/src/Entity/Document.php +++ b/src/Entity/Document.php @@ -34,13 +34,17 @@ /** * @ORM\Entity(repositoryClass="App\Repository\DocumentRepository") * @ORM\Table(name="document", indexes={ - * @ORM\Index(name="index_ruleid_status", columns={"rule_id","status"}), - * @ORM\Index(name="index_parent_id", columns={"parent_id"}), - * @ORM\Index(name="global_status", columns={"global_status"}), - * @ORM\Index(name="source_id", columns={"source_id"}), - * @ORM\Index(name="target_id", columns={"target_id"}), - * @ORM\Index(name="rule_id", columns={"rule_id"}), - * @ORM\Index(name="date_modified", columns={"date_modified"}) + * @ORM\Index(name="index_rule_gbstatus_status", columns={"rule_id","global_status","status","deleted"}), + * @ORM\Index(name="index_gbstatus", columns={"global_status","deleted"}), + * @ORM\Index(name="index_parent_id", columns={"parent_id","deleted"}), + * @ORM\Index(name="index_rule_source", columns={"rule_id","source_id","deleted"}), + * @ORM\Index(name="index_rule_target", columns={"rule_id","target_id","deleted"}), + * @ORM\Index(name="index_rule_date_modified", columns={"rule_id","date_modified","deleted"}), + * @ORM\Index(name="index_rule_status_modified", columns={"rule_id","status","source_date_modified","deleted"}), + * @ORM\Index(name="index_source_id", columns={"source_id","deleted"}), + * @ORM\Index(name="index_target_id", columns={"target_id","deleted"}), + * @ORM\Index(name="index_date_modified", columns={"date_modified","deleted"}), + * @ORM\Index(name="index_job_lock", columns={"job_lock"}) * }) */ class Document @@ -138,6 +142,12 @@ class Document * @ORM\OneToMany(targetEntity="Log", mappedBy="document") */ private $logs; + + /** + * @ORM\Column(name="job_lock", type="string", length=23, nullable=false) + */ + private string $jobLock; + public function __construct() { @@ -465,5 +475,16 @@ public function setModifiedBy(?User $modifiedBy): self return $this; } + + public function setJobLock($jobLock): self + { + $this->jobLock = $jobLock; + return $this; + } + + public function getJobLock(): string + { + return $this->jobLock; + } } diff --git a/src/Entity/Rule.php b/src/Entity/Rule.php index cc8de0861..e94a95d4b 100755 --- a/src/Entity/Rule.php +++ b/src/Entity/Rule.php @@ -35,7 +35,10 @@ * @ORM\Entity(repositoryClass="App\Repository\RuleRepository") * @ORM\HasLifecycleCallbacks() * - * @ORM\Table(name="rule", indexes={@ORM\Index(name="Krule_name", columns={"name"})}) + * @ORM\Table(name="rule", indexes={ + * @ORM\Index(name="Krule_name", columns={"name"}), + * @ORM\Index(name="index_read_job_lock", columns={"read_job_lock"}) + * }) */ class Rule { @@ -159,6 +162,11 @@ class Rule * @ORM\OrderBy({"sourceDateModified" : "ASC"}) */ private $documents; + + /** + * @ORM\Column(name="read_job_lock", type="string", length=23, nullable=true, options={"default":NULL}) + */ + private string $readJobLock; public function __construct() { @@ -286,6 +294,20 @@ public function getNameSlug(): string { return $this->nameSlug; } + + public function setReadJobLock($readJobLock): self + { + $this->readJobLock = $readJobLock; + return $this; + } + + public function getReadJobLock(): string + { + if (empty($this->readJobLock)) { + return ''; + } + return $this->readJobLock; + } public function setConnectorSource(Connector $connectorSource): self { @@ -675,4 +697,5 @@ public function isModuleTargetSet(): bool { return isset($this->moduleTarget); } + } diff --git a/src/Form/JobSchedulerCronType.php b/src/Form/JobSchedulerCronType.php index 412c2b523..0b5320398 100644 --- a/src/Form/JobSchedulerCronType.php +++ b/src/Form/JobSchedulerCronType.php @@ -4,6 +4,7 @@ use Symfony\Component\Form\AbstractType; use Symfony\Component\Form\Extension\Core\Type\CheckboxType; +use Symfony\Component\Form\Extension\Core\Type\IntegerType; use Symfony\Component\Form\Extension\Core\Type\SubmitType; use Symfony\Component\Form\Extension\Core\Type\TextType; use Symfony\Component\Form\FormBuilderInterface; @@ -20,6 +21,8 @@ public function buildForm(FormBuilderInterface $builder, array $options) ]) ->add('command', TextType::class, ['mapped' => false]) ->add('description', TextType::class) + ->add('runningInstances', IntegerType::class) + ->add('maxInstances', IntegerType::class) ->add('period', TextType::class) ->add('save', SubmitType::class, [ 'attr' => [ diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 431f93c34..55fda7552 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -60,6 +60,7 @@ class documentcore protected $documentType; protected bool $jobActive = true; protected $attempt; + protected $jobLock; protected $userId; protected $status; protected $document_data; @@ -203,7 +204,18 @@ public function setDocument($id_doc) $this->ruleMode = $this->document_data['mode']; $this->documentType = $this->document_data['type']; $this->attempt = $this->document_data['attempt']; - + $this->jobLock = $this->document_data['job_lock']; + // A document can be loaded only if there is no lock or if the lock is on the current job. + if ( + !empty($this->jobLock) + AND $this->jobLock != $this->jobId + ) { + throw new \Exception('This document is locked by the task '.$this->jobLock.'. '); + // No setlock if $this->jobLock == $this->jobId + } elseif (!empty($this->jobLock)) { + $this->setLock(); + } + // Get source data and create data attribut $this->sourceData = $this->getDocumentData('S'); $this->data = $this->sourceData; @@ -218,10 +230,10 @@ public function setDocument($id_doc) $this->logger->error('Failed to retrieve Document '.$id_doc.'.'); } } catch (\Exception $e) { - $this->message .= 'Failed to retrieve document : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; - $this->typeError = 'E'; - $this->logger->error($this->message); - $this->createDocLog(); + // Remove the lock because there is not status changed (lock is usually remove when we change the status) + $this->unsetLock(); + // Stop the process + throw new \Exception('Failed to load the document : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } } @@ -229,63 +241,73 @@ public function setDocument($id_doc) // Clear parameter is used when we call the same instance of the Document to manage several documents (from RuleManager class) public function setParam($param, $clear = false, $clearRule = true) { - if ($clear) { - $this->clearAttributes($clearRule); - } - // Chargement des solution si elles sont présentent dans les paramètres de construction - if (!empty($param['solutionTarget'])) { - $this->solutionTarget = $param['solutionTarget']; - } - if (!empty($param['solutionSource'])) { - $this->solutionSource = $param['solutionSource']; - } - if (!empty($param['jobId'])) { - $this->jobId = $param['jobId']; - } - if (!empty($param['api'])) { - $this->api = $param['api']; - } - if (!empty($param['parentId'])) { - $this->parentId = $param['parentId']; - } - if (!empty($param['ruleDocuments'])) { - $this->ruleDocuments = $param['ruleDocuments']; - } + try { + if ($clear) { + $this->clearAttributes($clearRule); + } + // Chargement des solution si elles sont présentent dans les paramètres de construction + if (!empty($param['solutionTarget'])) { + $this->solutionTarget = $param['solutionTarget']; + } + if (!empty($param['solutionSource'])) { + $this->solutionSource = $param['solutionSource']; + } + if (!empty($param['jobId'])) { + $this->jobId = $param['jobId']; + } + if (!empty($param['api'])) { + $this->api = $param['api']; + } + if (!empty($param['parentId'])) { + $this->parentId = $param['parentId']; + } + if (!empty($param['ruleDocuments'])) { + $this->ruleDocuments = $param['ruleDocuments']; + } - // Init attribut of the class Document - if (!empty($param['id_doc_myddleware'])) { - // Instanciate attribut sourceData - $this->setDocument($param['id_doc_myddleware']); - } else { - $this->id = uniqid('', true); - $this->dateCreated = gmdate('Y-m-d H:i:s'); - $this->ruleName = $param['rule']['name_slug']; - $this->ruleMode = $param['rule']['mode']; - $this->ruleId = $param['rule']['id']; - $this->ruleFields = $param['ruleFields']; - $this->data = $param['data']; - $this->sourceId = $this->data['id']; - $this->userId = $param['rule']['created_by']; - $this->status = 'New'; - $this->attempt = 0; - // Set the deletion type if myddleware deletion flag is true - if (!empty($this->data['myddleware_deletion'])) { - $this->documentType = 'D'; - } - } - // Ajout des paramètre de la règle - if (empty($this->ruleParams)) { - $this->setRuleParam(); - } - // Mise à jour des tableaux s'ils existent. - if (!empty($param['ruleFields'])) { - $this->ruleFields = $param['ruleFields']; - } - if (!empty($param['ruleRelationships'])) { - $this->ruleRelationships = $param['ruleRelationships']; + // Init attribut of the class Document + if (!empty($param['id_doc_myddleware'])) { + // Instanciate attribut sourceData + $this->setDocument($param['id_doc_myddleware']); + } else { + $this->id = uniqid('', true); + $this->dateCreated = gmdate('Y-m-d H:i:s'); + $this->ruleName = $param['rule']['name_slug']; + $this->ruleMode = $param['rule']['mode']; + $this->ruleId = $param['rule']['id']; + $this->ruleFields = $param['ruleFields']; + $this->data = $param['data']; + $this->sourceId = $this->data['id']; + $this->userId = $param['rule']['created_by']; + $this->status = 'New'; + $this->attempt = 0; + $this->jobLock = $this->jobId; + // Set the deletion type if myddleware deletion flag is true + if (!empty($this->data['myddleware_deletion'])) { + $this->documentType = 'D'; + } + } + // Ajout des paramètre de la règle + if (empty($this->ruleParams)) { + $this->setRuleParam(); + } + // Mise à jour des tableaux s'ils existent. + if (!empty($param['ruleFields'])) { + $this->ruleFields = $param['ruleFields']; + } + if (!empty($param['ruleRelationships'])) { + $this->ruleRelationships = $param['ruleRelationships']; + } + // Init type error for each new document + $this->typeError = 'S'; + } catch (\Exception $e) { + $this->message .= $e->getMessage(); + $this->typeError = 'E'; + $this->logger->error($this->message); + $this->createDocLog(); + return false; } - // Init type error for each new document - $this->typeError = 'S'; + return true; } // Clear all class attributes @@ -335,11 +357,11 @@ public function createDocument(): bool return false; } // Création du header de la requête - $query_header = 'INSERT INTO document (id, rule_id, date_created, date_modified, created_by, modified_by, source_id, source_date_modified, mode, type, parent_id) VALUES'; + $query_header = 'INSERT INTO document (id, rule_id, date_created, date_modified, created_by, modified_by, source_id, source_date_modified, mode, type, parent_id, job_lock) VALUES'; // Création de la requête d'entête $date_modified = $this->data['date_modified']; // Source_id could contain accent - $query_header .= "('$this->id','$this->ruleId','$this->dateCreated','$this->dateCreated','$this->userId','$this->userId','".utf8_encode($this->sourceId)."','$date_modified','$this->ruleMode','$this->documentType','$this->parentId')"; + $query_header .= "('$this->id','$this->ruleId','$this->dateCreated','$this->dateCreated','$this->userId','$this->userId','".utf8_encode($this->sourceId)."','$date_modified','$this->ruleMode','$this->documentType','$this->parentId', '')"; $stmt = $this->connection->prepare($query_header); $result = $stmt->executeQuery(); $this->updateStatus('New'); @@ -348,7 +370,6 @@ public function createDocument(): bool } catch (\Exception $e) { $this->message .= 'Failed to create document (id source : '.$this->sourceId.'): '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->logger->error($this->message); - return false; } } @@ -448,6 +469,76 @@ public function setDocIdRefError($docIdRefError) $this->docIdRefError = $docIdRefError; } + // Set the document lock + protected function setLock() { + try { + // Get the job lock on the document + $documentQuery = 'SELECT * FROM document WHERE id = :doc_id'; + $stmt = $this->connection->prepare($documentQuery); + $stmt->bindValue(':doc_id', $this->id); + $documentResult = $stmt->executeQuery(); + $documentData = $documentResult->fetchAssociative(); // 1 row + + // If document already lock by the current job, we return true; + if ($documentData['job_lock'] == $this->jobId) { + return array('success' => true); + // If document not locked, we lock it. + } elseif (empty($documentData['job_lock'])) { + $now = gmdate('Y-m-d H:i:s'); + $query = ' UPDATE document + SET + date_modified = :now, + job_lock = :job_id + WHERE + id = :id + '; + $stmt = $this->connection->prepare($query); + $stmt->bindValue(':now', $now); + $stmt->bindValue(':job_id', $this->jobId); + $stmt->bindValue(':id', $this->id); + $result = $stmt->executeQuery(); + return array('success' => true); + // Error for all other cases + } else { + return array('success' => false, 'error' => 'The document is locked by the task '.$documentData['job_lock'].'. '); + } + } catch (\Exception $e) { + // $this->connection->rollBack(); // -- ROLLBACK TRANSACTION + return array('success' => false, 'error' => 'Failed to lock the document '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + } + + // Set the document lock + public function unsetLock() { + try { + // Get the job lock on the document + $documentQuery = 'SELECT * FROM document WHERE id = :doc_id'; + $stmt = $this->connection->prepare($documentQuery); + $stmt->bindValue(':doc_id', $this->id); + $documentResult = $stmt->executeQuery(); + $documentData = $documentResult->fetchAssociative(); // 1 row + + // If document already lock by the current job, we return true; + if ($documentData['job_lock'] == $this->jobId) { + $now = gmdate('Y-m-d H:i:s'); + $query = " UPDATE document + SET + date_modified = :now, + job_lock = '' + WHERE + id = :id + "; + $stmt = $this->connection->prepare($query); + $stmt->bindValue(':now', $now); + $stmt->bindValue(':id', $this->id); + $result = $stmt->executeQuery(); + return true; + } + } catch (\Exception $e) { + return false; + } + } + // Permet d'indiquer si le filtreest rempli ou pas protected function checkFilter($fieldValue, $operator, $filterValue): bool { @@ -680,18 +771,20 @@ public function checkPredecessorDocument(): bool // Set the status Predecessor_OK $this->updateStatus('Predecessor_OK'); - // Check compatibility between rule mode et document tupe - // A rule in create mode can't update data excpt for a child rule - if ( - 'C' == $this->ruleMode - and 'U' == $this->documentType - and !$this->isChild() - ) { - $this->message .= 'Rule mode only allows to create data. Filter because this document updates or deletes data.'; - $this->updateStatus('Filter'); - // In case we flter the document, we return false to stop the process when this method is called in the rerun process - return false; - } + // Check compatibility between rule mode et document type + // A rule in create mode can't update data except for a child rule + if ( + $this->ruleMode == 'C' + and $this->documentType == 'U' + ) { + // Check child in a second time to avoid to run a query each time + if (!$this->isChild()) { + $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; + $this->updateStatus('Filter'); + // In case we flter the document, we return false to stop the process when this method is called in the rerun process + return false; + } + } return true; } catch (\Exception $e) { @@ -803,10 +896,23 @@ public function checkParentDocument(): bool if ('U' == $this->documentType) { $this->updateTargetId($this->targetId); $this->updateType('U'); + // Check compatibility between rule mode et document type + // A rule in create mode can't update data except for a child rule + if ( + $this->ruleMode == 'C' + and $this->documentType == 'U' + ) { + // Check child in a second time to avoid to run a query each time + if (!$this->isChild()) { + $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; + $this->updateStatus('Filter'); + // In case we flter the document, we return false to stop the process when this method is called in the rerun process + return false; + } + } } } $this->updateStatus('Relate_OK'); - return true; } catch (\Exception $e) { $this->message .= 'Failed to check document related : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; @@ -841,6 +947,20 @@ public function transformDocument(): bool $this->targetId = $target['Myddleware_element_id']; if ($this->updateTargetId($this->targetId)) { $this->updateType('U'); + // Check compatibility between rule mode et document type + // A rule in create mode can't update data except for a child rule + if ( + $this->ruleMode == 'C' + and $this->documentType == 'U' + ) { + // Check child in a second time to avoid to run a query each time + if (!$this->isChild()) { + $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; + $this->updateStatus('Filter'); + // In case we flter the document, we return false to stop the process when this method is called in the rerun process + return false; + } + } } else { throw new \Exception('The type of this document is Update. Failed to update the target id '.$this->targetId.' on this document. This document is queued. '); } @@ -976,6 +1096,20 @@ public function getTargetDataDocument(): bool } else { $this->updateStatus('Ready_to_send'); $this->updateType('U'); + // Check compatibility between rule mode et document type + // A rule in create mode can't update data except for a child rule + if ( + $this->ruleMode == 'C' + and $this->documentType == 'U' + ) { + // Check child in a second time to avoid to run a query each time + if (!$this->isChild()) { + $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; + $this->updateStatus('Filter'); + // In case we flter the document, we return false to stop the process when this method is called in the rerun process + return false; + } + } } $this->updateTargetId($history['id']); } @@ -1263,6 +1397,7 @@ protected function insertDataTable($data, $type): bool } } } + // We save the relationship field too if (!empty($this->ruleRelationships)) { foreach ($this->ruleRelationships as $ruleRelationship) { @@ -1282,6 +1417,7 @@ protected function insertDataTable($data, $type): bool $documentData->setType($type); // Source $documentData->setData(json_encode($dataInsert)); // Encode in JSON $this->entityManager->persist($documentData); + $this->entityManager->flush(); } catch (\Exception $e) { $this->message .= 'Failed : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; @@ -1919,7 +2055,6 @@ public function changeDeleteFlag($deleteFlag) */ public function updateStatus($new_status) { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { // On ajoute un contôle dans le cas on voudrait changer le statut $new_status = $this->beforeStatusChange($new_status); @@ -1936,7 +2071,8 @@ public function updateStatus($new_status) date_modified = :now, global_status = :globalStatus, attempt = :attempt, - status = :new_status + status = :new_status, + job_lock = :jobLock WHERE id = :id '; @@ -1947,21 +2083,21 @@ public function updateStatus($new_status) ) { echo 'status '.$new_status.' id = '.$this->id.' '.$now.chr(10); } - // Suppression de la dernière virgule $stmt = $this->connection->prepare($query); $stmt->bindValue(':now', $now); $stmt->bindValue(':globalStatus', $globalStatus); $stmt->bindValue(':attempt', $this->attempt); $stmt->bindValue(':new_status', $new_status); $stmt->bindValue(':id', $this->id); + // Remove the lock on the document in the class and in the database + $this->jobLock = ''; + $stmt->bindValue(':jobLock', $this->jobLock); $result = $stmt->executeQuery(); $this->message .= 'Status : '.$new_status; - $this->connection->commit(); // -- COMMIT TRANSACTION $this->status = $new_status; $this->afterStatusChange($new_status); $this->createDocLog(); } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->message .= 'Error status update : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; $this->logger->error($this->message); @@ -1974,7 +2110,6 @@ public function updateStatus($new_status) */ public function updateDeleteFlag($deleted) { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { $now = gmdate('Y-m-d H:i:s'); $query = ' UPDATE document @@ -1997,10 +2132,8 @@ public function updateDeleteFlag($deleted) $stmt->bindValue(':id', $this->id); $result = $stmt->executeQuery(); $this->message .= (!empty($deleted) ? 'Remove' : 'Restore').' document'; - $this->connection->commit(); // -- COMMIT TRANSACTION $this->createDocLog(); } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->message .= 'Failed to '.(!empty($deleted) ? 'Remove ' : 'Restore ').' : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; $this->logger->error($this->message); @@ -2045,7 +2178,6 @@ protected function afterStatusChange($new_status) */ public function updateType($new_type) { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { $now = gmdate('Y-m-d H:i:s'); $query = ' UPDATE document @@ -2062,12 +2194,10 @@ public function updateType($new_type) $stmt->bindValue(':id', $this->id); $result = $stmt->executeQuery(); $this->message .= 'Type : '.$new_type; - $this->connection->commit(); // -- COMMIT TRANSACTION $this->createDocLog(); // Change the document type for the current process $this->documentType = $new_type; } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->message .= 'Error type : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; $this->logger->error($this->message); @@ -2080,7 +2210,6 @@ public function updateType($new_type) */ public function updateTargetId($target_id): bool { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { $now = gmdate('Y-m-d H:i:s'); $query = ' UPDATE document @@ -2098,14 +2227,12 @@ public function updateTargetId($target_id): bool $stmt->bindValue(':id', $this->id); $result = $stmt->executeQuery(); $this->message .= 'Target id : '.$target_id; - $this->connection->commit(); // -- COMMIT TRANSACTION $this->createDocLog(); // Change the target id for the current process $this->targetId = $target_id; return true; } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->message .= 'Error target id : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; $this->logger->error($this->message); @@ -2116,7 +2243,7 @@ public function updateTargetId($target_id): bool } // Function to manually edit the data inside a Myddleware Document - public function updateDocumentData(string $docId, array $newValues, string $dataType) + public function updateDocumentData(string $docId, array $newValues, string $dataType, bool $refreshData = false) { // check if data of that type with this docid and this data fields if (empty($docId)) { @@ -2150,18 +2277,22 @@ public function updateDocumentData(string $docId, array $newValues, string $data // Compare data $oldData = json_decode($documentDataEntity->getData()); if(!empty($oldData)){ - foreach ($newValues as $key => $Value) { - foreach ($oldData as $oldKey => $oldValue) { - if ($oldKey === $key) { - if ($oldValue !== $Value) { - $newValues[$oldKey] = $Value; - $this->message .= ($dataType == 'S' ? 'Source' : ($dataType == 'T' ? 'Target' : 'History')).' document value changed from '.$oldValue.' to '.$Value.'. '; - } - } else { - $newValues[$oldKey] = $oldValue; - } - } - } + if (!$refreshData) { + foreach ($newValues as $key => $Value) { + foreach ($oldData as $oldKey => $oldValue) { + if ($oldKey === $key) { + if ($oldValue !== $Value) { + $newValues[$oldKey] = $Value; + $this->message .= ($dataType == 'S' ? 'Source' : ($dataType == 'T' ? 'Target' : 'History')).' document value changed from '.$oldValue.' to '.$Value.'. '; + } + } else { + $newValues[$oldKey] = $oldValue; + } + } + } + } else { + $this->message .= ($dataType == 'S' ? 'Source' : ($dataType == 'T' ? 'Target' : 'History')).' document value changed by '.print_r($newValues,true).'. '; + } $this->typeError = 'I'; $this->createDocLog(); // Update the data of the right type @@ -2367,7 +2498,6 @@ public function getStatus() */ protected function createDocLog() { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { $now = gmdate('Y-m-d H:i:s'); $query_header = 'INSERT INTO log (created, type, msg, rule_id, doc_id, ref_doc_id, job_id) VALUES (:created,:typeError,:message,:rule_id,:doc_id,:ref_doc_id,:job_id)'; @@ -2381,9 +2511,7 @@ protected function createDocLog() $stmt->bindValue(':job_id', $this->jobId); $result = $stmt->executeQuery(); $this->message = ''; - $this->connection->commit(); // -- COMMIT TRANSACTION } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to create log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } } diff --git a/src/Manager/JobManager.php b/src/Manager/JobManager.php index 8731e339a..450f4fdf0 100644 --- a/src/Manager/JobManager.php +++ b/src/Manager/JobManager.php @@ -206,6 +206,11 @@ public function setRule($filter): bool return false; } } + + // Unset the lock on the rule + public function unsetRuleLock() { + return $this->ruleManager->unsetRuleLock(); + } // Permet de contrôler si un docuement de la même règle pour le même enregistrement n'est pas close public function createDocuments() @@ -267,6 +272,7 @@ public function sendDocuments() public function runError($limit, $attempt) { try { + $ruleId = ''; // Récupération de tous les flux en erreur ou des flux en attente (new) qui ne sont pas sur règles actives (règle child pour des règles groupées) $sqlParams = " SELECT * FROM document @@ -285,43 +291,35 @@ public function runError($limit, $attempt) if (!empty($documentsError)) { // include_once 'rule.php'; foreach ($documentsError as $documentError) { - $this->ruleManager->setRule($documentError['rule_id']); - $this->ruleManager->setJobId($this->id); - $this->ruleManager->setManual($this->manual); - $this->ruleManager->setApi($this->api); + // Load the rule only if it has changed + if ($ruleId != $documentError['rule_id']) { + $this->ruleManager->setRule($documentError['rule_id']); + $this->ruleManager->setJobId($this->id); + $this->ruleManager->setManual($this->manual); + $this->ruleManager->setApi($this->api); + } $errorActionDocument = $this->ruleManager->actionDocument($documentError['id'], 'rerun'); if (!empty($errorActionDocument)) { $this->message .= print_r($errorActionDocument, true); } + $ruleId = $documentError['rule_id']; } } } catch (Exception $e) { - $this->logger->error('Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); - $this->message .= 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $this->logger->error('Error AAA : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $this->message .= 'Error AAA : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } } /** * @throws \Doctrine\DBAL\Exception */ - public function initJob(string $paramJob, bool $force = false): array + public function initJob(string $paramJob): array { $this->paramJob = $paramJob; $this->id = uniqid('', true); $this->start = microtime(true); - // Check if a job is already running except if force = true (api call or manuel call) - if (!$force) { - $sqlJobOpen = "SELECT * FROM job WHERE status = 'Start' LIMIT 1"; - $stmt = $this->connection->prepare($sqlJobOpen); - $result = $stmt->executeQuery(); - $job = $result->fetchAssociative(); // 1 row - // Error if one job is still running - if (!empty($job)) { - $this->message .= $this->tools->getTranslation(['messages', 'rule', 'another_task_running']).';'.$job['id']; - return ['success' => false, 'message' => $this->message]; - } - } // Create Job $insertJob = $this->insertJob(); if ($insertJob) { @@ -356,7 +354,6 @@ public function actionMassTransfer($event, $datatype, $param) $paramJob[] = $event; $paramJob[] = $datatype; $paramJob[] = implode(',', $param); - $paramJob[] = 1; // Force run even if another task is running return $this->runBackgroundJob('massaction', $paramJob); } else { @@ -398,7 +395,7 @@ public function runBackgroundJob($job, $param) } catch (IOException $e) { throw new Exception('An error occurred while creating your directory'); } - exec($php.' '.__DIR__.'/../../bin/console myddleware:'.$job.' '.$params.' 1 --env='.$this->env.' > '.$fileTmp.' &'); + exec($php.' '.__DIR__.'/../../bin/console myddleware:'.$job.' '.$params.' --env='.$this->env.' > '.$fileTmp.' &'); $cpt = 0; // Boucle tant que le fichier n'existe pas while (!file_exists($fileTmp)) { @@ -509,7 +506,7 @@ public function massAction($action, $dataType, $ids, $forceAll, $fromStatus, $to $error = $this->ruleManager->actionDocument($document['id'], $action, $toStatus); // Save the error if exists if (!empty($error)) { - $errors[] = $error[0]; + $errors[] = current($error); } } } else { @@ -1312,7 +1309,7 @@ public function getLogData() */ protected function updateJob(): bool { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION + // $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { $close = $this->logData['Close']; $cancel = $this->logData['Cancel']; @@ -1341,10 +1338,11 @@ protected function updateJob(): bool $stmt->bindValue('error', $error); $stmt->bindValue('message', $message); $stmt->bindValue('id', $this->id); - $result = $stmt->executeQuery(); - $this->connection->commit(); // -- COMMIT TRANSACTION + $result = $stmt->executeQuery(); + + // $this->connection->commit(); // -- COMMIT TRANSACTION } catch (Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION + // $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to update Job : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); $this->message .= 'Failed to update Job : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index f989c91d1..230d7b05f 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -27,6 +27,7 @@ use App\Entity\Config; use App\Entity\DocumentData; +use App\Entity\Document; use App\Entity\Rule; use App\Entity\RuleParam; use App\Entity\RuleParamAudit as RuleParamAudit; @@ -177,6 +178,54 @@ public function setApi($api) $this->api = $api; } + // Unset the lock on the rule + protected function setRuleLock() { + try { + // Get the rule details + $rule = $this->entityManager->getRepository(Rule::class)->findOneBy(['id' => $this->ruleId, 'deleted' => false]); + // If read lock empty, we set the lock with the job id + if (empty($rule->getReadJobLock())) { + $rule->setReadJobLock($this->jobId); + $this->entityManager->persist($rule); + $this->entityManager->flush(); + return true; + } + } catch (Exception $e) { + $this->logger->error('Failed set the lock on the rule '.$this->ruleId.' : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + return false; + } + + // Unset the lock on the rule + public function unsetRuleLock() { + try { + // Get the rule details + $rule = $this->entityManager->getRepository(Rule::class)->findOneBy(['id' => $this->ruleId, 'deleted' => false]); + // If read lock empty, we set the lock with the job id + $readJobLock = $rule->getReadJobLock(); + if ( + !empty($readJobLock) + AND $readJobLock == $this->jobId + ) { + $rule->setReadJobLock(''); + $this->entityManager->persist($rule); + $this->entityManager->flush(); + } elseif (!empty($readJobLock)) { + return false; + } + } catch (Exception $e) { + $this->logger->error('Failed unset the lock on the rule '.$this->ruleId.' : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + return false; + } + return true; + } + + protected function getRuleLock() { + // Get the rule details + $rule = $this->entityManager->getRepository(Rule::class)->findOneBy(['id' => $this->ruleId, 'deleted' => false]); + return $rule->getReadJobLock(); + } + /** * Generate a document for the current rule for a specific id in the source application. We don't use the reference for the function read. * If parameter readSource is false, it means that the data source are already in the parameter param, so no need to read in the source application. @@ -185,7 +234,6 @@ public function setApi($api) */ public function generateDocuments($idSource, $readSource = true, $param = '', $idFiledName = 'id') { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit try { $documents = []; if ($readSource) { @@ -241,11 +289,8 @@ public function generateDocuments($idSource, $readSource = true, $param = '', $i $documents[] = $childDocument; } } - $this->commit(false); // -- COMMIT TRANSACTION - return $documents; } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $error = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->logger->error($error); $errorObj = new \stdClass(); @@ -329,26 +374,36 @@ public function createDocuments() ) ) ) { - // lecture des données dans la source - $readSource = $this->readSource(); - if (empty($readSource['error'])) { - $readSource['error'] = ''; - } - - // Si erreur - if (!isset($readSource['count'])) { - return $readSource; - } + // Check the rule isn't locked + if (!$this->setRuleLock()) { + return array('error' => 'The rule '.$this->ruleId.' is locked by the task '.$this->getRuleLock().'. Failed to read the source application. '); + } + $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit try { + // lecture des données dans la source + $readSource = $this->readSource(); + if (empty($readSource['error'])) { + $readSource['error'] = ''; + } + // If error we unlock the rule and we return the result + if (!isset($readSource['count'])) { + $this->unsetRuleLock(); + $this->connection->commit(); + return $readSource; + } + if ($readSource['count'] > 0) { + // Before creating the documents, we check the job id is the one in the rule lock + if ($this->getRuleLock() != $this->jobId) { + throw new \Exception('The rule '.$this->ruleId.' is locked by the task '.$this->getRuleLock().'. Failed to generate the documents. '); + } $param['rule'] = $this->rule; $param['ruleFields'] = $this->ruleFields; $param['ruleRelationships'] = $this->ruleRelationships; // Set the param of the rule one time for all $this->documentManager->setRuleId($this->ruleId); $this->documentManager->setRuleParam(); - $i = 0; if ($this->dataSource['values']) { // Set all config parameters $this->setConfigParam(); @@ -358,17 +413,13 @@ public function createDocuments() } // Boucle sur chaque document foreach ($this->dataSource['values'] as $row) { - if ($i >= $this->limitReadCommit) { - $this->commit(true); // -- COMMIT TRANSACTION - $i = 0; - } - ++$i; $param['data'] = $row; $param['jobId'] = $this->jobId; $param['api'] = $this->api; // Set the param values and clear all document attributes but not rule attributes - $this->documentManager->setParam($param, true, false); - $createDocument = $this->documentManager->createDocument(); + if($this->documentManager->setParam($param, true)) { + $createDocument = $this->documentManager->createDocument(); + } if (!$createDocument) { $readSource['error'] .= $this->documentManager->getMessage(); } @@ -379,16 +430,22 @@ public function createDocuments() } // If params has been added in the output of the rule we saved it $this->updateParams(); + + // No error management because we don't want any rollback because of the lock. + // If the lock isn't removed, the next task will generate an error + $this->unsetRuleLock(); // Rollback if the job has been manually stopped if ('Start' != $this->getJobStatus()) { - throw new \Exception('The task has been stopped manually during the document creation. No document generated. '); + throw new \Exception('The task has been stopped manually. No document generated. '); } - $this->commit(false); // -- COMMIT TRANSACTION + $this->connection->commit(); } catch (\Exception $e) { $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to create documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); $readSource['error'] = 'Failed to create documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + // The process is finished even if there is an exception so we unlock the rule + $this->unsetRuleLock(); } } // On affiche pas d'erreur si la lecture est désactivée @@ -619,26 +676,21 @@ public function filterDocuments($documents = null): array // Pour tous les docuements sélectionnés on vérifie les prédécesseurs if (!empty($documents)) { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit try { + if ('Start' != $this->getJobStatus()) { + throw new \Exception('The task has been stopped manually. No document generated. '); + } $this->setRuleFilter(); - $i = 0; foreach ($documents as $document) { - if ($i >= $this->limitReadCommit) { - $this->commit(true); // -- COMMIT TRANSACTION - $i = 0; - } - ++$i; $param['id_doc_myddleware'] = $document['id']; $param['jobId'] = $this->jobId; $param['api'] = $this->api; // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $response[$document['id']] = $this->documentManager->filterDocument($this->ruleFilters); + if($this->documentManager->setParam($param, true)) { + $response[$document['id']] = $this->documentManager->filterDocument($this->ruleFilters); + } } - $this->commit(false); // -- COMMIT TRANSACTION } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to filter documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); $readSource['error'] = 'Failed to filter documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } @@ -664,26 +716,21 @@ public function checkPredecessorDocuments($documents = null): array } // Pour tous les docuements sélectionnés on vérifie les prédécesseurs if (!empty($documents)) { - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit try { - $i = 0; + if ('Start' != $this->getJobStatus()) { + throw new \Exception('The task has been stopped manually. No document generated. '); + } foreach ($documents as $document) { - if ($i >= $this->limitReadCommit) { - $this->commit(true); // -- COMMIT TRANSACTION - $i = 0; - } - ++$i; $param['id_doc_myddleware'] = $document['id']; $param['jobId'] = $this->jobId; $param['api'] = $this->api; $param['ruleRelationships'] = $this->ruleRelationships; // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $response[$document['id']] = $this->documentManager->checkPredecessorDocument(); + if($this->documentManager->setParam($param, true)) { + $response[$document['id']] = $this->documentManager->checkPredecessorDocument(); + } } - $this->commit(false); // -- COMMIT TRANSACTION } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to check predecessors : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); $readSource['error'] = 'Failed to check predecessors : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } @@ -725,27 +772,22 @@ public function checkParentDocuments($documents = null): array } } } - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit try { - $i = 0; + if ('Start' != $this->getJobStatus()) { + throw new \Exception('The task has been stopped manually. No document generated. '); + } // Pour tous les docuements sélectionnés on vérifie les parents foreach ($documents as $document) { - if ($i >= $this->limitReadCommit) { - $this->commit(true); // -- COMMIT TRANSACTION - $i = 0; - } - ++$i; $param['id_doc_myddleware'] = $document['id']; $param['jobId'] = $this->jobId; $param['api'] = $this->api; $param['ruleRelationships'] = $this->ruleRelationships; // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $response[$document['id']] = $this->documentManager->checkParentDocument(); + if($this->documentManager->setParam($param, true)) { + $response[$document['id']] = $this->documentManager->checkParentDocument(); + } } - $this->commit(false); // -- COMMIT TRANSACTION } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to check parents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); $readSource['error'] = 'Failed to check parents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } @@ -788,26 +830,22 @@ public function transformDocuments($documents = null): array } } } - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit + try { - $i = 0; + if ('Start' != $this->getJobStatus()) { + throw new \Exception('The task has been stopped manually. No document generated. '); + } // Transformation de tous les docuements sélectionnés foreach ($documents as $document) { - if ($i >= $this->limitReadCommit) { - $this->commit(true); // -- COMMIT TRANSACTION - $i = 0; - } - ++$i; $param['id_doc_myddleware'] = $document['id']; // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $response[$document['id']] = $this->documentManager->transformDocument(); + if($this->documentManager->setParam($param, true)) { + $response[$document['id']] = $this->documentManager->transformDocument(); + } } - $this->commit(false); // -- COMMIT TRANSACTION } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to transform documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); - $readSource['error'] = 'Failed to transform documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $response['error'] = 'Failed to transform documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } } @@ -837,16 +875,12 @@ public function getTargetDataDocuments($documents = null): array if (!empty($documents)) { // Connexion à la solution cible pour rechercher les données $this->connexionSolution('target'); - $this->connection->beginTransaction(); // -- BEGIN TRANSACTION suspend auto-commit try { - $i = 0; + if ('Start' != $this->getJobStatus()) { + throw new \Exception('The task has been stopped manually. No document generated. '); + } // Récupération de toutes les données dans la cible pour chaque document foreach ($documents as $document) { - if ($i >= $this->limitReadCommit) { - $this->commit(true); // -- COMMIT TRANSACTION - $i = 0; - } - ++$i; $param['id_doc_myddleware'] = $document['id']; $param['solutionTarget'] = $this->solutionTarget; $param['ruleFields'] = $this->ruleFields; @@ -854,13 +888,12 @@ public function getTargetDataDocuments($documents = null): array $param['jobId'] = $this->jobId; $param['api'] = $this->api; // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $response[$document['id']] = $this->documentManager->getTargetDataDocument(); - $response['doc_status'] = $this->documentManager->getStatus(); + if($this->documentManager->setParam($param, true)) { + $response[$document['id']] = $this->documentManager->getTargetDataDocument(); + $response['doc_status'] = $this->documentManager->getStatus(); + } } - $this->commit(false); // -- COMMIT TRANSACTION } catch (\Exception $e) { - $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to create documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); $readSource['error'] = 'Failed to create documents : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } @@ -888,7 +921,6 @@ public function sendDocuments(): array $sendTarget['error'] .= 'Failed to logout from the target solution'; } } - return $sendTarget; } @@ -1126,7 +1158,7 @@ protected function changeDeleteFlag($id_document, $deleteFlag) /** * @throws \Doctrine\DBAL\Exception */ - protected function changeStatus($id_document, $toStatus, $message = null, $docIdRefError = null) + protected function changeStatus($id_document, $toStatus, $message = null, $docIdRefError = null, $typeError = null) { $param['id_doc_myddleware'] = $id_document; $param['jobId'] = $this->jobId; @@ -1136,6 +1168,9 @@ protected function changeStatus($id_document, $toStatus, $message = null, $docId if (!empty($message)) { $this->documentManager->setMessage($message); } + if (!empty($typeError)) { + $this->documentManager->setTypeError($typeError); + } if (!empty($docIdRefError)) { $this->documentManager->setDocIdRefError($docIdRefError); } @@ -1160,19 +1195,19 @@ protected function runMyddlewareJob($ruleId, $event = null, $documentId = null) throw new \Exception($this->tools->getTranslation(['messages', 'rule', 'failed_create_directory'])); } if ($documentId !== null) { - exec($php.' '.__DIR__.'/../../bin/console myddleware:readrecord '.$ruleId.' id '.$documentId.' 1 --env='.$this->env.' > '.$fileTmp.' &', $output); + exec($php.' '.__DIR__.'/../../bin/console myddleware:readrecord '.$ruleId.' id '.$documentId.' --env='.$this->env.' > '.$fileTmp.' &', $output); } //if user clicked on cancel all transfers of a rule elseif ('cancelDocumentJob' === $event) { exec($php.' '.__DIR__.'/../../bin/console myddleware:massaction cancel rule '.$ruleId.' --env='.$this->env.' > '.$fileTmp.' &', $output); //if user clicked on delete all transfers from a rule } elseif ('deleteDocumentJob' === $event) { - exec($php.' '.__DIR__.'/../../bin/console myddleware:massaction remove rule '.$ruleId.' 1 Y --env='.$this->env.' > '.$fileTmp.' &', $output); + exec($php.' '.__DIR__.'/../../bin/console myddleware:massaction remove rule '.$ruleId.' Y --env='.$this->env.' > '.$fileTmp.' &', $output); } elseif ('ALL' == $ruleId) { // We don't set the parameter force to 1 when we synchronize all rules exec($php.' '.__DIR__.'/../../bin/console myddleware:synchro '.$ruleId.' --env='.$this->env.' > '.$fileTmp.' &', $output); } else { - exec($php.' '.__DIR__.'/../../bin/console myddleware:synchro '.$ruleId.' 1 --env='.$this->env.' > '.$fileTmp.' &', $output); + exec($php.' '.__DIR__.'/../../bin/console myddleware:synchro '.$ruleId.' --env='.$this->env.' > '.$fileTmp.' &', $output); } $cpt = 0; // Boucle tant que le fichier n'existe pas @@ -1225,126 +1260,130 @@ protected function runMyddlewareJob($ruleId, $event = null, $documentId = null) */ protected function rerun($id_document): array { - $session = new Session(); - $msg_error = []; - $msg_success = []; - $msg_info = []; - // Récupération du statut du document - $param['id_doc_myddleware'] = $id_document; - $param['jobId'] = $this->jobId; - $param['api'] = $this->api; - // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $status = $this->documentManager->getStatus(); - // Si la règle n'est pas chargée alors on l'initialise. - if (empty($this->ruleId)) { - $this->ruleId = $this->documentManager->getRuleId(); - $this->setRule($this->ruleId); - $this->setRuleRelationships(); - $this->setRuleParam(); - $this->setRuleField(); - } + try { + $session = new Session(); + $msg_error = []; + $msg_success = []; + $msg_info = []; + // Récupération du statut du document + $param['id_doc_myddleware'] = $id_document; + $param['jobId'] = $this->jobId; + $param['api'] = $this->api; + // Set the param values and clear all document attributes + $this->documentManager->setParam($param, true); + $status = $this->documentManager->getStatus(); + // Si la règle n'est pas chargée alors on l'initialise. + if (empty($this->ruleId)) { + $this->ruleId = $this->documentManager->getRuleId(); + $this->setRule($this->ruleId); + $this->setRuleRelationships(); + $this->setRuleParam(); + $this->setRuleField(); + } - $response[$id_document] = false; - // On lance des méthodes différentes en fonction du statut en cours du document et en fonction de la réussite ou non de la fonction précédente - if (in_array($status, ['New', 'Filter_KO'])) { - $response = $this->filterDocuments([['id' => $id_document]]); - if (true === $response[$id_document]) { - $msg_success[] = 'Transfer id '.$id_document.' : Status change => Filter_OK'; - } elseif (-1 == $response[$id_document]) { - $msg_info[] = 'Transfer id '.$id_document.' : Status change => Filter'; - } else { - $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer => Filter_KO'; - } - // Update status if an action has been executed - $status = $this->documentManager->getStatus(); - } - if (in_array($status, ['Filter_OK', 'Predecessor_KO'])) { - $response = $this->checkPredecessorDocuments([['id' => $id_document]]); - if (true === $response[$id_document]) { - $msg_success[] = 'Transfer id '.$id_document.' : Status change => Predecessor_OK'; - } else { - $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer => Predecessor_KO'; - } - // Update status if an action has been executed - $status = $this->documentManager->getStatus(); - } - if (in_array($status, ['Predecessor_OK', 'Relate_KO'])) { - $response = $this->checkParentDocuments([['id' => $id_document]]); - if (true === $response[$id_document]) { - $msg_success[] = 'Transfer id '.$id_document.' : Status change => Relate_OK'; - } else { - $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer => Relate_KO'; - } - // Update status if an action has been executed - $status = $this->documentManager->getStatus(); - } - if (in_array($status, ['Relate_OK', 'Error_transformed'])) { - $response = $this->transformDocuments([['id' => $id_document]]); - if (true === $response[$id_document]) { - $msg_success[] = 'Transfer id '.$id_document.' : Status change : Transformed'; - } else { - $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer : Error_transformed'; - } - // Update status if an action has been executed - $status = $this->documentManager->getStatus(); - } - if (in_array($status, ['Transformed', 'Error_checking', 'Not_found'])) { - $response = $this->getTargetDataDocuments([['id' => $id_document]]); - if (true === $response[$id_document]) { - if ('S' == $this->rule['mode']) { - $msg_success[] = 'Transfer id '.$id_document.' : Status change : '.$response['doc_status']; - } else { - $msg_success[] = 'Transfer id '.$id_document.' : Status change : '.$response['doc_status']; - } - } else { - $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer : '.$response['doc_status']; - } - // Update status if an action has been executed - $status = $this->documentManager->getStatus(); - } - // Si la règle est en mode recherche alors on n'envoie pas de données - // Si on a un statut compatible ou si le doc vient de passer dans l'étape précédente et qu'il n'est pas no_send alors on envoie les données - if ( - 'S' != $this->rule['mode'] - && ( - in_array($status, ['Ready_to_send', 'Error_sending']) - || ( - true === $response[$id_document] - && ( - empty($response['doc_status']) - || ( - !empty($response['doc_status']) - && 'No_send' != $response['doc_status'] - ) - ) - ) - ) - ) { - $response = $this->sendTarget('', $id_document); - if ( - !empty($response[$id_document]['id']) - && empty($response[$id_document]['error']) - && empty($response['error']) // Error can be on the document or can be a general error too - ) { - $msg_success[] = 'Transfer id '.$id_document.' : Status change : Send'; - } else { - $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer : Error_sending. '.(!empty($response['error']) ? $response['error'] : $response[$id_document]['error']); - } - } - // If the job is manual, we display error in the UI - if ($this->manual) { - if (!empty($msg_error)) { - $session->set('error', $msg_error); - } - if (!empty($msg_success)) { - $session->set('success', $msg_success); - } - if (!empty($msg_info)) { - $session->set('info', $msg_info); - } + $response[$id_document] = false; + // On lance des méthodes différentes en fonction du statut en cours du document et en fonction de la réussite ou non de la fonction précédente + if (in_array($status, ['New', 'Filter_KO'])) { + $response = $this->filterDocuments([['id' => $id_document]]); + if (true === $response[$id_document]) { + $msg_success[] = 'Transfer id '.$id_document.' : Status change => Filter_OK'; + } elseif (-1 == $response[$id_document]) { + $msg_info[] = 'Transfer id '.$id_document.' : Status change => Filter'; + } else { + $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer => Filter_KO'; + } + // Update status if an action has been executed + $status = $this->documentManager->getStatus(); + } + if (in_array($status, ['Filter_OK', 'Predecessor_KO'])) { + $response = $this->checkPredecessorDocuments([['id' => $id_document]]); + if (true === $response[$id_document]) { + $msg_success[] = 'Transfer id '.$id_document.' : Status change => Predecessor_OK'; + } else { + $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer => Predecessor_KO'; + } + // Update status if an action has been executed + $status = $this->documentManager->getStatus(); + } + if (in_array($status, ['Predecessor_OK', 'Relate_KO'])) { + $response = $this->checkParentDocuments([['id' => $id_document]]); + if (true === $response[$id_document]) { + $msg_success[] = 'Transfer id '.$id_document.' : Status change => Relate_OK'; + } else { + $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer => Relate_KO'; + } + // Update status if an action has been executed + $status = $this->documentManager->getStatus(); + } + if (in_array($status, ['Relate_OK', 'Error_transformed'])) { + $response = $this->transformDocuments([['id' => $id_document]]); + if (true === $response[$id_document]) { + $msg_success[] = 'Transfer id '.$id_document.' : Status change : Transformed'; + } else { + $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer : Error_transformed'; + } + // Update status if an action has been executed + $status = $this->documentManager->getStatus(); + } + if (in_array($status, ['Transformed', 'Error_checking', 'Not_found'])) { + $response = $this->getTargetDataDocuments([['id' => $id_document]]); + if (true === $response[$id_document]) { + if ('S' == $this->rule['mode']) { + $msg_success[] = 'Transfer id '.$id_document.' : Status change : '.$response['doc_status']; + } else { + $msg_success[] = 'Transfer id '.$id_document.' : Status change : '.$response['doc_status']; + } + } else { + $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer : '.$response['doc_status']; + } + // Update status if an action has been executed + $status = $this->documentManager->getStatus(); + } + // Si la règle est en mode recherche alors on n'envoie pas de données + // Si on a un statut compatible ou si le doc vient de passer dans l'étape précédente et qu'il n'est pas no_send alors on envoie les données + if ( + 'S' != $this->rule['mode'] + && ( + in_array($status, ['Ready_to_send', 'Error_sending']) + || ( + true === $response[$id_document] + && ( + empty($response['doc_status']) + || ( + !empty($response['doc_status']) + && 'No_send' != $response['doc_status'] + ) + ) + ) + ) + ) { + $response = $this->sendTarget('', $id_document); + if ( + !empty($response[$id_document]['id']) + && empty($response[$id_document]['error']) + && empty($response['error']) // Error can be on the document or can be a general error too + ) { + $msg_success[] = 'Transfer id '.$id_document.' : Status change : Send'; + } else { + $msg_error[] = 'Transfer id '.$id_document.' : Error, status transfer : Error_sending. '.(!empty($response[$id_document]['error']) ? $response[$id_document]['error'] : $response['error'] ); + } + } + // If the job is manual, we display error in the UI + if ($this->manual) { + if (!empty($msg_error)) { + $session->set('error', $msg_error); + } + if (!empty($msg_success)) { + $session->set('success', $msg_success); + } + if (!empty($msg_info)) { + $session->set('info', $msg_info); + } + } + } catch (Exception $e) { + $this->logger->error('Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $msg_error[] = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } - return $msg_error; } @@ -1557,7 +1596,7 @@ public function isChild(): bool protected function sendTarget($type, $documentId = null): array { - try { + try { // Permet de charger dans la classe toutes les relations de la règle $response = []; $response['error'] = ''; @@ -1569,7 +1608,6 @@ protected function sendTarget($type, $documentId = null): array $type = $documentData['type']; } } - // Récupération du contenu de la table target pour tous les documents à envoyer à la cible $send['data'] = $this->getSendDocuments($type, $documentId); @@ -1628,8 +1666,12 @@ protected function sendTarget($type, $documentId = null): array $response['error'] = 'Type transfer '.$type.' unknown. '; } } else { - $response[$documentId] = false; - $response['error'] = $connect['error']; + $response[$documentId] = false; + // If we couldn't connect the target application, we set the status error_sending to all documents not sent + foreach ($send['data'] as $idDoc => $data) { + $this->changeStatus($idDoc, 'Error_sending', 'Failed to connect to the target application.', null, 'E'); + } + throw new \Exception('Failed to connect to the target application.'); } } @@ -1690,7 +1732,6 @@ protected function massSendTarget($type, $documentId = null) } foreach($arrayDocumentsIds as $documentId){ - // $send['data'][$documentId] = $this->getSendDocuments($type, $documentId); $sendDataDocumentArrayElement = $this->getSendDocuments($type, $documentId); $send['data'] = (object) [$documentId => $sendDataDocumentArrayElement[$documentId]]; } @@ -1908,23 +1949,43 @@ protected function checkDuplicate($transformedData) protected function selectDocuments($status, $type = '') { + $this->connection->beginTransaction(); // -- BEGIN TRANSACTION try { - $query_documents = " SELECT * - FROM document + // Select documents depending of the status + $queryDocuments = " SELECT d + FROM App\Entity\Document d WHERE - rule_id = :ruleId - AND status = :status - AND document.deleted = 0 - ORDER BY document.source_date_modified ASC - LIMIT $this->limit + d.rule = :ruleId + AND d.status = :status + AND d.deleted = 0 + AND ( + d.jobLock = '' + OR d.jobLock = :jobId + ) + ORDER BY d.sourceDateModified ASC "; - $stmt = $this->connection->prepare($query_documents); - $stmt->bindValue(':ruleId', $this->ruleId); - $stmt->bindValue(':status', $status); - $result = $stmt->executeQuery(); - - return $result->fetchAllAssociative(); + + $query = $this->entityManager->createQuery($queryDocuments); + $query->setParameters([ + 'ruleId' => $this->entityManager->getRepository(Rule::class)->findOneBy(['id' => $this->ruleId, 'deleted' => false]), + 'status' => $status, + 'jobId' => $this->jobId, + ]); + $query->setMaxResults($this->limit); + // Lock the table during the query until all documents are locked + $query->setLockMode(\Doctrine\DBAL\LockMode::PESSIMISTIC_WRITE); + $documents = $query->getArrayResult(); // array of ForumUser objects getArrayResult + // Lock all the document + if (!empty($documents)) { + foreach ($documents as $document) { + // Lock focument without checking bacause we have selected only document with no lock + $this->setDocumentLock($document['id'], false); + } + } + $this->connection->commit(); // -- COMMIT TRANSACTION - release documents + return $documents; } catch (\Exception $e) { + $this->connection->rollBack(); // -- ROLLBACK TRANSACTION - release documents $this->logger->error('Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } } @@ -1959,20 +2020,35 @@ public function getSendDocuments($type, $documentId=null, $table = 'target', $pa // Si un document est en paramètre alors on filtre la requête sur le document if (!empty($documentId)) { $documentFilter = " document.id = '$documentId' - AND document.deleted = 0 "; + AND document.deleted = 0 + AND ( + document.job_lock = '' + OR document.job_lock = '$this->jobId' + ) + "; } elseif (!empty($parentDocId)) { $documentFilter = " document.parent_id = '$parentDocId' AND document.rule_id = '$parentRuleId' - AND document.deleted = 0 "; + AND document.deleted = 0 + AND ( + document.job_lock = '' + OR document.job_lock = '$this->jobId' + ) + "; // No limit when it comes to child rule. A document could have more than $limit child documents $limit = ''; } // Sinon on récupère tous les documents élligible pour l'envoi else { - $documentFilter = " document.rule_id = '$this->ruleId' + $documentFilter = " document.rule_id = '$this->ruleId' AND document.status = 'Ready_to_send' - AND document.deleted = 0 - AND document.type = '$type' "; + AND document.deleted = 0 + AND document.type = '$type' + AND ( + document.job_lock = '' + OR document.job_lock = '$this->jobId' + ) + "; } // Sélection de tous les documents au statut transformed en attente de création pour la règle en cours $sql = "SELECT document.id id_doc_myddleware, document.target_id, document.source_date_modified @@ -1983,7 +2059,6 @@ public function getSendDocuments($type, $documentId=null, $table = 'target', $pa $stmt = $this->connection->prepare($sql); $result = $stmt->executeQuery(); $documents = $result->fetchAllAssociative(); - foreach ($documents as $document) { // If the rule is a parent, we have to get the data of all rules child $childRules = $this->getChildRules(); @@ -2008,14 +2083,34 @@ public function getSendDocuments($type, $documentId=null, $table = 'target', $pa } } } - $data = $this->getDocumentData($document['id_doc_myddleware'], strtoupper(substr($table, 0, 1))); - if (!empty($data)) { - $return[$document['id_doc_myddleware']] = array_merge($document, $data); + + // Lock the document + $documentLock = $this->setDocumentLock($document['id_doc_myddleware']); + if($documentLock['success']) { + // Get document data + $data = $this->getDocumentData($document['id_doc_myddleware'], strtoupper(substr($table, 0, 1))); + if (!empty($data)) { + // Document is added to the result to be sent + $return[$document['id_doc_myddleware']] = array_merge($document, $data); + } else { + $error = 'No data found in the document'; + } } else { - $return['error'] = 'No data found in the document'; - } + $error = $documentLock['error']; + } + // If error we create a log on the document but we keep the status ready to send + if (!empty($error)) { + $param['id_doc_myddleware'] = $document['id_doc_myddleware']; + $param['jobId'] = $this->jobId; + $param['api'] = $this->api; + // Set the param values and clear all document attributes + if($this->documentManager->setParam($param, true)) { + $this->documentManager->generateDocLog('W', $error); + } else { + $this->logger->error('Job '.$this->jobId.' : Failed to create log for the document '.$document['id_doc_myddleware'].'. '); + } + } } - if (!empty($return)) { return $return; } @@ -2023,6 +2118,56 @@ public function getSendDocuments($type, $documentId=null, $table = 'target', $pa return null; } + // Set the document lock + protected function setDocumentLock($docId, $check = true) { + try { + // Get the job lock on the document + if ($check) { + $documentQuery = 'SELECT * FROM document WHERE id = :doc_id'; + $stmt = $this->connection->prepare($documentQuery); + $stmt->bindValue(':doc_id', $docId); + $documentResult = $stmt->executeQuery(); + $documentData = $documentResult->fetchAssociative(); // 1 row + } + + // If document already lock by the current job (rerun action for example), we return true; + if ( + $check + AND $documentData['job_lock'] == $this->jobId + ) { + return array('success' => true); + // If document not locked, we lock it. + } elseif ( + !$check + OR ( + $check + AND empty($documentData['job_lock']) + ) + ) { + $now = gmdate('Y-m-d H:i:s'); + $query = ' UPDATE document + SET + date_modified = :now, + job_lock = :job_id + WHERE + id = :id + '; + $stmt = $this->connection->prepare($query); + $stmt->bindValue(':now', $now); + $stmt->bindValue(':job_id', $this->jobId); + $stmt->bindValue(':id', $docId); + $result = $stmt->executeQuery(); + return array('success' => true); + // Error for all other cases + } else { + return array('success' => false, 'error' => 'The document is locked by the task '.$documentData['job_lock'].'. '); + } + } catch (\Exception $e) { + // $this->connection->rollBack(); // -- ROLLBACK TRANSACTION + return array('success' => false, 'error' => 'Failed to lock the document '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + } + // Permet de charger tous les champs de la règle protected function setRuleField() { @@ -2220,23 +2365,6 @@ protected function setConfigParam() } } - /** - * Checks whether a job is still active then commits the transaction. - * - * @throws \Doctrine\DBAL\Exception - */ - protected function commit($newTransaction) - { - // Rollback if the job has been manually stopped - if ('Start' != $this->getJobStatus()) { - throw new \Exception('The task has been stopped manually during the document creation. No document generated. '); - } - $this->connection->commit(); // -- COMMIT TRANSACTION - $this->entityManager->flush(); - if ($newTransaction) { - $this->connection->beginTransaction(); - } - } /** * Parameter de la règle choix utilisateur. diff --git a/src/Repository/DocumentRepository.php b/src/Repository/DocumentRepository.php index ea34b4cbf..cc02bf289 100644 --- a/src/Repository/DocumentRepository.php +++ b/src/Repository/DocumentRepository.php @@ -429,4 +429,17 @@ public static function findStatusType(EntityManagerInterface $entityManager) $finalResults = array_flip($curatedResults); return $finalResults; } + + // Remove lock from document using a job id + public function removeLock($jobId) { + $empty = null; + $q = $this->createQueryBuilder('d') + ->update() + ->set('d.jobLock', ':empty') + ->where('d.jobLock = :jobLock') + ->setParameter('jobLock', $jobId) + ->setParameter('empty', $empty) + ->getQuery(); + $q->execute(); + } } diff --git a/src/Repository/RuleRepository.php b/src/Repository/RuleRepository.php index a41783d6b..7c5b68b09 100644 --- a/src/Repository/RuleRepository.php +++ b/src/Repository/RuleRepository.php @@ -305,4 +305,17 @@ public static function findNameSlug(EntityManagerInterface $entityManager) return $finalResults; } + // Remove lock from rule using a job id + public function removeLock($jobId) { + $empty = null; + $qr = $this->createQueryBuilder('r') + ->update() + ->set('r.readJobLock', ':empty') + ->where('r.readJobLock = :readJobLock') + ->setParameter('readJobLock', $jobId) + ->setParameter('empty', $empty) + ->getQuery(); + $qr->execute(); + } + } diff --git a/src/Solutions/airtable.php b/src/Solutions/airtable.php index 49aeef991..8a1b879c7 100644 --- a/src/Solutions/airtable.php +++ b/src/Solutions/airtable.php @@ -403,6 +403,12 @@ public function upsert(string $method, array $param): array ++$i; continue; } + + // If no data we skip this record + if (empty($data)) { + unset($records[$idDoc]); + continue; + } $body['records'][$i]['fields'] = $data; @@ -435,7 +441,7 @@ public function upsert(string $method, array $param): array ++$i; } - // Airtable fiueld can contains space which is not compatible in Myddleware. + // Airtable field can contains space which is not compatible in Myddleware. // Because we replace space by ___ in Myddleware, we change ___ to space before sending data to Airtable if (!empty($body['records'])) { foreach ($body['records'] as $keyRecord => $record) { @@ -473,23 +479,25 @@ public function upsert(string $method, array $param): array $content = $response->toArray(); if (!empty($content)) { $i = 0; - foreach ($records as $idDoc => $data) { - $record = $content['records'][$i]; - if (!empty($record['id'])) { - $result[$idDoc] = [ - 'id' => $record['id'], - 'error' => false, - ]; - } else { - $result[$idDoc] = [ - 'id' => '-1', - 'error' => 'Failed to send data. Message from Airtable : '.print_r($content['records'][$i], true), - ]; - } - ++$i; - // Modification du statut du flux - $this->updateDocumentStatus($idDoc, $result[$idDoc], $param); - } + if (!empty($records)) { + foreach ($records as $idDoc => $data) { + $record = $content['records'][$i]; + if (!empty($record['id'])) { + $result[$idDoc] = [ + 'id' => $record['id'], + 'error' => false, + ]; + } else { + $result[$idDoc] = [ + 'id' => '-1', + 'error' => 'Failed to send data. Message from Airtable : '.print_r($content['records'][$i], true), + ]; + } + ++$i; + // Modification du statut du flux + $this->updateDocumentStatus($idDoc, $result[$idDoc], $param); + } + } } else { throw new Exception('Failed to send the record but no error returned by Airtable. '); } diff --git a/src/Solutions/database.php b/src/Solutions/database.php index fec2f3320..cc55443ad 100644 --- a/src/Solutions/database.php +++ b/src/Solutions/database.php @@ -207,7 +207,6 @@ public function get_module_fields($module, $type = 'source', $param = null): arr public function readData($param) { $result = []; - $result['date_ref'] = $param['date_ref']; // Decode field name (converted in method get_module_fields) $param['fields'] = array_map('rawurldecode', $param['fields']); try { @@ -215,6 +214,8 @@ public function readData($param) if (empty($param['date_ref'])) { $param['date_ref'] = 0; } + $result['date_ref'] = $param['date_ref']; + if (empty($param['limit'])) { $param['limit'] = 100; } diff --git a/src/Solutions/moodle.php b/src/Solutions/moodle.php index fb27229e6..0d20bb4b8 100644 --- a/src/Solutions/moodle.php +++ b/src/Solutions/moodle.php @@ -261,8 +261,8 @@ public function read($param): array } } } catch (\Exception $e) { - $result['error'] = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->logger->error($result['error']); + throw new \Exception('Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } return $result; } diff --git a/src/Solutions/solution.php b/src/Solutions/solution.php index 0091f5e71..b0ac2cc75 100644 --- a/src/Solutions/solution.php +++ b/src/Solutions/solution.php @@ -353,7 +353,7 @@ public function readData($param) $result['date_ref'] = $param['date_ref']; } } catch (\Exception $e) { - $result['error'] = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $result['error'] = (!empty($param['rule']['id']) ? 'Error in rule '.$param['rule']['id'].' : ' : 'Error : ').$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } return $result; diff --git a/src/Solutions/sugarcrm.php b/src/Solutions/sugarcrm.php index 534da49ac..12721f23a 100644 --- a/src/Solutions/sugarcrm.php +++ b/src/Solutions/sugarcrm.php @@ -153,7 +153,7 @@ public function get_modules($type = 'source') /** * @throws \Exception */ - protected function isManyToManyRel($module): bool + protected function isManyToManyRel($module) { if ( !empty($module) @@ -274,14 +274,62 @@ public function get_module_fields($module, $type = 'source', $param = null): arr } } - // public function readData($param) - + // public function readRelationship($param) /** * @throws \Exception */ + public function readRelationship($param, $rel) { + // Set the parameters for relationship reading + $filterArgs = [ + 'max_num' => $param['limit'], + 'offset' => 0, + 'fields' => implode(',',$param['fields']), + 'order_by' => 'date_modified', + 'deleted' => $param['ruleParams']['deletion'], + ]; + // The call for relationship required the id of the root record + if (empty($param['query'])) { + throw new \Exception('No query parameter, failed to read relationship'); + } + // The query parameter must be the reference id. + // For example if we try to read the users from a team, query must contains the team id + $refereceId = current($param['query']); + + // Get the record using the input parameters + $getRecords = $this->sugarAPI->filterRelated(key($param['query']), $refereceId, $rel->rhs_table)->execute($filterArgs); + $response = $getRecords->getResponse(); + // Format response if http return = 200 + if ('200' == $response->getStatus()) { + $body = $getRecords->getResponse()->getBody(false); + if (!empty($body->records)) { + $records = $body->records; + } + } else { + $bodyError = $response->getBody(); + throw new \Exception('Status '.$response->getStatus().' : '.$bodyError['error'].', '.$bodyError['error_message']); + } + // Format records to result format + if (!empty($records)) { + foreach ($records as $record) { + // The record id is build with both ids from the relationship + $recordId = $refereceId.'_'.$record->id; + $result[$recordId]['id'] = $recordId; + $result[$recordId]['date_modified'] = $record->date_modified; + // Get both ids + $result[$recordId][$rel->join_key_lhs] = $refereceId; + $result[$recordId][$rel->join_key_rhs] = $record->id; + } + } + return $result; + } + public function read($param) { $result = []; + $rel = $this->isManyToManyRel($param['module']); + if ($rel !== false) { + return $this->readRelationship($param, $rel); + } // Manage delete option to enable $deleted = false; @@ -332,10 +380,11 @@ public function read($param) $bodyError = $response->getBody(); throw new \Exception('Status '.$response->getStatus().' : '.$bodyError['error'].', '.$bodyError['error_message']); } + // Format records to result format - if (!empty($records)) { + if (!empty($records)) { + // Manage deletion by adding the flag Myddleware_deletion to the record foreach ($records as $record) { - // Manage deletion by adding the flag Myddleware_deletion to the record if ( true == $deleted and !empty($record->deleted) @@ -345,8 +394,21 @@ public function read($param) // At least one non deleted record read $onlyDeletion = false; } - - foreach ($param['fields'] as $field) { + } + // Error if only deletion records read + if ($onlyDeletion) { + if (count($result) >= $param['limit']) { + throw new \Exception('Only deletion records read. It is not possible to determine the reference date with only deletion. Please increase the rule limit to include non deletion records.'); + } else { + // If only deletion without new or modified record, we send no result. We wait for new or modified record. + // Otherwise we will read the deleted record until a new or modified record is read because Sugar doesn't return modified date for deleted record. + return array(); + } + } + + // Build the results + foreach ($records as $record) { + foreach ($param['fields'] as $field) { // Sugar returns multilist value as array if ( !empty($record->$field) @@ -354,7 +416,12 @@ public function read($param) ) { // Some fields can be an object like teamname field if (is_object($record->$field[0])) { - $record->$field = $record->$field[0]->name; + $fieldObjectList = array(); + // Get all ids of the object list + foreach($record->$field as $fieldObject) { + $fieldObjectList[] = $fieldObject->id; + } + $record->$field = implode(',', $fieldObjectList); } else { $record->$field = implode(',', $record->$field); } @@ -366,17 +433,6 @@ public function read($param) $result[$record->id]['date_modified'] = end($records)->date_modified; } } - - // Error if only deletion records read - if ($onlyDeletion) { - if (count($result) >= $param['limit']) { - throw new \Exception('Only deletion records read. It is not possible to determine the reference date with only deletion. Please increase the rule limit to include non deletion records.'); - } else { - // If only deletion without new or modified record, we send no result. We wait for new or modified record. - // Otherwise we will read the deleted record until a new or modified record is read because Sugar doesn't return modified date for deleted record. - return array(); - } - } } return $result; } diff --git a/templates/JobScheduler/crontab_list.html.twig b/templates/JobScheduler/crontab_list.html.twig index 4c7085863..bc7580b92 100644 --- a/templates/JobScheduler/crontab_list.html.twig +++ b/templates/JobScheduler/crontab_list.html.twig @@ -44,6 +44,8 @@ + + @@ -54,7 +56,9 @@ {{ value.command }} {{ value.period }} - {{ value.description }} + {{ value.description }} + {{ value.runningInstances }} + {{ value.maxInstances }} {{ value.lastUse|date("d/m/Y H:i:s", timezone) }} {{ value.nextRun|date("d/m/Y H:i:s", timezone) }} {{ value.enable }} diff --git a/templates/JobScheduler/edit_crontab.html.twig b/templates/JobScheduler/edit_crontab.html.twig index e492059a8..034bcc7b7 100644 --- a/templates/JobScheduler/edit_crontab.html.twig +++ b/templates/JobScheduler/edit_crontab.html.twig @@ -18,6 +18,20 @@ {{ form_widget( edit_form.description) }} +
+
+ +
+ {{ form_widget( edit_form.runningInstances) }} +
+
+
+
+ +
+ {{ form_widget( edit_form.maxInstances) }} +
+
diff --git a/templates/JobScheduler/show_crontab.html.twig b/templates/JobScheduler/show_crontab.html.twig index 8fea2b9dd..b8a883525 100644 --- a/templates/JobScheduler/show_crontab.html.twig +++ b/templates/JobScheduler/show_crontab.html.twig @@ -1,54 +1,118 @@ {% extends 'base.html.twig' %} -{% block titlesm %} {{ 'crontab.title'|trans }}{% endblock %} +{% block titlesm %} + {{ 'crontab.title'|trans }} +{% endblock %} {% block body %} -
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - > - - - - - - - - - - - - - - - -
{{ entity.id }}
{{ entity.createdAt|date('Y-m-d H:i:s') }}
{{ entity.updatedAt|date('d/m/Y') }}
{{ entity.command }}
{{ entity.arguments }}
{{ entity.description }}
{{ entity.runningInstances }}
{{ entity.maxInstances }}
{{ entity.number }}
{% if entity.enable %} true {% else %} false {% endif %}
-
-
-
-{% endblock %} \ No newline at end of file +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + > + + + + + + + + + + + + + + + +
+ + {{ entity.id }}
+ + {{ entity.createdAt|date('Y-m-d H:i:s') }}
+ + {{ entity.updatedAt|date('d/m/Y') }}
+ + {{ entity.command }}
+ + {{ entity.arguments }}
+ + {{ entity.description }}
+ + {{ entity.runningInstances }}
+ + {{ entity.maxInstances }}
+ + {{ entity.number }}
+ + + {% if entity.enable %} + true + {% else %} + false + {% endif %} +
+
+
+ + +

{{ 'crontab.results'|trans }}

+
{{ "Sorting: Default" }}
+ + + + + + + + + + + + + + + {% for result in entityResult %} + + + + + + + + + + {% endfor %} + +
{{ 'jobscheduler.id'|trans }}{{ 'jobscheduler.date_created'|trans }}{{ 'jobscheduler.date_modified'|trans }}{{ 'jobscheduler.output'|trans }}{{ 'jobscheduler.run_at'|trans }}{{ 'jobscheduler.run_time'|trans }}{{ 'jobscheduler.status_code'|trans }}
{{ result.id }}{{ result.createdAt|date('Y-m-d H:i:s') }}{{ result.updatedAt|date('Y-m-d H:i:s') }}{{ result.output|raw }}{{ result.runAt|date('Y-m-d H:i:s') }}{{ result.runTime }}{{ result.statusCode }}
+
+ +
+ + + + {% endblock %} diff --git a/translations/messages.en.yml b/translations/messages.en.yml index a559b7cab..4bcce1d04 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -758,7 +758,12 @@ email_alert: Kind regards, jobscheduler: + id: Id command: Command + output: Output + run_at: Run at + run_time: Run time + status_code: Status code date_created: Date created active: Active list: List @@ -888,6 +893,9 @@ crontab: disableAllCrons: Your crontabs have been disabled enableAllCrons: Your crontabs have been enable create: Create crontab + running_instances: Running instances + max_instances: Max instances + results: Results massdisable: Disable all tasks massenable: Enable all tasks enableCron: On diff --git a/webpack.config.js b/webpack.config.js index cfe9678e3..4a5f39861 100755 --- a/webpack.config.js +++ b/webpack.config.js @@ -27,6 +27,7 @@ Encore .addEntry('rulelist', './assets/js/rulelist.js') .addEntry('fiche', './assets/js/fiche.js') .addEntry('filter', './assets/js/filter.js') + .addEntry('crontab', './assets/js/crontab.js') // enables the Symfony UX Stimulus bridge (used in assets/bootstrap.js) .enableStimulusBridge('./assets/controllers.json') diff --git a/yarn.lock b/yarn.lock index b844a4870..023a49e57 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1506,7 +1506,6 @@ acorn-import-assertions@^1.7.6: resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== - acorn@^8.0.5, acorn@^8.5.0, acorn@^8.7.1: version "8.8.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73" @@ -4017,7 +4016,7 @@ minimist-options@4.1.0: is-plain-obj "^1.1.0" kind-of "^6.0.3" -minimist@^1.2.0, minimist@^1.2.5: +minimist@^1.2.0: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== From 332704155961b3f9cbd57809e13c24ec384fc8c6 Mon Sep 17 00:00:00 2001 From: Myddleware Date: Thu, 11 Jan 2024 10:07:23 +0100 Subject: [PATCH 02/88] Hotfix (#1110) * Hotfix333 (#1064) * SuiteCRM : Add field filecontents for notes module Signed-off-by: myddleware * Fix : Catch all errors in formula Signed-off-by: myddleware * Fix : catch formula errors Signed-off-by: myddleware * Fix : remove slashes to compare data Signed-off-by: myddleware * Manage null value for comparaison Signed-off-by: myddleware * Fix bug on relate when a several source ids exist for a target id and one of them is deleted Signed-off-by: myddleware * Fix bug target id Signed-off-by: myddleware --------- Signed-off-by: myddleware * Fix : Do not search duplicates on an empty field Signed-off-by: myddleware * Change index on job table Signed-off-by: myddleware * Rebuild all document table indexes Fix home page queries according to the new indexes Signed-off-by: myddleware * Change index on contacts for document search purpose Signed-off-by: myddleware * Feat : manage the cancellation of the document using formula and variable mdw_cancel_document Signed-off-by: myddleware * Feat : improve performance rerunerror job Signed-off-by: myddleware * Moolde : manage error return Signed-off-by: myddleware * Change hotfix version Signed-off-by: myddleware * Fix : Remove user filter from document search query Signed-off-by: myddleware * Solution : get logic to calculate ref date in getReferenceCall even if no result Sendinblue : event read function - change logic to not use offset anymore Signed-off-by: myddleware * Remove unuse method Signed-off-by: myddleware * Fix : No rerun if no document Signed-off-by: myddleware * Change return Signed-off-by: myddleware --------- Signed-off-by: myddleware --- .env | 2 +- src/Controller/FilterController.php | 5 +- src/Controller/FluxController.php | 7 +- src/Entity/Job.php | 2 +- src/Manager/DocumentManager.php | 56 +++++---- src/Manager/JobManager.php | 9 +- src/Repository/DocumentRepository.php | 9 +- src/Solutions/moodle.php | 2 +- src/Solutions/sendinblue.php | 170 ++++++++++---------------- src/Solutions/solution.php | 16 +-- src/Solutions/suitecrm.php | 12 +- 11 files changed, 134 insertions(+), 156 deletions(-) diff --git a/.env b/.env index 5e73d9fa9..6241cee00 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -MYDDLEWARE_VERSION=3.3.4a +MYDDLEWARE_VERSION=3.3.4b APP_SECRET=Thissecretisnotsosecretchangeit APP_ENV=prod diff --git a/src/Controller/FilterController.php b/src/Controller/FilterController.php index 604f51757..afccfab1c 100644 --- a/src/Controller/FilterController.php +++ b/src/Controller/FilterController.php @@ -731,16 +731,13 @@ protected function searchDocuments($data, $page = 1, $limit = 1000) { document.type, document.attempt, document.global_status, - users.username, rule.name as rule_name, rule.module_source, rule.module_target, rule.id as rule_id FROM document INNER JOIN rule - ON document.rule_id = rule.id - INNER JOIN users - ON document.created_by = users.id " + ON document.rule_id = rule.id " .$join. " WHERE document.deleted = 0 " diff --git a/src/Controller/FluxController.php b/src/Controller/FluxController.php index 1c5be7e4c..9594546fd 100644 --- a/src/Controller/FluxController.php +++ b/src/Controller/FluxController.php @@ -504,22 +504,17 @@ protected function searchDocuments($data, $page = 1, $limit = 1000) { document.type, document.attempt, document.global_status, - users.username, rule.name as rule_name, rule.id as rule_id FROM document INNER JOIN rule - ON document.rule_id = rule.id - INNER JOIN users - ON document.created_by = users.id " + ON document.rule_id = rule.id " .$join. " WHERE document.deleted = 0 " .$where. " ORDER BY document.date_modified DESC" ." LIMIT ". $limit; - - $stmt = $this->getDoctrine()->getManager()->getConnection()->prepare($query); // Add parameters to the query // Source content diff --git a/src/Entity/Job.php b/src/Entity/Job.php index 3583480b6..3da3c3e50 100755 --- a/src/Entity/Job.php +++ b/src/Entity/Job.php @@ -33,7 +33,7 @@ /** * @ORM\Entity(repositoryClass="App\Repository\JobRepository") * @ORM\Table(name="job", indexes={ - * @ORM\Index(name="index_status", columns={"status"}) + * @ORM\Index(name="index_status_begin", columns={"status","begin"}) *}) */ class Job diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 55fda7552..7030f2dbd 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -936,7 +936,14 @@ public function transformDocument(): bool try { // Transformation des données et insertion dans la table target $transformed = $this->updateTargetTable(); - if ($transformed) { + if (!empty($transformed)) { + // If the value mdw_cancel_document is found in the target data of the document after transformation we cancel the document + if (array_search('mdw_cancel_document',$transformed) !== false) { + $this->message .= 'The document contains the value mdw_cancel_document. '; + $this->typeError = 'W'; + $this->updateStatus('Cancel'); + return false; + } // If the type of this document is Create and if the field Myddleware_element_id isn't empty, // it means that the target ID is mapped in the rule field // In this case, we force the document's type to Update because Myddleware will update the record into the target application @@ -1062,7 +1069,6 @@ public function getTargetDataDocument(): bool if (!empty($target[$duplicate_field])) { $searchFields[$duplicate_field] = $target[$duplicate_field]; } - $searchFields[$duplicate_field] = $target[$duplicate_field]; } if (!empty($searchFields)) { $history = $this->getDocumentHistory($searchFields); @@ -1227,10 +1233,12 @@ protected function checkNoChange($history): bool // If one is different we stop the function if (!empty($this->ruleFields)) { foreach ($this->ruleFields as $field) { - if ( - trim($history[$field['target_field_name']]) != trim($target[$field['target_field_name']]) - ) { - // We check if both are empty not depending of the type 0 = "" + if (stripslashes(trim($history[$field['target_field_name']])) != stripslashes(trim($target[$field['target_field_name']]))) { + // Null text is considered as empty for comparaison + if ($target[$field['target_field_name']] == 'null') { + $target[$field['target_field_name']] = ''; + } + // We check if both are empty not depending of the type 0 = "" if ( empty($history[$field['target_field_name']]) and empty($target[$field['target_field_name']]) @@ -1450,7 +1458,7 @@ protected function updateHistoryTable($dataTarget): bool } // Mise à jour de la table des données cibles - protected function updateTargetTable(): bool + protected function updateTargetTable() { try { // Loop on every target field and calculate the value @@ -1491,14 +1499,14 @@ protected function updateTargetTable(): bool throw new \Exception('No target data found. Failed to create target data. '); } - return true; + return $targetField; } catch (Exception $e) { $this->message .= 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; $this->logger->error($this->message); } - return false; + return null; } /* @@ -2343,8 +2351,8 @@ protected function getTargetId($ruleRelationship, $record_id) if ('-1' == $direction) { $sqlParams = " SELECT source_id record_id, - document.id document_id, - document.type document_type + GROUP_CONCAT(DISTINCT document.id) document_id, + GROUP_CONCAT(DISTINCT document.type) types FROM document WHERE document.rule_id = :ruleRelateId @@ -2355,13 +2363,14 @@ protected function getTargetId($ruleRelationship, $record_id) document.global_status = 'Close' OR document.status = 'No_send' ) - ORDER BY date_created DESC + GROUP BY source_id + HAVING types NOT LIKE '%D%' LIMIT 1"; } elseif ('1' == $direction) { $sqlParams = " SELECT target_id record_id, - document.id document_id, - document.type document_type + GROUP_CONCAT(DISTINCT document.id) document_id, + GROUP_CONCAT(DISTINCT document.type) types FROM document WHERE document.rule_id = :ruleRelateId @@ -2372,7 +2381,8 @@ protected function getTargetId($ruleRelationship, $record_id) document.global_status = 'Close' OR document.status = 'No_send' ) - ORDER BY date_created DESC + GROUP BY target_id + HAVING types NOT LIKE '%D%' LIMIT 1"; } else { throw new \Exception('Failed to find the direction of the relationship with the rule_id '.$ruleRelationship['field_id'].'. '); @@ -2420,16 +2430,18 @@ protected function getTargetId($ruleRelationship, $record_id) $stmt->bindValue(':record_id', $record_id); $result = $stmt->executeQuery(); $result = $result->fetchAssociative(); - } - if (!empty($result['record_id'])) { - // If the latest valid document sent is a deleted one, then the target id can't be use as the record has been deleted from the target solution - if ('D' == $result['document_type']) { - return null; - } + // In cas of several document found we get only the last one + if ( + !empty($result['document_id']) + AND strpos($result['document_id'], ',') + ) { + $result['document_id'] = end(explode(',',$result['document_id'])); + } + } + if (!empty($result['record_id'])) { return $result; } - return null; } catch (\Exception $e) { $this->message .= 'Error getTargetId : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; diff --git a/src/Manager/JobManager.php b/src/Manager/JobManager.php index 450f4fdf0..0a3d943af 100644 --- a/src/Manager/JobManager.php +++ b/src/Manager/JobManager.php @@ -269,7 +269,7 @@ public function sendDocuments() } // Ecriture dans le système source et mise à jour de la table document - public function runError($limit, $attempt) + public function runError($limit, $attempt) { try { $ruleId = ''; @@ -282,14 +282,17 @@ public function runError($limit, $attempt) global_status = 'Error' AND deleted = 0 AND attempt <= :attempt - ORDER BY ruleorder.order ASC, source_date_modified ASC + ORDER BY ruleorder.order ASC, document.rule_id, source_date_modified ASC LIMIT $limit"; $stmt = $this->connection->prepare($sqlParams); $stmt->bindValue('attempt', $attempt); $result = $stmt->executeQuery(); $documentsError = $result->fetchAllAssociative(); if (!empty($documentsError)) { - // include_once 'rule.php'; + $ruleId = null; + $this->ruleManager->setJobId($this->id); + $this->ruleManager->setManual($this->manual); + $this->ruleManager->setApi($this->api); foreach ($documentsError as $documentError) { // Load the rule only if it has changed if ($ruleId != $documentError['rule_id']) { diff --git a/src/Repository/DocumentRepository.php b/src/Repository/DocumentRepository.php index cc02bf289..818704427 100644 --- a/src/Repository/DocumentRepository.php +++ b/src/Repository/DocumentRepository.php @@ -176,16 +176,11 @@ public function countTransferRule(User $user = null) ->select('COUNT(d) as nb, rule.name') ->join('d.rule', 'rule') ->andWhere('d.deleted = 0') - ->andWhere('d.globalStatus = :close') - ->setParameter('close', 'Close') + ->andWhere('d.status = :send') + ->setParameter('send', 'Send') ->groupBy('rule.name') ->orderBy('nb', 'DESC'); - if ($user && !$user->isAdmin()) { - $qb->andWhere('d.createdBy = :user') - ->setParameter('user', $user); - } - return $qb->getQuery()->getResult(); } diff --git a/src/Solutions/moodle.php b/src/Solutions/moodle.php index 0d20bb4b8..87de84960 100644 --- a/src/Solutions/moodle.php +++ b/src/Solutions/moodle.php @@ -484,7 +484,7 @@ public function updateData($param): array // Check if there is a warning if ( - !empty($xml) + !empty($xml->SINGLE->KEY) AND $xml->count() != 0 // Empty xml AND $xml->SINGLE->KEY->attributes()->__toString() == 'warnings' AND !empty($xml->SINGLE->KEY->MULTIPLE->SINGLE->KEY[3]) diff --git a/src/Solutions/sendinblue.php b/src/Solutions/sendinblue.php index a9e0eb3f1..8ce2f1596 100644 --- a/src/Solutions/sendinblue.php +++ b/src/Solutions/sendinblue.php @@ -183,12 +183,11 @@ public function read($param) $result = []; // Function are differents depending on the type of record we read from Sendinblue switch ($param['module']) { - case 'transactionalEmailActivity': + case 'transactionalEmailActivity': // event is required if (empty($param['ruleParams']['event'])) { throw new \Exception('No event selected. Please select an event on your rule. '); } - // As we build the id (it doesn't exist in Sendinblue), we add it to the param field $param['fields'][] = 'id'; @@ -219,50 +218,18 @@ public function read($param) } // Set call parameters when we read transactional email using reference date } else { - // Because offset is managed in this function, we don't need the +1 in the rule param limit - if ($param['limit'] > 1) { - --$param['limit']; - } - $event = $param['ruleParams']['event']; // if simulation, we init the date start to today - 30 days if ('simulation' == $param['call_type']) { - $dateStartObj = new \DateTime('NOW'); - $dateStartObj->sub(new \DateInterval('P30D')); + $dateRefObj = new \DateTime('NOW'); + $dateRefObj->sub(new \DateInterval('P30D')); } else { - $dateStartObj = new \DateTime($param['date_ref']); + // TO BE CHANGED + $dateRefObj = date_create($param['date_ref'], new \DateTimeZone("Europe/Paris")); } // Only date (not datetime) are used to filter transaction email activity - $dateStart = $dateStartObj->format('Y-m-d'); - $dateEnd = $this->getDateEnd($dateStartObj); - - // Make sure that offset exists - if (empty($param['ruleParams']['offset'])) { - $param['ruleParams']['offset'] = 0; - } else { - // Change offset if the parameter exists on the rule - $offset = $param['ruleParams']['offset']; - } - - // Max call limit = 100 - // Nb call depend on limit param - if ($param['limit'] > $this->limitEmailActivity) { - $nbCall = floor($param['limit'] / $this->limitEmailActivity); - // Add 1 call if needid (using modulo function) - if ($param['limit'] % $this->limitEmailActivity != 0) { - $limitLastCall = $param['limit'] % $this->limitEmailActivity; - ++$nbCall; - } - $limitCall = $this->limitEmailActivity; - } else { - // If rule limit < $limitCall , there is no need to call more records - $limitCall = $param['limit']; - } - - // If rule limit modulo limitcall is null then last call will be equal to limitcall - if ($param['limit'] % $this->limitEmailActivity == 0) { - $limitLastCall = $this->limitEmailActivity; - } + $dateStart = $dateRefObj->format('Y-m-d'); + $dateEnd = $dateRefObj->format('Y-m-d'); } $apiInstance = new \SendinBlue\Client\Api\TransactionalEmailsApi(new \GuzzleHttp\Client(), $this->config); @@ -270,66 +237,47 @@ public function read($param) if (false !== $contactRequested) { $apiContactInstance = new \SendinBlue\Client\Api\ContactsApi(new \GuzzleHttp\Client(), $this->config); } - - for ($i = 1; $i <= $nbCall; ++$i) { - // The limit can be different for the last call (in case of several call) - if ( - $i == $nbCall - and $nbCall > 1 - ) { - $limitCall = $limitLastCall; - } - $resultApi = $apiInstance->getEmailEventReport($limitCall, $offset, $dateStart, $dateEnd, null, null, $event, null, $messageId, null, 'asc'); - - if (!empty(current($resultApi)['events'])) { - $events = current($resultApi)['events']; - - // Add records read into result array - foreach ($events as $record) { - ++$offset; - $record['id'] = $record['messageId'].'__'.$record['event']; - - // if the contactid is requested, we use the email to get it - if (false !== $contactRequested) { - try { - $resultContactApi = $apiContactInstance->getContactInfo($record['email']); - if (!empty(current($resultContactApi)['id'])) { - $record['contactId'] = current($resultContactApi)['id']; - } - } catch (\Exception $e) { - $record['contactId'] = ''; - } - } - - $records[] = $record; - // IF the date change, we set the offset to 0 (because filter is only on date and not dateTime - /// Date start we be changed with the date of the current record - // Date ref will also be changed - $dateRecordObj = new \DateTime($record['date']); - if ( - empty($param['query']['id']) // No offset management if search by id - and $dateRecordObj->format('Y-m-d') != $dateStartObj->format('Y-m-d') - ) { - $dateStartObj = $dateRecordObj; - $dateStart = $dateStartObj->format('Y-m-d'); - $dateEnd = $this->getDateEnd($dateStartObj); - - // Offset = 1 not 0 becquse we have alredy read the first record of the day - $offset = 1; - } - } - - // If the limit hasn't been reached, it means there is no more result to read. We stop the read action. - if (count($events) < $this->limitEmailActivity) { - break; - } - } - } - // Save the offset value on the rule - // No offset management if search by id - if (empty($param['query']['id'])) { - $result['ruleParams'][] = ['name' => 'offset', 'value' => $offset]; - } + $offset = 0; + $exit = false; + do { + $resultApi = $apiInstance->getEmailEventReport($this->limitEmailActivity, $offset, $dateStart, $dateEnd, null, null, $event, null, $messageId, null, 'desc'); + + // Exit the loop if no result + if (empty(current($resultApi)['events'])) { + $exit = true; + break; + } + // Add records read into result array + $events = current($resultApi)['events']; + foreach ($events as $record) { + $offset++; + $record['id'] = $record['messageId'].'__'.$record['event']; + + // if the contactid is requested, we use the email to get it + if (false !== $contactRequested) { + try { + $resultContactApi = $apiContactInstance->getContactInfo($record['email']); + if (!empty(current($resultContactApi)['id'])) { + $record['contactId'] = current($resultContactApi)['id']; + } + } catch (\Exception $e) { + $record['contactId'] = ''; + } + } + + $dateRecordObj = \DateTime::createFromFormat(DATE_RFC3339_EXTENDED, $record['date']); + if ($dateRefObj->format('U') > $dateRecordObj->format('U')) { + $exit = true; + break; + } + $records[] = $record; + } + + // If the limit hasn't been reached, it means there is no more result to read. We stop the read action. + if (count($events) < $this->limitEmailActivity) { + $exit = true; + } + } while (!$exit); break; case 'contacts': $apiInstance = new \SendinBlue\Client\Api\ContactsApi(new \GuzzleHttp\Client(), $this->config); @@ -423,11 +371,27 @@ public function read($param) } } } - return $result; } - protected function getDateEnd($dateObj): string + // Method de find the date ref after a read call + protected function getReferenceCall($param, $result) + { + if ($param['module'] == 'transactionalEmailActivity') { + $currentDate = new DateTime(); + $dateRefObj = new \DateTime($param['date_ref']); + // If date ref < today then we force the referenece date to date+1 at midnight + if ($dateRefObj->format('Y-m-d') < $currentDate->format('Y-m-d')) { + $dateRefObj->modify('+1 day'); + $dateRefObj->setTime(0, 0, 0); + return $dateRefObj->format('Y-m-d H:i:s'); + } + } + // Call parent function (calsse solution + return parent::getReferenceCall($param, $result); + } + +/* protected function getDateEnd($dateObj): string { $dateEndObj = clone $dateObj; $dateEndObj->add(new \DateInterval('P30D')); @@ -438,7 +402,7 @@ protected function getDateEnd($dateObj): string } return $dateEndObj->format('Y-m-d'); - } + } */ //fonction for get all your transactional email activity public function EmailTransactional($param): \SendinBlue\Client\Model\GetEmailEventReport diff --git a/src/Solutions/solution.php b/src/Solutions/solution.php index b0ac2cc75..4cbdbd165 100644 --- a/src/Solutions/solution.php +++ b/src/Solutions/solution.php @@ -341,17 +341,15 @@ public function readData($param) break; } } - - // Calculate the reference call - $result['date_ref'] = $this->getReferenceCall($param, $result); - if (empty($result['date_ref'])) { - throw new \Exception('Failed to get the reference call.'); - } } else { // Init values if no result $result['count'] = 0; - $result['date_ref'] = $param['date_ref']; } + // Calculate the reference call + $result['date_ref'] = $this->getReferenceCall($param, $result); + if (empty($result['date_ref'])) { + throw new \Exception('Failed to get the reference call.'); + } } catch (\Exception $e) { $result['error'] = (!empty($param['rule']['id']) ? 'Error in rule '.$param['rule']['id'].' : ' : 'Error : ').$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } @@ -1037,6 +1035,10 @@ protected function getParamLogin($connId): array // Method de find the date ref after a read call protected function getReferenceCall($param, $result) { + // Keep the same date ref if no result + if (empty($result['count'])) { + return $param['date_ref']; + } // Result is sorted, the last one is the oldest one return end($result['values'])['date_modified']; } diff --git a/src/Solutions/suitecrm.php b/src/Solutions/suitecrm.php index f6f5dd3bd..20c981882 100644 --- a/src/Solutions/suitecrm.php +++ b/src/Solutions/suitecrm.php @@ -352,7 +352,17 @@ public function get_module_fields($module, $type = 'source', $param = null): arr } } } - + // Add field filecontents for notes module + if ($module == 'Notes') { + $this->moduleFields['filecontents'] = [ + 'label' => 'File contents', + 'type' => 'text', + 'type_bdd' => 'text', + 'required' => 0, + 'required_relationship' => 0, + 'relate' => false, + ]; + } return $this->moduleFields; } catch (\Exception $e) { return false; From fbc263b8dbaf177d47084718e98c1f9a4489cbf8 Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 11 Jan 2024 10:14:21 +0100 Subject: [PATCH 03/88] Init version Signed-off-by: myddleware --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 6241cee00..1683b5eb6 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -MYDDLEWARE_VERSION=3.3.4b +MYDDLEWARE_VERSION=3.4.0a APP_SECRET=Thissecretisnotsosecretchangeit APP_ENV=prod From 18b39f4af7317a39932d07e3492e25af99a7c44f Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 12 Jan 2024 16:55:50 +0100 Subject: [PATCH 04/88] Prestashop : manage reference date if no result Signed-off-by: myddleware --- src/Solutions/prestashop.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Solutions/prestashop.php b/src/Solutions/prestashop.php index b2cc58c5f..8ed37b9bf 100644 --- a/src/Solutions/prestashop.php +++ b/src/Solutions/prestashop.php @@ -593,7 +593,10 @@ public function read($param): ?array */ protected function getReferenceCall($param, $result) { - // IF the reference is a date + // Keep the same date ref if no result + if (empty($result['count'])) { + return $param['date_ref']; + } if ($this->referenceIsDate($param['module'])) { // Add 1 second to the date ref because the read function is a >= not a > $date = new \DateTime(end($result['values'])['date_modified']); From 9b80281b137fc3b93cc0c05d98ad8715073dd45c Mon Sep 17 00:00:00 2001 From: myddleware Date: Mon, 22 Jan 2024 11:56:57 +0100 Subject: [PATCH 05/88] Airtable : New API version + fix Signed-off-by: myddleware --- src/Solutions/airtable.php | 63 ++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/src/Solutions/airtable.php b/src/Solutions/airtable.php index 8a1b879c7..6006dbcc1 100644 --- a/src/Solutions/airtable.php +++ b/src/Solutions/airtable.php @@ -57,7 +57,7 @@ class airtablecore extends solution * This is initialised to 'Contacts' by default as I've assumed that would be the most common possible value. * However, this can of course be changed to any table value already present in your Airtable base. */ - protected array $tableName = []; + protected array $tableName; /** * Can't be greater than 100. @@ -106,7 +106,11 @@ public function login($paramConnexion) $this->token = $this->paramConnexion['apikey']; // We test the connection to the API with a request on Module/Table (change the value of tableName to fit your needs) $client = HttpClient::create(); - $options = ['auth_bearer' => $this->token]; + $options = [ + 'headers' => [ + 'Authorization' => 'Bearer ' . $this->token, + ], + ]; $response = $client->request('GET', $this->airtableURL.$this->projectID.'/'.$this->tableName[$this->projectID], $options); $statusCode = $response->getStatusCode(); $contentType = $response->getHeaders()['content-type'][0]; @@ -274,9 +278,16 @@ public function readData($param): array ++$currentCount; foreach ($param['fields'] as $field) { if (!empty($record['fields'][$field])) { - // If teh value is an array (relation), we take the first entry + // If the value is an array (relation), we take the first entry if (is_array($record['fields'][$field])) { - $result['values'][$record['id']][$field] = $record['fields'][$field][0]; + // if the array has a length of 1, we take the first entry + if (count($record['fields'][$field]) === 1) { + $result['values'][$record['id']][$field] = $record['fields'][$field][0]; + } else { + // if the array has a larger length than 1 we convert the array into a string separated by comma and space + $result['values'][$record['id']][$field] = implode(',', $record['fields'][$field]); + } + } else { $result['values'][$record['id']][$field] = $record['fields'][$field]; } @@ -403,12 +414,6 @@ public function upsert(string $method, array $param): array ++$i; continue; } - - // If no data we skip this record - if (empty($data)) { - unset($records[$idDoc]); - continue; - } $body['records'][$i]['fields'] = $data; @@ -441,7 +446,7 @@ public function upsert(string $method, array $param): array ++$i; } - // Airtable field can contains space which is not compatible in Myddleware. + // Airtable fiueld can contains space which is not compatible in Myddleware. // Because we replace space by ___ in Myddleware, we change ___ to space before sending data to Airtable if (!empty($body['records'])) { foreach ($body['records'] as $keyRecord => $record) { @@ -479,25 +484,23 @@ public function upsert(string $method, array $param): array $content = $response->toArray(); if (!empty($content)) { $i = 0; - if (!empty($records)) { - foreach ($records as $idDoc => $data) { - $record = $content['records'][$i]; - if (!empty($record['id'])) { - $result[$idDoc] = [ - 'id' => $record['id'], - 'error' => false, - ]; - } else { - $result[$idDoc] = [ - 'id' => '-1', - 'error' => 'Failed to send data. Message from Airtable : '.print_r($content['records'][$i], true), - ]; - } - ++$i; - // Modification du statut du flux - $this->updateDocumentStatus($idDoc, $result[$idDoc], $param); - } - } + foreach ($records as $idDoc => $data) { + $record = $content['records'][$i]; + if (!empty($record['id'])) { + $result[$idDoc] = [ + 'id' => $record['id'], + 'error' => false, + ]; + } else { + $result[$idDoc] = [ + 'id' => '-1', + 'error' => 'Failed to send data. Message from Airtable : '.print_r($content['records'][$i], true), + ]; + } + ++$i; + // Modification du statut du flux + $this->updateDocumentStatus($idDoc, $result[$idDoc], $param); + } } else { throw new Exception('Failed to send the record but no error returned by Airtable. '); } From 9d8acb7c3b2fdb533baca864da408d0dc33645de Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 24 Jan 2024 02:14:51 +0100 Subject: [PATCH 06/88] Change status from filter to no_send when a document is changed from UPDATE in a CREATE rule after the check duplicate process. Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 7030f2dbd..f643eb5c0 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -905,7 +905,7 @@ public function checkParentDocument(): bool // Check child in a second time to avoid to run a query each time if (!$this->isChild()) { $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; - $this->updateStatus('Filter'); + $this->updateStatus('No_send'); // In case we flter the document, we return false to stop the process when this method is called in the rerun process return false; } From 6f8e5cd49ef69762316e69ddf8ec19a82f1bcb0b Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 8 Feb 2024 21:05:34 +0100 Subject: [PATCH 07/88] Change status if rule in crate_only and document is update Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index f643eb5c0..c9827d980 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -905,7 +905,7 @@ public function checkParentDocument(): bool // Check child in a second time to avoid to run a query each time if (!$this->isChild()) { $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; - $this->updateStatus('No_send'); + $this->updateStatus('Filter'); // In case we flter the document, we return false to stop the process when this method is called in the rerun process return false; } @@ -1111,7 +1111,8 @@ public function getTargetDataDocument(): bool // Check child in a second time to avoid to run a query each time if (!$this->isChild()) { $this->message .= 'Rule mode only allows to create data. Filter because this document updates data.'; - $this->updateStatus('Filter'); + $this->updateStatus('No_send'); + $this->updateTargetId($history['id']); // In case we flter the document, we return false to stop the process when this method is called in the rerun process return false; } From 7efdb3ea14a029342d8e43ff46e5e834423003fe Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 8 Feb 2024 22:16:24 +0100 Subject: [PATCH 08/88] Manage rule in UPDATE_ONLY Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index c9827d980..520c0568c 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1136,12 +1136,14 @@ public function getTargetDataDocument(): bool ) { $this->checkNoChange($history); } - // Error if rule mode is update only and the document is a creation + // Cancel if rule mode is update only and the document is a creation if ( 'C' == $this->documentType and 'U' == $this->ruleMode ) { - throw new \Exception('The document is a creation but the rule mode is UPDATE ONLY. '); + $this->message .= 'The document is a creation but the rule mode is UPDATE ONLY.'; + $this->updateStatus('Filter'); + return false; } } catch (\Exception $e) { $this->message .= $e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; @@ -1152,10 +1154,8 @@ public function getTargetDataDocument(): bool $this->updateStatus('Error_checking'); } $this->logger->error($this->message); - return false; } - return true; } From acb6a315395b9b8d6a466518dcdb66a4a89e9843 Mon Sep 17 00:00:00 2001 From: myddleware Date: Sun, 11 Feb 2024 00:14:47 +0100 Subject: [PATCH 09/88] Fix bug for rerun record by id Signed-off-by: myddleware --- src/Command/MassActionCommand.php | 2 +- src/Command/ReadRecordCommand.php | 6 +- src/Manager/JobManager.php | 87 ++++++++++------ src/Manager/RuleManager.php | 165 +----------------------------- 4 files changed, 61 insertions(+), 199 deletions(-) diff --git a/src/Command/MassActionCommand.php b/src/Command/MassActionCommand.php index 12ec28b0d..d22a6f8ab 100644 --- a/src/Command/MassActionCommand.php +++ b/src/Command/MassActionCommand.php @@ -98,7 +98,7 @@ protected function configure() ->setDescription('Action massive sur les flux') ->addArgument('action', InputArgument::REQUIRED, 'Action (rerun, cancel, remove, restore or changeStatus)') ->addArgument('dataType', InputArgument::REQUIRED, 'Data type (rule or document)') - ->addArgument('ids', InputArgument::REQUIRED, 'Rule or document ids') // id séparés par des ";" + ->addArgument('ids', InputArgument::REQUIRED, 'Rule or document ids') // id séparés par des "," ->addArgument('forceAll', InputArgument::OPTIONAL, 'Set Y to process action on all documents (not only open and error ones)') ->addArgument('fromStatus', InputArgument::OPTIONAL, 'Get all document with this status(Only with changeStatus action)') ->addArgument('toStatus', InputArgument::OPTIONAL, 'Set this status (Only with changeStatus action)') diff --git a/src/Command/ReadRecordCommand.php b/src/Command/ReadRecordCommand.php index 96ad0ee6f..1ce59c688 100644 --- a/src/Command/ReadRecordCommand.php +++ b/src/Command/ReadRecordCommand.php @@ -93,11 +93,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $output->writeln('1;'.$this->jobManager->getId()); // This is requiered to display the log (link creation with job id) when the job is run manually - // If the query is by id, we use the flag that leads to massIdRerun() - // Otherwise we use the regular rerun - if ($filterQuery === 'id') { - $this->jobManager->readRecord($rule, $filterQuery, $filterValues, 1); - }else $this->jobManager->readRecord($rule, $filterQuery, $filterValues); + $this->jobManager->readRecord($rule, $filterQuery, $filterValues); // Close job if it has been created $responseCloseJob = $this->jobManager->closeJob(); diff --git a/src/Manager/JobManager.php b/src/Manager/JobManager.php index 0a3d943af..44271eb3c 100644 --- a/src/Manager/JobManager.php +++ b/src/Manager/JobManager.php @@ -528,13 +528,18 @@ public function massAction($action, $dataType, $ids, $forceAll, $fromStatus, $to return true; } - // Fonction permettant d'annuler massivement des documents - - // In order to add extra components to the function without disturbing its regular use, we added a flag argument. - // This $usesDocumentIds flag is either null or 1 - public function readRecord($ruleId, $filterQuery, $filterValues, $usesDocumentIds = null): bool + // Function to create one of several document using a query + public function readRecord($ruleId, $filterQuery, $filterValues): bool { try { + $responseFilter = []; + $responseCheckPredecessor = []; + $responseCheckParent = []; + $responseTransform = []; + $responseGetTargetData = []; + $responseSend = []; + $documentIds = []; + // Get the filter values $filterValuesArray = explode(',', $filterValues); if (empty($filterValuesArray)) { @@ -557,11 +562,6 @@ public function readRecord($ruleId, $filterQuery, $filterValues, $usesDocumentId $this->ruleManager->setManual($this->manual); $this->ruleManager->setApi($this->api); - // We create an array that will match the initial structure of the function - if ($usesDocumentIds === 1) { - $arrayOfDocumentIds = []; - } - // Try to read data for each values foreach ($filterValuesArray as $value) { // Generate documents @@ -569,38 +569,67 @@ public function readRecord($ruleId, $filterQuery, $filterValues, $usesDocumentId if (!empty($documents->error)) { throw new Exception($documents->error); } - // We assign the id to an id section of the array - if ($usesDocumentIds === 1 && !empty($documents)) { - $arrayOfDocumentIds[] = $documents[0]->id; - continue; - } elseif (!empty($documents)) { - // Run documents - foreach ($documents as $doc) { - $errors = $this->ruleManager->actionDocument($doc->id, 'rerun'); - // Check errors - if (!empty($errors)) { - $this->message .= 'Document '.$doc->id.' in error (rule '.$ruleId.') : '.$errors[0].'. '; - } - } + if (!empty($documents[0]->id)) { + $documentIds[]['id'] = $documents[0]->id; } } - // Since the actionDocument takes a string and not an array of ids, we recompose the ids into a string separated by commas - if ($usesDocumentIds === 1) { - $stringOfDocumentIds = implode(',', $arrayOfDocumentIds); - $errors = $this->ruleManager->actionDocument($stringOfDocumentIds, 'rerun'); + // Filter documents + if (!empty($documentIds)) { + $responseFilter = $this->ruleManager->filterDocuments($documentIds); + } + + // Check predecessor + $documentIds = $this->refreshDocumentList($documentIds, $responseFilter); + if (!empty($documentIds)) { + $responseCheckPredecessor = $this->ruleManager->checkPredecessorDocuments($documentIds); + } + + // Check parent + $documentIds = $this->refreshDocumentList($documentIds, $responseCheckPredecessor); + if (!empty($documentIds)) { + $responseCheckParent = $this->ruleManager->checkParentDocuments($documentIds); + } + + // Transform document + $documentIds = $this->refreshDocumentList($documentIds, $responseCheckParent); + if (!empty($documentIds)) { + $responseTransform = $this->ruleManager->transformDocuments($documentIds); + } + + // Get history + $documentIds = $this->refreshDocumentList($documentIds, $responseTransform); + if (!empty($documentIds)) { + $responseGetTargetData = $this->ruleManager->getTargetDataDocuments($documentIds); + } + + // Send document + $documentIds = $this->refreshDocumentList($documentIds, $responseGetTargetData); + if (!empty($documentIds)) { + $responseSend = $this->ruleManager->sendDocuments($documentIds); } } catch (Exception $e) { $this->message .= 'Error : '.$e->getMessage(); $this->logger->error('Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); - return false; } - return true; } + // Remove the document from the list if the retrun value is not true + public function refreshDocumentList($documentIds, $response) { + $documentListRefresh = []; + if (!empty($response)) { + foreach($response as $docId => $return) { + if ($return) { + $documentListRefresh[]['id'] = $docId; + } + } + } + return $documentListRefresh; + } + // Remove all data flagged deleted in the database public function pruneDatabase(): void { diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 230d7b05f..41ad2f2f9 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -931,14 +931,7 @@ public function actionDocument($id_document, $event, $param1 = null) { switch ($event) { case 'rerun': - // We use the method massIdRerun if there is a , in the id string, which implies that it is actually several ids - // Otherwise we use the regular rerun - if ((strpos($id_document, ',') !== false)) { - return $this->massIdRerun($id_document); - } else { - return $this->rerun($id_document); - } - break; + return $this->rerun($id_document); case 'cancel': return $this->cancel($id_document); case 'remove': @@ -1387,162 +1380,6 @@ protected function rerun($id_document): array return $msg_error; } - // Function to rerun several documents by their ids. The process of creating the document (status: New) is separated from the other statuses (Filter, Predecessor...etc) - // if we have 5 documents, we will have 5 statuses New then the rest of the operations will happen after that. - protected function massIdRerun(string $documentIds) - { - $session = new Session(); - $msg_error = []; - $msg_success = []; - $msg_info = []; - // Récupération du statut du document - $param['id_doc_myddleware'] = $documentIds; - $param['jobId'] = $this->jobId; - $param['api'] = $this->api; - // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $status = $this->documentManager->getStatus(); - // Si la règle n'est pas chargée alors on l'initialise. - if (empty($this->ruleId)) { - $this->ruleId = $this->documentManager->getRuleId(); - $this->setRule($this->ruleId); - $this->setRuleRelationships(); - $this->setRuleParam(); - $this->setRuleField(); - } - - // Manually setting the status to New - // In this method, the statuses are set mannually because the method for getting statuses doesn't understand the string of several ids. - $status = "New"; - - $response[$documentIds] = false; - - $arrayIdDocument = []; - - //we separate the string into an array in order to be able to match the associative array id => structure - $arrayDocIdOriginal = explode(",", $documentIds); - foreach ($arrayDocIdOriginal as $document) { - $arrayIdDocument[] = ['id' => $document]; - } - // On lance des méthodes différentes en fonction du statut en cours du document et en fonction de la réussite ou non de la fonction précédente - if (in_array($status, ['New', 'Filter_KO'])) { - $response = $this->filterDocuments($arrayIdDocument); - if (true === $this->verifyMultiIdResponse($response)) { - $msg_success[] = 'Transfer id '.$documentIds.' : Status change => Filter_OK'; - // Update status if an action has been executed - $status = 'Filter_OK'; - } elseif (-1 == $response[$documentIds]) { - $msg_info[] = 'Transfer id '.$documentIds.' : Status change => Filter'; - } else { - // Update status if an action has been executed - $status = 'Filter_KO'; - $msg_error[] = 'Transfer id '.$documentIds.' : Error, status transfer => Filter_KO'; - } - } - if (in_array($status, ['Filter_OK', 'Predecessor_KO'])) { - $response = $this->checkPredecessorDocuments($arrayIdDocument); - if (true === $this->verifyMultiIdResponse($response)) { - // Update status if an action has been executed - $status = 'Predecessor_OK'; - $msg_success[] = 'Transfer id '.$documentIds.' : Status change => Predecessor_OK'; - } else { - $msg_error[] = 'Transfer id '.$documentIds.' : Error, status transfer => Predecessor_KO'; - // Update status if an action has been executed - $status = 'Predecessor_KO'; - } - } - if (in_array($status, ['Predecessor_OK', 'Relate_KO'])) { - $response = $this->checkParentDocuments($arrayIdDocument); - if (true === $this->verifyMultiIdResponse($response)) { - // Update status if an action has been executed - $status = 'Relate_OK'; - $msg_success[] = 'Transfer id '.$documentIds.' : Status change => Relate_OK'; - } else { - $msg_error[] = 'Transfer id '.$documentIds.' : Error, status transfer => Relate_KO'; - // Update status if an action has been executed - $status = 'Relate_KO'; - } - } - if (in_array($status, ['Relate_OK', 'Error_transformed'])) { - $response = $this->transformDocuments($arrayIdDocument); - if (true === $this->verifyMultiIdResponse($response)) { - // Update status if an action has been executed - $status = 'Transformed'; - $msg_success[] = 'Transfer id '.$documentIds.' : Status change : Transformed'; - } else { - $msg_error[] = 'Transfer id '.$documentIds.' : Error, status transfer : Error_transformed'; - // Update status if an action has been executed - $status = 'Error_transformed'; - } - } - if (in_array($status, ['Transformed', 'Error_checking', 'Not_found'])) { - $response = $this->getTargetDataDocuments($arrayIdDocument); - if (true === $this->verifyMultiIdResponse($response)) { - if ('S' == $this->rule['mode']) { - $msg_success[] = 'Transfer id '.$documentIds.' : Status change : '.$response['doc_status']; - } else { - $msg_success[] = 'Transfer id '.$documentIds.' : Status change : '.$response['doc_status']; - } - } else { - $msg_error[] = 'Transfer id '.$documentIds.' : Error, status transfer : '.$response['doc_status']; - } - // Update status if an action has been executed - $status = $this->documentManager->getStatus(); - } - // Si la règle est en mode recherche alors on n'envoie pas de données - // Si on a un statut compatible ou si le doc vient de passer dans l'étape précédente et qu'il n'est pas no_send alors on envoie les données - if ( - 'S' != $this->rule['mode'] - && ( - in_array($status, ['Ready_to_send', 'Error_sending']) - || ( - true === $response[$documentIds] - && !empty($response['doc_status']) - && in_array($response['doc_status'], ['Ready_to_send', 'Error_sending']) - ) - ) - ) { - $response = $this->massSendTarget('', $documentIds); - if ( - !empty($response[$documentIds]['id']) - && empty($response[$documentIds]['error']) - && empty($response['error']) // Error can be on the document or can be a general error too - ) { - $msg_success[] = 'Transfer id '.$documentIds.' : Status change : Send'; - } else { - $msg_error[] = 'Transfer id '.$documentIds.' : Error, status transfer : Error_sending. '.(!empty($response['error']) ? $response['error'] : $response[$documentIds]['error']); - } - } - // If the job is manual, we display error in the UI - if ($this->manual) { - if (!empty($msg_error)) { - $session->set('error', $msg_error); - } - if (!empty($msg_success)) { - $session->set('success', $msg_success); - } - if (!empty($msg_info)) { - $session->set('info', $msg_info); - } - } - - return $msg_error; - } - - // Function to verify the state of the response. It will be valid if at least one document valid. - // That way if all of the documents fail we can abort the process, but if some succeed, then the failed ones will be logged. - public function verifyMultiIdResponse(array $response) - { - $atLeastOneDocumentValid = false; - - foreach ($response as $documentState) { - if ($documentState === true || $documentState === 'Ready_to_send') { - $atLeastOneDocumentValid = true; - break; - } - } - return $atLeastOneDocumentValid; - } protected function clearSendData($sendData) { From 00ed4d633b68cfce94d030f1d8c6f5831072a070 Mon Sep 17 00:00:00 2001 From: myddleware Date: Tue, 13 Feb 2024 14:11:31 +0100 Subject: [PATCH 10/88] SugarCRM : date modified was empty if the last record read was a deletion Signed-off-by: myddleware --- src/Solutions/sugarcrm.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Solutions/sugarcrm.php b/src/Solutions/sugarcrm.php index 12721f23a..eb43a2ba0 100644 --- a/src/Solutions/sugarcrm.php +++ b/src/Solutions/sugarcrm.php @@ -326,6 +326,7 @@ public function readRelationship($param, $rel) { public function read($param) { $result = []; + $maxDateModified = ''; $rel = $this->isManyToManyRel($param['module']); if ($rel !== false) { return $this->readRelationship($param, $rel); @@ -394,6 +395,10 @@ public function read($param) // At least one non deleted record read $onlyDeletion = false; } + // Keep the date modified max (date modified is empty for deletion record) + if (!empty($record->date_modified)) { + $maxDateModified = $record->date_modified; + } } // Error if only deletion records read if ($onlyDeletion) { @@ -430,7 +435,7 @@ public function read($param) } // No date modified returned if record deleted, we set a default date (the last reference date read) if (!empty($result[$record->id]['myddleware_deletion'])) { - $result[$record->id]['date_modified'] = end($records)->date_modified; + $result[$record->id]['date_modified'] = $maxDateModified; } } } From a1c8922dbca3c396b95f5fdcd97484f00f4cc12b Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 14 Feb 2024 13:05:54 +0100 Subject: [PATCH 11/88] SuiteCRM : remove favorite parameter Signed-off-by: myddleware --- src/Solutions/suitecrm.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Solutions/suitecrm.php b/src/Solutions/suitecrm.php index 20c981882..495932c16 100644 --- a/src/Solutions/suitecrm.php +++ b/src/Solutions/suitecrm.php @@ -444,7 +444,6 @@ public function read($param) 'link_name_to_fields_array' => $link_name_to_fields_array, 'max_results' => $this->limitCall, 'deleted' => $deleted, - 'Favorites' => '', ]; $get_entry_list_result = $this->call('get_entry_list', $get_entry_list_parameters); // Construction des données de sortie From 7b5c58eb0e14fc0b4cf1a2c5d59bd7df9868e86f Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 16 Feb 2024 19:16:43 +0100 Subject: [PATCH 12/88] Add the possibility to adapt the list of bidirectional rules Signed-off-by: myddleware --- src/Controller/DefaultController.php | 2 +- src/Manager/RuleManager.php | 7 +++++-- src/Solutions/solution.php | 8 ++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Controller/DefaultController.php b/src/Controller/DefaultController.php index 6a2f9aab1..c5261966f 100644 --- a/src/Controller/DefaultController.php +++ b/src/Controller/DefaultController.php @@ -1917,7 +1917,7 @@ public function ruleStepThree(Request $request) $bidirectional_params['module']['source'] = $module['source']; $bidirectional_params['module']['cible'] = $module['cible']; - $bidirectional = RuleManager::getBidirectionalRules($this->connection, $bidirectional_params); + $bidirectional = RuleManager::getBidirectionalRules($this->connection, $bidirectional_params, $solution_source, $solution_cible); if ($bidirectional) { $rule_params = array_merge($rule_params, $bidirectional); } diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 41ad2f2f9..506c85823 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -1049,9 +1049,13 @@ protected function getRuleDocuments($ruleId, $sourceId = true, $targetId = false // Permet de récupérer les règles potentiellement biderectionnelle. // Cette fonction renvoie les règles qui utilisent les même connecteurs et modules que la règle en cours mais en sens inverse (source et target inversées) // On est sur une méthode statique c'est pour cela que l'on récupère la connexion e paramètre et non dans les attributs de la règle - public static function getBidirectionalRules($connection, $params): ?array + public static function getBidirectionalRules($connection, $params, $solutionSource, $solutionTarget): ?array { try { + // Call solutions in cas target ans source modules hasn't the same name (ex Moodle manual_enrol_users and get_enrolments_by_date) + $params = $solutionSource->beforeGetBidirectionalRules($params, 'source'); + $params = $solutionTarget->beforeGetBidirectionalRules($params, 'target'); + // Récupération des règles opposées à la règle en cours de création $queryBidirectionalRules = 'SELECT id, @@ -1094,7 +1098,6 @@ public static function getBidirectionalRules($connection, $params): ?array } catch (\Exception $e) { return null; } - return null; } diff --git a/src/Solutions/solution.php b/src/Solutions/solution.php index 4cbdbd165..120ab4bd9 100644 --- a/src/Solutions/solution.php +++ b/src/Solutions/solution.php @@ -975,6 +975,14 @@ protected function checkDataBeforeDelete($param, $data) return $data; } + /** + * @throws \Exception + */ + public function beforeGetBidirectionalRules($param, $type) + { + return $param; + } + /** * @throws \Doctrine\DBAL\Exception * @throws \Exception From 6192588b7ce41bc2c9352fa1f3be463300128032 Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 22 Feb 2024 18:43:29 +0100 Subject: [PATCH 13/88] Fix : manage error for history call Signed-off-by: myddleware --- src/Solutions/solution.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Solutions/solution.php b/src/Solutions/solution.php index 120ab4bd9..e065906eb 100644 --- a/src/Solutions/solution.php +++ b/src/Solutions/solution.php @@ -296,6 +296,13 @@ public function readData($param) // Read data $readResult = $this->read($param); + // In case of error in an history call, we return directly the error + if ( + $param['call_type'] == 'history' + AND !empty($readResult['error']) + ) { + return $readResult; + } // Save the new rule params into attribut dataSource if (!empty($readResult['ruleParams'])) { From 8b8c42b9d7cb05405551f9c22b5193b53ed0cabd Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 22 Feb 2024 18:49:33 +0100 Subject: [PATCH 14/88] Moodle : add function local_myddleware_search_enrolment Signed-off-by: myddleware --- src/Solutions/moodle.php | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Solutions/moodle.php b/src/Solutions/moodle.php index 87de84960..ada6b87bd 100644 --- a/src/Solutions/moodle.php +++ b/src/Solutions/moodle.php @@ -46,8 +46,9 @@ class moodlecore extends solution protected array $FieldsDuplicate = [ 'users' => ['email', 'username'], 'courses' => ['shortname', 'idnumber'], + 'manual_enrol_users' => ['userid', 'courseid'], + 'manual_unenrol_users' => ['userid', 'courseid'], ]; - protected array $createOnlyFields = [ 'courses' => ['lang'], ]; @@ -604,6 +605,12 @@ protected function formatResponse($method, $response, $param) // Get the function name protected function getFunctionName($param): string { + if ( + $param['call_type'] == 'history' + AND in_array($param['module'], array('manual_enrol_users', 'manual_unenrol_users')) + ) { + return 'local_myddleware_search_enrolment'; + } // In case of duplicate search (search with a criteria) if ( !empty($param['query']) @@ -615,7 +622,7 @@ protected function getFunctionName($param): string } elseif ('courses' == $param['module']) { return 'core_course_get_courses_by_field'; } - // In case of read by date or search a specific record with an id for specific modules user or course + // In case of read by date or search a specific record with an id for specific modules user or course } else { if ('users' == $param['module']) { return 'local_myddleware_get_users_by_date'; @@ -638,9 +645,23 @@ protected function getFunctionName($param): string */ protected function setParameters($param): array { + // Get the function name $functionName = $this->getFunctionName($param); - $parameters['time_modified'] = $this->dateTimeFromMyddleware($param['date_ref']); + // Specific parameters for function local_myddleware_search_enrolment + if ($functionName == 'local_myddleware_search_enrolment') { + if ( + empty($param['query']['userid']) + OR empty($param['query']['courseid']) + ) { + throw new \Exception('CourseId and UserId are both requiered to check if an enrolment exists. One of them is empty here. '); + } + $parameters['userid'] = $param['query']['userid']; + $parameters['courseid'] = $param['query']['courseid']; + return $parameters; + } + // If standard function called to search by criteria + $parameters['time_modified'] = $this->dateTimeFromMyddleware($param['date_ref']); if (in_array($functionName, ['core_user_get_users', 'core_course_get_courses_by_field'])) { if (!empty($param['query'])) { foreach ($param['query'] as $key => $value) { @@ -654,7 +675,8 @@ protected function setParameters($param): array } else { throw new \Exception('Filter criteria empty. Not allowed to run function '.$functionName.' without filter criteria.'); } - } elseif (!empty($param['query']['id'])) { + } + elseif (!empty($param['query']['id'])) { $parameters['id'] = $param['query']['id']; } From ec8735cee38e7317e809243b5ac2a3750d2f7273 Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 22 Feb 2024 18:51:03 +0100 Subject: [PATCH 15/88] Moodle : add info in error message Signed-off-by: myddleware --- src/Solutions/moodle.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Solutions/moodle.php b/src/Solutions/moodle.php index ada6b87bd..a42877aa5 100644 --- a/src/Solutions/moodle.php +++ b/src/Solutions/moodle.php @@ -213,7 +213,7 @@ public function read($param): array $xml = $this->formatResponse('read', $response, $param); if (!empty($xml->ERRORCODE)) { - throw new \Exception("Error $xml->ERRORCODE : $xml->MESSAGE"); + throw new \Exception("Error code $xml->ERRORCODE : $xml->MESSAGE. ".(!empty($xml->DEBUGINFO) ? "Info : $xml->DEBUGINFO" : "")); } // Transform the data to Myddleware format if (!empty($xml->MULTIPLE->SINGLE)) { From 620909ac6127c38af7246ce98bbc0c870a64d256 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 6 Mar 2024 23:09:22 +0100 Subject: [PATCH 16/88] Fix Api bugs on deletion record function Signed-off-by: myddleware --- src/Controller/ApiController.php | 89 ++++++++++++++++++++------------ src/Manager/DocumentManager.php | 1 + src/Manager/JobManager.php | 2 +- 3 files changed, 59 insertions(+), 33 deletions(-) diff --git a/src/Controller/ApiController.php b/src/Controller/ApiController.php index 280835fff..1c49cfcb4 100644 --- a/src/Controller/ApiController.php +++ b/src/Controller/ApiController.php @@ -3,7 +3,10 @@ namespace App\Controller; use App\Manager\JobManager; +use App\Manager\FormulaManager; use App\Manager\RuleManager; +use App\Manager\DocumentManager; +use App\Manager\SolutionManager; use App\Repository\DocumentRepository; use App\Repository\JobRepository; use App\Repository\RuleRepository; @@ -32,6 +35,9 @@ class ApiController extends AbstractController private KernelInterface $kernel; private LoggerInterface $logger; private JobManager $jobManager; + private SolutionManager $solutionManager; + private DocumentManager $documentManager; + private FormulaManager $formulaManager; private ParameterBagInterface $parameterBag; private EntityManagerInterface $entityManager; @@ -43,12 +49,18 @@ public function __construct( JobRepository $jobRepository, DocumentRepository $documentRepository, ParameterBagInterface $parameterBag, - EntityManagerInterface $entityManager + EntityManagerInterface $entityManager, + FormulaManager $formulaManager, + DocumentManager $documentManager, + SolutionManager $solutionManager ) { $this->ruleRepository = $ruleRepository; $this->jobRepository = $jobRepository; $this->documentRepository = $documentRepository; $this->jobManager = $jobManager; + $this->solutionManager = $solutionManager; + $this->formulaManager = $formulaManager; + $this->documentManager = $documentManager; $this->logger = $logger; $this->kernel = $kernel; $this->env = $kernel->getEnvironment(); @@ -103,7 +115,7 @@ public function synchroAction(Request $request): JsonResponse $return['jobId'] = substr($content, 2, 23); $job = $this->jobRepository->find($return['jobId']); // Get the job statistics - $this->jobManager->setId($job->getId()); + $this->jobManager->setId($this->jobManager->getId()); $jobData = $this->jobManager->getLogData(); if (!empty($jobData['jobError'])) { throw new Exception('Failed to get the job statistics. '.$jobData['jobError']); @@ -174,7 +186,7 @@ public function readRecordAction(Request $request): JsonResponse // Get the job statistics $job = $this->jobRepository->find($return['jobId']); - $this->jobManager->setId($job->getId()); + $this->jobManager->setId($this->jobManager->getId()); $jobData = $this->jobManager->getLogData(); if (!empty($jobData['jobError'])) { throw new Exception('Failed to get the job statistics. '.$jobData['jobError']); @@ -193,15 +205,19 @@ public function readRecordAction(Request $request): JsonResponse */ public function deleteRecordAction(Request $request): JsonResponse { - try { - $connection = $this->container->get('database_connection'); - $connection->beginTransaction(); // -- BEGIN TRANSACTION +// $this->logger->error('test SF01 '); + try { + $connection = $this->entityManager->getConnection(); + // $connection = $this->container->get('database_connection'); + // $connection->beginTransaction(); // -- BEGIN TRANSACTION $return = []; $return['error'] = ''; // Get input data - $data = $request->request->all(); + $data = json_decode($request->getContent(), true); +// $return['error'] = 'test SF01 '.print_r($data,true); +// return $this->json($return); // Check parameter if (empty($data['rule'])) { @@ -210,6 +226,9 @@ public function deleteRecordAction(Request $request): JsonResponse if (empty($data['recordId'])) { throw new Exception('recordId is missing. recordId is the id of the record you want to delete. '); } +// $return['error'] = 'test SF02'; +// $return['error'] = 'test SF02 '.print_r($data,true); +// return $this->json($return); // Set the document values foreach ($data as $key => $value) { @@ -227,35 +246,43 @@ public function deleteRecordAction(Request $request): JsonResponse } } $docParam['values']['myddleware_deletion'] = true; // Force deleted record type + + // $this->logger->error('test'); +// $return['error'] = 'test SF03 '.print_r($docParam,true); +// return $this->json($return); // Create job instance - $job = $this->container->get('myddleware_job.job'); - $job->setApi(1); - $job->initJob('Delete record '.$data['recordId'].' in rule '.$data['rule']); + $this->jobManager->setApi(1); + $this->jobManager->initJob('Delete record '.$data['recordId'].' in rule '.$data['rule']); +// $return['error'] = 'test SF04 '.print_r($docParam,true); +// return $this->json($return); // Instantiate the rule $ruleParam['ruleId'] = $data['rule']; - $ruleParam['jobId'] = $job->id; + $ruleParam['jobId'] = $this->jobManager->getId(); $ruleParam['api'] = 1; + + $rule = new RuleManager( - $this->container->get('logger'), + $this->logger, $connection, $this->entityManager, $this->parameterBag, - // $ruleParam, $this->formulaManager, $this->solutionManager, $this->documentManager ); - +// $return['error'] = 'test SF05 '.print_r($ruleParam,true); +// return $this->json($return); + $rule->setRule($data['rule']); $document = $rule->generateDocuments($data['recordId'], false, $docParam); // Stop the process if error during the data transfer creation as we won't be able to manage it in Myddleware if (!empty($document->error)) { throw new Exception('Error during data transfer creation (rule '.$data['rule'].') : '.$document->error.'. '); } - $connection->commit(); // -- COMMIT TRANSACTION + // $connection->commit(); // -- COMMIT TRANSACTION } catch (Exception $e) { - $connection->rollBack(); // -- ROLLBACK TRANSACTION + // $connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error($e->getMessage()); $return['error'] .= $e->getMessage(); // Stop the process if document hasn't been created @@ -278,22 +305,22 @@ public function deleteRecordAction(Request $request): JsonResponse // Close job if it has been created try { - $connection->beginTransaction(); // -- BEGIN TRANSACTION - if (true === $job->createdJob) { - $job->closeJob(); + // $connection->beginTransaction(); // -- BEGIN TRANSACTION + if (true === $this->jobManager->createdJob) { + $this->jobManager->closeJob(); } // Get the job statistics even if the job has failed - if (!empty($job->id)) { - $return['jobId'] = $job->id; - $jobData = $job->getLogData(); + if (!empty($this->jobManager->id)) { + $return['jobId'] = $this->jobManager->id; + $jobData = $this->jobManager->getLogData(); if (!empty($jobData['jobError'])) { $return['error'] .= $jobData['jobError']; } $return['jobData'] = $jobData; } - $connection->commit(); // -- COMMIT TRANSACTION + // $connection->commit(); // -- COMMIT TRANSACTION } catch (Exception $e) { - $connection->rollBack(); // -- ROLLBACK TRANSACTION + // $connection->rollBack(); // -- ROLLBACK TRANSACTION $this->logger->error('Failed to get the job statistics. '.$e->getMessage()); $return['error'] .= 'Failed to get the job statistics. '.$e->getMessage(); } @@ -311,7 +338,7 @@ public function massActionAction(Request $request): JsonResponse $return['error'] = ''; // Get input data - $data = $request->request->all(); + $data = json_decode($request->getContent(), true); // Check parameter if (empty($data['action'])) { @@ -358,9 +385,8 @@ public function massActionAction(Request $request): JsonResponse $return['jobId'] = substr($content, 0, 23); // Get the job statistics - $job = $this->container->get('myddleware_job.job'); - $job->id = $return['jobId']; - $jobData = $job->getLogData(); + $this->jobManager->id = $return['jobId']; + $jobData = $this->jobManager->getLogData(); if (!empty($jobData['jobError'])) { throw new Exception('Failed to get the job statistics. '.$jobData['jobError']); } @@ -383,7 +409,7 @@ public function rerunErrorAction(Request $request): JsonResponse $return['error'] = ''; // Get input data - $data = $request->request->all(); + $data = json_decode($request->getContent(), true); // Check parameter if (empty($data['limit'])) { @@ -423,9 +449,8 @@ public function rerunErrorAction(Request $request): JsonResponse $return['jobId'] = substr($content, 0, 23); // Get the job statistics - $job = $this->container->get('myddleware_job.job'); - $job->id = $return['jobId']; - $jobData = $job->getLogData(); + $this->jobManager->id = $return['jobId']; + $jobData = $this->jobManager->getLogData(); $return['jobData'] = $jobData; } catch (Exception $e) { $this->logger->error($e->getMessage()); diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 520c0568c..d2ec8d109 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1470,6 +1470,7 @@ protected function updateTargetTable() throw new \Exception('Failed to transform the field '.$ruleField['target_field_name'].'.'); } $targetField[$ruleField['target_field_name']] = $value; + // If the target value equals mdw_no_send_field, the field isn't sent to the target if ($value === "mdw_no_send_field") { unset($targetField[$ruleField['target_field_name']]); $this->notSentFields[] = $ruleField['target_field_name']; diff --git a/src/Manager/JobManager.php b/src/Manager/JobManager.php index 44271eb3c..ab267d30c 100644 --- a/src/Manager/JobManager.php +++ b/src/Manager/JobManager.php @@ -622,7 +622,7 @@ public function refreshDocumentList($documentIds, $response) { $documentListRefresh = []; if (!empty($response)) { foreach($response as $docId => $return) { - if ($return) { + if ($return == true) { $documentListRefresh[]['id'] = $docId; } } From 3f2b8cc0490a77c5346f91f38e8146a332966333 Mon Sep 17 00:00:00 2001 From: myddleware Date: Tue, 12 Mar 2024 11:25:18 +0100 Subject: [PATCH 17/88] Add massaction action : unlock Signed-off-by: myddleware --- src/Command/MassActionCommand.php | 20 +++----------------- src/Manager/DocumentManager.php | 8 +++++--- src/Manager/JobManager.php | 4 ++++ src/Manager/RuleManager.php | 27 +++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/Command/MassActionCommand.php b/src/Command/MassActionCommand.php index d22a6f8ab..939b1c532 100644 --- a/src/Command/MassActionCommand.php +++ b/src/Command/MassActionCommand.php @@ -27,39 +27,25 @@ // Documentation : Overview - The myddleware:massaction command allows you to do various actions on one or multiple documents in Myddleware. This is especially useful when you need to batch process document cancellations or reload without manually intervening for each. For windows and laragon, use php bin/console. - php bin/console myddleware:massaction cancel document [document_ids] [optional_arguments] - Parameters - document_ids: A comma-separated list of the document IDs that you want to cancel. For example: 64f8847a69b2a2.66477108,64f8841bbd8cd9.89905176. - Optional Arguments - forceAll: Usage: forceAll [Y/N] This argument is used when you want to cancel documents regardless of their current status. By default, only documents with the status "open" or "error" can be canceled. If you want to cancel documents with other statuses, set this argument to Y. Example: forceAll Y - Examples - Basic Usage: Cancelling specific documents based on their IDs: - php bin/console myddleware:massaction cancel document 64f8847a69b2a2.66477108,64f8841bbd8cd9.89905176 - + php bin/console myddleware:massaction cancel document 64f8847a69b2a2.66477108,64f8841bbd8cd9.89905176 this will cancel theses documents only if they are in open or error status. - Using forceAll: Cancelling documents regardless of their status: - php bin/console myddleware:massaction cancel document 64f8847a69b2a2.66477108,64f8841bbd8cd9.89905176 forceAll Y - - - *********************************************************************************/ namespace App\Command; @@ -96,7 +82,7 @@ protected function configure() $this ->setName('myddleware:massaction') ->setDescription('Action massive sur les flux') - ->addArgument('action', InputArgument::REQUIRED, 'Action (rerun, cancel, remove, restore or changeStatus)') + ->addArgument('action', InputArgument::REQUIRED, 'Action (rerun, cancel, remove, restore, changeStatus, unlock)') ->addArgument('dataType', InputArgument::REQUIRED, 'Data type (rule or document)') ->addArgument('ids', InputArgument::REQUIRED, 'Rule or document ids') // id séparés par des "," ->addArgument('forceAll', InputArgument::OPTIONAL, 'Set Y to process action on all documents (not only open and error ones)') @@ -144,7 +130,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $output->writeln('1;'.$this->jobManager->getId()); // Do not remove, used for manual job and webservices (display logs) // Récupération des paramètres - if (!in_array($action, ['rerun', 'cancel', 'remove', 'restore', 'changeStatus'])) { + if (!in_array($action, ['rerun', 'cancel', 'remove', 'restore', 'changeStatus', 'unlock'])) { throw new Exception('Action '.$action.' unknown. Please use action rerun, cancel or remove.'); } if (!in_array($dataType, ['document', 'rule'])) { diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index d2ec8d109..1a9fd7783 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -509,7 +509,7 @@ protected function setLock() { } // Set the document lock - public function unsetLock() { + public function unsetLock($force = false) { try { // Get the job lock on the document $documentQuery = 'SELECT * FROM document WHERE id = :doc_id'; @@ -517,9 +517,11 @@ public function unsetLock() { $stmt->bindValue(':doc_id', $this->id); $documentResult = $stmt->executeQuery(); $documentData = $documentResult->fetchAssociative(); // 1 row - // If document already lock by the current job, we return true; - if ($documentData['job_lock'] == $this->jobId) { + if ( + $documentData['job_lock'] == $this->jobId + OR $force === true + ) { $now = gmdate('Y-m-d H:i:s'); $query = " UPDATE document SET diff --git a/src/Manager/JobManager.php b/src/Manager/JobManager.php index ab267d30c..0598812ab 100644 --- a/src/Manager/JobManager.php +++ b/src/Manager/JobManager.php @@ -481,6 +481,10 @@ public function massAction($action, $dataType, $ids, $forceAll, $fromStatus, $to if ('changeStatus' == $action) { $where .= " AND document.status = '$fromStatus' "; } + // Filter on document locked + if ('unlock' == $action) { + $where .= " AND document.job_lock != '' AND document.job_lock IS NOT NULL "; + } // Build the query $sqlParams = ' SELECT diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 506c85823..55d4721d2 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -940,6 +940,8 @@ public function actionDocument($id_document, $event, $param1 = null) return $this->changeDeleteFlag($id_document, false); case 'changeStatus': return $this->changeStatus($id_document, $param1); + case 'unlock': + return $this->unlockDocument($id_document); default: return 'Action '.$event.' unknown. Failed to run this action. '; } @@ -1125,6 +1127,31 @@ protected function cancel($id_document) } } } + + /** + * @throws \Doctrine\DBAL\Exception + */ + protected function unlockDocument($id_document) + { + $param['id_doc_myddleware'] = $id_document; + $param['jobId'] = $this->jobId; + $param['api'] = $this->api; + // Set the param values and clear all document attributes + $this->documentManager->setParam($param, true); + $this->documentManager->unsetLock(true); + $session = new Session(); + $message = $this->documentManager->getMessage(); + + // Si on a pas de jobId cela signifie que l'opération n'est pas massive mais sur un seul document + // On affiche alors le message directement dans Myddleware + if (empty($this->jobId)) { + if (empty($message)) { + $session->set('success', ['Data transfer has been successfully unlocked.']); + } else { + $session->set('error', [$this->documentManager->getMessage()]); + } + } + } /** * @throws \Doctrine\DBAL\Exception From 8a0ce1466056b20b25ad3b05a76cb1feeee39acd Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 15 Mar 2024 10:19:34 +0100 Subject: [PATCH 18/88] Salesforce : manageg relate Salesforce's fields Signed-off-by: myddleware --- src/Solutions/salesforce.php | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index 08ff8fb1d..1f77f4e8a 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -400,19 +400,40 @@ public function readData($param): array // Traitement des informations reçues // foreach($query_request_data['records'] as $record){ for ($i = 0; $i < $currentCount; $i++) { - $record = $query_request_data['records'][$i]; - foreach (array_keys($record) as $key) { + $record = $query_request_data['records'][$i]; + foreach (array_keys($record) as $key) { if($key == $DateRefField){ $record[$key] = $this->dateTimeToMyddleware($record[$key]); $row['date_modified'] = $record[$key]; } + // Manage relationship fields stored in a sub array + elseif(substr($key,-3) == '__r') { + foreach($record[$key] as $fieldKey => $fieldValue) { + // Don't save attributes + if($fieldKey != 'attributes'){ + // In case there are 2 levels of relationship (think about a recursive function here) + if(substr($fieldKey,-3) == '__r') { + foreach($fieldValue as $fieldKeyLevel2 => $fieldValueLevel2) { + if($fieldKeyLevel2 != 'attributes'){ + $row[mb_strtolower($fieldKeyLevel2)] = $fieldValueLevel2; + $row[$param['module'].'.'.$key.'.'.$fieldKey.'.'.$fieldKeyLevel2] = $fieldValueLevel2; + } + } + } + else { + $row[$param['module'].'.'.$key.'.'.$fieldKey] = $fieldValue; + } + } + } + } // On enlève le tableau "attributes" ajouté par Salesforce afin d'extraire les éléments souhaités - else if(!($key == 'attributes')){ + elseif(!($key == 'attributes')){ if($key == 'Id') $row[mb_strtolower($key)] = $record[$key]; else { - if($key == 'CreatedDate') + if($key == 'CreatedDate') { $record[$key] = $this->dateTimeToMyddleware($record[$key]); + } $row[$key] = $record[$key]; } } @@ -427,6 +448,7 @@ public function readData($param): array $row[$key] = $MailinAddress; } } + } $result['date_ref'] = $record[$DateRefField]; $result['values'][$record['Id']] = $row; @@ -453,14 +475,13 @@ public function readData($param): array && !empty($previousRefDate) && $previousRefDate == $result['date_ref'] ); - return $result; } catch (\Exception $e) { $error = $e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->logger->error($error); $result['error'] = $error; - return $result; } + return $result; } /** From 13674628a6cf445c987db7f715e4cb3033d2b16f Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 22 Mar 2024 16:44:44 +0100 Subject: [PATCH 19/88] Quick fix on Moodle and SugarCRM Signed-off-by: myddleware --- src/Solutions/moodle.php | 17 ++++++++++------- src/Solutions/salesforce.php | 9 ++++----- src/Solutions/sugarcrm.php | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Solutions/moodle.php b/src/Solutions/moodle.php index a42877aa5..dc69394d9 100644 --- a/src/Solutions/moodle.php +++ b/src/Solutions/moodle.php @@ -189,10 +189,14 @@ public function get_module_fields($module, $type = 'source', $param = null): arr public function read($param): array { try { - // No read action in case of history on enrolment module + // No read action in case of history on enrolment module (except if user_id and course_id are duplicate search parameters) if ( in_array($param['module'], array('manual_enrol_users', 'manual_unenrol_users')) AND $param['call_type'] == 'history' + AND ( + empty($param['query']['userid']) + OR empty($param['query']['courseid']) + ) ) { return array(); } @@ -645,18 +649,17 @@ protected function getFunctionName($param): string */ protected function setParameters($param): array { - // Get the function name - $functionName = $this->getFunctionName($param); + $functionName = $this->getFunctionName($param); // Specific parameters for function local_myddleware_search_enrolment if ($functionName == 'local_myddleware_search_enrolment') { if ( empty($param['query']['userid']) - OR empty($param['query']['courseid']) + OR empty($param['query']['courseid']) ) { - throw new \Exception('CourseId and UserId are both requiered to check if an enrolment exists. One of them is empty here. '); + throw new \Exception('CourseId and UserId are both requiered to check if an enrolment exists. One of them is empty here or is not added as duplicate serach parameter in the rule. '); } - $parameters['userid'] = $param['query']['userid']; - $parameters['courseid'] = $param['query']['courseid']; + $parameters['userid'] = $param['query']['userid']; + $parameters['courseid'] = $param['query']['courseid']; return $parameters; } diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index 1f77f4e8a..7548138a5 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -380,7 +380,7 @@ public function readData($param): array $queryWhere = $this->getWhere($param); // Gestion du ORDER $queryOrder = $this->getOrder($param); - + // Gstion du LIMIT $queryLimit .= "+LIMIT+" . $param['limit']; // Ajout de la limite souhaitée // On lit les données dans Salesforce @@ -392,7 +392,7 @@ public function readData($param): array $query = $baseQuery.$querySelect.$queryFrom.$queryWhere.$queryOrder.$queryLimit.$queryOffset; $query_request_data = $this->call($query, false); $query_request_data = $this->formatResponse($param,$query_request_data); - + // Affectation du nombre de résultat à $result['count'] if (isset($query_request_data['totalSize'])){ $currentCount = $query_request_data['totalSize']; @@ -447,8 +447,7 @@ public function readData($param): array $MailinAddress = rtrim($MailinAddress,' '); $row[$key] = $MailinAddress; } - } - + } } $result['date_ref'] = $record[$DateRefField]; $result['values'][$record['Id']] = $row; @@ -465,7 +464,7 @@ public function readData($param): array } else { $result['error'] = $query_request_data; - } + } } // On continue si : // 1. Le nombre de résultat du dernier appel est égal à la limite diff --git a/src/Solutions/sugarcrm.php b/src/Solutions/sugarcrm.php index eb43a2ba0..2b6c30214 100644 --- a/src/Solutions/sugarcrm.php +++ b/src/Solutions/sugarcrm.php @@ -407,7 +407,7 @@ public function read($param) } else { // If only deletion without new or modified record, we send no result. We wait for new or modified record. // Otherwise we will read the deleted record until a new or modified record is read because Sugar doesn't return modified date for deleted record. - return array(); + throw new \Exception('Only deletion records read. It is not possible to determine the reference date with only deletion. Waiting for a new record to be created in the source application for this rule.'); } } From 4fa7d3181ff03c40e9e020699cb618c38d16a283 Mon Sep 17 00:00:00 2001 From: Myddleware Date: Thu, 4 Apr 2024 00:18:20 +0200 Subject: [PATCH 20/88] Workflow (#1130) Feature : workflow (generate document and send notification) --- src/Entity/Document.php | 34 +++ src/Entity/Job.php | 37 +++ src/Entity/Rule.php | 36 +++ src/Entity/Workflow.php | 275 ++++++++++++++++++++ src/Entity/WorkflowAction.php | 215 +++++++++++++++ src/Entity/WorkflowLog.php | 204 +++++++++++++++ src/Manager/DocumentManager.php | 220 +++++++++++++--- src/Manager/RuleManager.php | 34 +++ src/Manager/ToolsManager.php | 34 +++ src/Repository/WorkflowActionRepository.php | 38 +++ src/Repository/WorkflowLogRepository.php | 38 +++ src/Repository/WorkflowRepository.php | 38 +++ src/Utils/workflowVariables.php | 3 + 13 files changed, 1169 insertions(+), 37 deletions(-) create mode 100644 src/Entity/Workflow.php create mode 100644 src/Entity/WorkflowAction.php create mode 100644 src/Entity/WorkflowLog.php create mode 100644 src/Repository/WorkflowActionRepository.php create mode 100644 src/Repository/WorkflowLogRepository.php create mode 100644 src/Repository/WorkflowRepository.php create mode 100644 src/Utils/workflowVariables.php diff --git a/src/Entity/Document.php b/src/Entity/Document.php index 1586a15cd..c3bbaf4d9 100755 --- a/src/Entity/Document.php +++ b/src/Entity/Document.php @@ -143,6 +143,11 @@ class Document */ private $logs; + /** + * @ORM\OneToMany(targetEntity="WorkflowLog", mappedBy="document") + */ + private $triggerDocuments; + /** * @ORM\Column(name="job_lock", type="string", length=23, nullable=false) */ @@ -153,6 +158,7 @@ public function __construct() { $this->datas = new ArrayCollection(); $this->logs = new ArrayCollection(); + $this->triggerDocuments = new ArrayCollection(); } public function setId($id): self @@ -452,6 +458,34 @@ public function removeLog(Log $log): self return $this; } + /** + * @return Collection|triggerDocument[] + */ + public function getTriggerDocuments(): Collection + { + return $this->triggerDocuments; + } + + public function addTriggerDocument(Workflowlog $triggerDocuments): self + { + if (!$this->triggerDocuments->contains($triggerDocument)) { + $this->triggerDocuments[] = $triggerDocument; + $triggerDocument->setDocument($this); + } + return $this; + } + + public function removeTriggerDocument(Workflowlog $triggerDocument): self + { + if ($this->triggerDocuments->removeElement($triggerDocument)) { + // set the owning side to null (unless already changed) + if ($triggerDocument->getDocument() === $this) { + $triggerDocument->setDocument(null); + } + } + return $this; + } + public function getCreatedBy(): ?User { return $this->createdBy; diff --git a/src/Entity/Job.php b/src/Entity/Job.php index 3da3c3e50..315305ce3 100755 --- a/src/Entity/Job.php +++ b/src/Entity/Job.php @@ -103,11 +103,20 @@ class Job * @ORM\OneToMany(targetEntity="Log", mappedBy="job") */ private $logs; + + /** + * @var WorkflowLog[] + * + * @ORM\OneToMany(targetEntity="WorkflowLog", mappedBy="job") + */ + private $workflowLogs; + public function __construct() { $this->begin = new DateTime(); $this->logs = new ArrayCollection(); + $this->workflowLogs = new ArrayCollection(); } public function setId($id): self @@ -284,4 +293,32 @@ public function removeLog(Log $log): self return $this; } + + /** + * @return Collection|WorkflowLog[] + */ + public function getWorkflowLogs(): Collection + { + return $this->workflowLogs; + } + + public function addWorkflowLogs(WorkflowLog $workflowLog): self + { + if (!$this->workflowLogs->contains($workflowLog)) { + $this->workflowLogs[] = $workflowLog; + $job->setJob($this); + } + return $this; + } + + public function removeWorkflowLog(WorkflowLog $workflowLog): self + { + if ($this->workflowLogs->removeElement($workflowLog)) { + // set the owning side to null (unless already changed) + if ($workflowLog->getWorkflow() === $this) { + $workflowLog->setJob(null); + } + } + return $this; + } } diff --git a/src/Entity/Rule.php b/src/Entity/Rule.php index e94a95d4b..a012cae37 100755 --- a/src/Entity/Rule.php +++ b/src/Entity/Rule.php @@ -155,6 +155,13 @@ class Rule */ private $audits; + /** + * @var Workflow[] + * + * @ORM\OneToMany(targetEntity="Workflow", mappedBy="rule") + */ + private $workflows; + /** * @var Document[] * @@ -177,6 +184,7 @@ public function __construct() $this->fields = new ArrayCollection(); $this->audits = new ArrayCollection(); $this->documents = new ArrayCollection(); + $this->workflows = new ArrayCollection(); } /** @@ -544,6 +552,34 @@ public function removeField(RuleField $field): self return $this; } + /** + * @return Collection|Workflow[] + */ + public function getWorkflows(): Collection + { + return $this->workflows; + } + + public function addWorkflows(Workflow $workflow): self + { + if (!$this->workflows->contains($workflow)) { + $this->workflows[] = $workflow; + $workflow->setRule($this); + } + return $this; + } + + public function removeWorkflow(Workflow $workflow): self + { + if ($this->workflows->removeElement($workflow)) { + // set the owning side to null (unless already changed) + if ($workflow->getRule() === $this) { + $workflow->setRule(null); + } + } + return $this; + } + /** * @return Collection|Document[] */ diff --git a/src/Entity/Workflow.php b/src/Entity/Workflow.php new file mode 100644 index 000000000..79a9d4d8c --- /dev/null +++ b/src/Entity/Workflow.php @@ -0,0 +1,275 @@ +. +*********************************************************************************/ + +namespace App\Entity; + +use DateTime; +use Doctrine\Common\Collections\ArrayCollection; +use Doctrine\Common\Collections\Collection; +use Doctrine\ORM\Mapping as ORM; +use Exception; + +/** + * @ORM\Table() + * @ORM\Entity(repositoryClass="App\Repository\WorkflowRepository") + * @ORM\Table(name="workflow", indexes={@ORM\Index(name="index_rule_id", columns={"rule_id"})}) + */ +class Workflow +{ + /** + * @ORM\Column(name="id", type="string") + * @ORM\Id + */ + private string $id; + + /** + * @ORM\ManyToOne(targetEntity="Rule", inversedBy="workflows") + * @ORM\JoinColumn(name="rule_id", referencedColumnName="id", nullable=false) + */ + private Rule $rule; + + /** + * @ORM\Column(name="date_created", type="datetime", nullable=false) + */ + private DateTime $dateCreated; + + /** + * @ORM\Column(name="date_modified", type="datetime", nullable=false) + */ + private DateTime $dateModified; + + /** + * @ORM\ManyToOne(targetEntity="User") + * @ORM\JoinColumn(name="created_by", referencedColumnName="id", nullable=false) + */ + private User $createdBy; + + /** + * @ORM\ManyToOne(targetEntity="User") + * @ORM\JoinColumn(name="modified_by", referencedColumnName="id", nullable=false) + */ + private User $modifiedBy; + + /** + * @ORM\Column(name="name", type="text", nullable=false) + */ + private string $name; + + /** + * @ORM\Column(name="description", type="text", nullable=true) + */ + private string $description; + + /** + * @ORM\Column(name="condition", type="text", nullable=false) + */ + private string $condition; + + /** + * @ORM\Column(name="deleted", type="boolean", options={"default":0}) + */ + private int $deleted; + + /** + * @var WorkflowAction[] + * + * @ORM\OneToMany(targetEntity="WorkflowAction", mappedBy="workflow") + */ + private $workflowActions; + + /** + * @var WorkflowLog[] + * + * @ORM\OneToMany(targetEntity="WorkflowLog", mappedBy="workflow") + */ + private $workflowLogs; + + public function __construct() + { + $this->workflowActions = new ArrayCollection(); + $this->workflowLogs = new ArrayCollection(); + } + + public function getId(): int + { + return $this->id; + } + + public function getRule(): ?Rule + { + return $this->rule; + } + + public function setRule(?Rule $rule): self + { + $this->rule = $rule; + return $this; + } + + public function setDateCreated($dateCreated): self + { + $this->dateCreated = $dateCreated; + return $this; + } + + public function getDateCreated(): DateTime + { + return $this->dateCreated; + } + + public function setDateModified($dateModified): self + { + $this->dateModified = $dateModified; + return $this; + } + + public function getDateModified(): DateTime + { + return $this->dateModified; + } + + public function getCreatedBy(): ?User + { + return $this->createdBy; + } + + public function setCreatedBy(?User $createdBy): self + { + $this->createdBy = $createdBy; + return $this; + } + + public function getModifiedBy(): ?User + { + return $this->modifiedBy; + } + + public function setModifiedBy(?User $modifiedBy): self + { + $this->modifiedBy = $modifiedBy; + return $this; + } + + public function setName($name): self + { + $this->name = $name; + return $this; + } + + public function getName(): string + { + return $this->name; + } + + public function setDescription($description): self + { + $this->description = $description; + return $this; + } + + public function getDescription(): string + { + return $this->description; + } + + public function getCondition(): array + { + return $this->condition; + } + + public function setCondition($condition): self + { + $this->condition = $condition; + return $this; + } + + public function setDeleted($deleted): self + { + $this->deleted = $deleted; + + return $this; + } + + public function getDeleted(): int + { + return $this->deleted; + } + + /** + * @return Collection|WorkflowAction[] + */ + public function getWorkflowActions(): Collection + { + return $this->workflowActions; + } + + public function addWorkflowActions(WorkflowAction $workflowAction): self + { + if (!$this->workflowActions->contains($workflowAction)) { + $this->workflowActions[] = $workflowAction; + $workflow->setWorkflow($this); + } + return $this; + } + + public function removeWorkflowAction(WorkflowAction $workflowAction): self + { + if ($this->workflowActions->removeElement($workflowAction)) { + // set the owning side to null (unless already changed) + if ($workflowAction->getWorkflow() === $this) { + $workflowAction->setWorkflow(null); + } + } + return $this; + } + + /** + * @return Collection|WorkflowLog[] + */ + public function getWorkflowLogs(): Collection + { + return $this->workflowLogs; + } + + public function addWorkflowLogs(WorkflowLog $workflowLog): self + { + if (!$this->workflowLogs->contains($workflowLog)) { + $this->workflowLogs[] = $workflowLog; + $workflow->setWorkflow($this); + } + return $this; + } + + public function removeWorkflowLog(WorkflowLog $workflowLog): self + { + if ($this->workflowLogs->removeElement($workflowLog)) { + // set the owning side to null (unless already changed) + if ($workflowLog->getWorkflow() === $this) { + $workflowLog->setWorkflow(null); + } + } + return $this; + } +} diff --git a/src/Entity/WorkflowAction.php b/src/Entity/WorkflowAction.php new file mode 100644 index 000000000..12f2595c7 --- /dev/null +++ b/src/Entity/WorkflowAction.php @@ -0,0 +1,215 @@ +. +*********************************************************************************/ + +namespace App\Entity; + +use DateTime; +use Doctrine\ORM\Mapping as ORM; + +/** + * @ORM\Table() + * @ORM\Entity(repositoryClass="App\Repository\WorkflowActionRepository") + * @ORM\Table(name="workflowaction", indexes={@ORM\Index(name="index_workflow_id", columns={"workflow_id"})}) + */ +class WorkflowAction +{ + /** + * @ORM\Column(name="id", type="string") + * @ORM\Id + */ + private string $id; + + /** + * @ORM\ManyToOne(targetEntity="Workflow", inversedBy="workflowActions") + * @ORM\JoinColumn(name="workflow_id", referencedColumnName="id", nullable=false) + */ + private Workflow $workflow; + + /** + * @ORM\Column(name="date_created", type="datetime", nullable=false) + */ + private DateTime $dateCreated; + + /** + * @ORM\Column(name="date_modified", type="datetime", nullable=false) + */ + private DateTime $dateModified; + + /** + * @ORM\ManyToOne(targetEntity="User") + * @ORM\JoinColumn(name="created_by", referencedColumnName="id", nullable=false) + */ + private User $createdBy; + + /** + * @ORM\ManyToOne(targetEntity="User") + * @ORM\JoinColumn(name="modified_by", referencedColumnName="id", nullable=false) + */ + private User $modifiedBy; + + /** + * @ORM\Column(name="name", type="text", nullable=false) + */ + private string $name; + + /** + * @ORM\Column(name="action", type="text", nullable=false) + */ + private string $action; + + /** + * @ORM\Column(name="arguments", type="array", nullable=false) + */ + private $arguments; + + /** + * @ORM\Column(name="description", type="text", nullable=true) + */ + private string $description; + + /** + * @ORM\Column(name="order", type="integer", length=3, nullable=false) + */ + private int $order; + + + public function getId(): string + { + return $this->id; + } + + public function getWorkflow(): ?Workflow + { + return $this->workflow; + } + + public function setWorkflow(?Workflow $workflow): self + { + $this->workflow = $workflow; + return $this; + } + + public function setDateCreated($dateCreated): self + { + $this->dateCreated = $dateCreated; + + return $this; + } + + public function getDateCreated(): DateTime + { + return $this->dateCreated; + } + + public function setDateModified($dateModified): self + { + $this->dateModified = $dateModified; + + return $this; + } + + public function getDateModified(): DateTime + { + return $this->dateModified; + } + + public function getCreatedBy(): ?User + { + return $this->createdBy; + } + + public function setCreatedBy(?User $createdBy): self + { + $this->createdBy = $createdBy; + return $this; + } + + public function getModifiedBy(): ?User + { + return $this->modifiedBy; + } + + public function setModifiedBy(?User $modifiedBy): self + { + $this->modifiedBy = $modifiedBy; + return $this; + } + + public function setName($name): self + { + $this->name = $name; + return $this; + } + + public function getName(): string + { + return $this->name; + } + + public function getAction(): array + { + return $this->action; + } + + public function setAction($action): self + { + $this->action = $action; + return $this; + } + + public function getArguments(): array + { + return $this->arguments; + } + + public function setArguments($arguments): self + { + $this->arguments = $arguments; + return $this; + } + + public function setDescription($description): self + { + $this->description = $description; + return $this; + } + + public function getDescription(): string + { + return $this->description; + } + + public function setOrder($order): self + { + $this->order = $order; + + return $this; + } + + public function getOrder(): string + { + return $this->order; + } +} diff --git a/src/Entity/WorkflowLog.php b/src/Entity/WorkflowLog.php new file mode 100644 index 000000000..b0c80262c --- /dev/null +++ b/src/Entity/WorkflowLog.php @@ -0,0 +1,204 @@ +. +*********************************************************************************/ + +namespace App\Entity; + +use DateTime; +use Doctrine\ORM\Mapping as ORM; +use Exception; + +/** + * @ORM\Table() + * @ORM\Entity(repositoryClass="App\Repository\WorkflowLogRepository") + * @ORM\Table(name="workflowlog", indexes={@ORM\Index(name="index_workflow_id", columns={"workflow_id"})}) + * @ORM\Table(name="workflowlog", indexes={@ORM\Index(name="index_job_id", columns={"job_id"})}) + */ +class WorkflowLog +{ + /** + * @ORM\Column(name="id", type="integer") + * @ORM\Id + * @ORM\GeneratedValue(strategy="AUTO") + */ + private int $id; + + /** + * @ORM\ManyToOne(targetEntity="Workflow", inversedBy="workflowLogs") + * @ORM\JoinColumn(name="workflow_id", referencedColumnName="id", nullable=false) + */ + private Workflow $workflow; + + /** + * @ORM\ManyToOne(targetEntity="Job", inversedBy="workflowLogs") + * @ORM\JoinColumn(name="job_id", referencedColumnName="id", nullable=false) + */ + private Job $job; + + /** + * @ORM\ManyToOne(targetEntity="Document", inversedBy="triggerDocuments") + * @ORM\JoinColumn(name="trigger_document_id", referencedColumnName="id", nullable=false) + */ + private Document $triggerDocument; + + /** + * @ORM\ManyToOne(targetEntity="Document", inversedBy="generateDocuments") + * @ORM\JoinColumn(name="generate_document_id", referencedColumnName="id", nullable=true) + */ + private ?Document $generateDocument= null; + + /** + * @ORM\ManyToOne(targetEntity="WorkflowAction") + * @ORM\JoinColumn(name="action_id", referencedColumnName="id", nullable=true) + */ + private WorkflowAction $action; + + /** + * @ORM\Column(name="status", type="string", nullable=true, options={"default":NULL}) + */ + private ?string $status; + + /** + * @ORM\Column(name="date_created", type="datetime", nullable=false) + */ + private DateTime $dateCreated; + + /** + * @ORM\ManyToOne(targetEntity="User") + * @ORM\JoinColumn(name="created_by", referencedColumnName="id", nullable=true) + */ + private User $createdBy; + + /** + * @ORM\Column(name="message", type="text", nullable=true) + */ + private string $message; + + + public function getId(): int + { + return $this->id; + } + + public function getWorkflow(): ?Workflow + { + return $this->workflow; + } + + public function setWorkflow(?Workflow $workflow): self + { + $this->workflow = $workflow; + return $this; + } + + public function getJob(): ?Job + { + return $this->job; + } + + public function setJob(?Job $job): self + { + $this->job = $job; + return $this; + } + + public function getTriggerDocument(): ?Document + { + return $this->triggerDocument; + } + + public function setTriggerDocument(?Document $document): self + { + $this->triggerDocument = $document; + return $this; + } + + public function getGenerateDocument(): ?Document + { + return $this->generateDocument; + } + + public function setGenerateDocument(?Document $document): self + { + $this->generateDocument = $document; + return $this; + } + + public function setStatus($status): self + { + $this->status = $status; + return $this; + } + + public function getStatus(): ?string + { + return $this->status; + } + + public function setDateCreated($dateCreated): self + { + $this->dateCreated = $dateCreated; + + return $this; + } + + public function getDateCreated(): DateTime + { + return $this->dateCreated; + } + + + public function getCreatedBy(): ?User + { + return $this->createdBy; + } + + public function setCreatedBy(?User $createdBy): self + { + $this->createdBy = $createdBy; + return $this; + } + + public function getAction(): array + { + return $this->action; + } + + public function setAction($action): self + { + $this->action = $action; + return $this; + } + + public function setMessage($message): self + { + $this->message = $message; + return $this; + } + + public function getMessage(): string + { + return $this->message; + } +} diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 1a9fd7783..1875d94ea 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -28,6 +28,10 @@ use App\Entity\Document; use App\Entity\DocumentData; use App\Entity\DocumentData as DocumentDataEntity; +use App\Entity\Workflow; +use App\Entity\WorkflowLog; +use App\Entity\WorkflowAction; +use App\Entity\Job; use App\Entity\DocumentRelationship as DocumentRelationship; use App\Repository\DocumentRepository; use App\Repository\RuleRelationShipRepository; @@ -36,6 +40,8 @@ use Exception; use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface; +use Swift_Mailer; +use Swift_Message; class documentcore { @@ -51,6 +57,7 @@ class documentcore protected $ruleId; protected $ruleFields; protected $ruleRelationships; + protected $ruleWorkflows; protected $ruleParams; protected $sourceId; protected $targetId; @@ -298,6 +305,9 @@ public function setParam($param, $clear = false, $clearRule = true) if (!empty($param['ruleRelationships'])) { $this->ruleRelationships = $param['ruleRelationships']; } + if (!empty($param['ruleWorkflows'])) { + $this->ruleWorkflows = $param['ruleWorkflows']; + } // Init type error for each new document $this->typeError = 'S'; } catch (\Exception $e) { @@ -320,6 +330,7 @@ protected function clearAttributes($clearRule = true) $this->ruleId = ''; $this->ruleFields = []; $this->ruleRelationships = []; + $this->ruleWorkflows = []; $this->ruleParams = []; } $this->id = ''; @@ -2109,6 +2120,7 @@ public function updateStatus($new_status) $this->status = $new_status; $this->afterStatusChange($new_status); $this->createDocLog(); + $this->runWorkflow(); } catch (\Exception $e) { $this->message .= 'Error status update : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; @@ -2503,8 +2515,177 @@ public function getStatus() { return $this->status; } + + protected function runWorkflow() { + try { + // Check if at least on workflow exist for the rule + if (!empty($this->ruleWorkflows)) { + // includ variables used in the formula + include __DIR__.'/../Utils/workflowVariables.php'; + if (file_exists( __DIR__.'/../Custom/Utils/workflowVariables.php')) { + include __DIR__.'/../Custom/Utils/workflowVariables.php'; + } + // Execute every workflow of the rule + foreach ($this->ruleWorkflows as $ruleWorkflow) { + // Check the condition + $this->formulaManager->init($ruleWorkflow['condition']); // mise en place de la règle dans la classe + $this->formulaManager->generateFormule(); // Genère la nouvelle formule à la forme PhP + $f = $this->formulaManager->execFormule(); + eval('$condition = '.$f.';'); // exec + // Execute the action if the condition is met + if ($condition == 1) { + // Execute all actions + if (!empty($ruleWorkflow['actions'])) { + // Call each actions + foreach($ruleWorkflow['actions'] as $action) { + // Check if the action has already been executed for the current document + // Only if attempt > 0, if it is the first attempt then the action has never been executed + if ($this->attempt > 0) { + // Search action for the current document + $workflowLogEntity = $this->entityManager->getRepository(WorkflowLog::class) + ->findOneBy([ + 'triggerDocument' => $this->id, + 'action' => $action['id'], + ] + ); + // If the current action has been found for the current document, we don't execute the current action + if ( + !empty($workflowLogEntity) + AND $workflowLogEntity->getStatus() == 'Success' + ) { + // GenerateDocument can be empty depending the action + if (!empty($workflowLogEntity->getGenerateDocument())) { + $this->docIdRefError = $workflowLogEntity->getGenerateDocument()->getId(); + } + $this->generateDocLog('W','Action ' . $action['id'] . ' already executed for this document. '); + continue; + } + } - /** + // Execute action depending of the function in the workflow + $arguments = unserialize($action['arguments']); + switch ($action['action']) { + case 'generateDocument': + $this->generateDocument($arguments['ruleId'],$this->sourceData[$arguments['searchValue']],$arguments['searchField'],$arguments['rerun'], $action); + break; + case 'sendNotification': + try { + $workflowStatus = 'Success'; + $error = ''; + // Method sendMessage throws an exception if it fails + $this->tools->sendMessage($arguments['to'],$arguments['subject'],$arguments['message']); + } catch (\Exception $e) { + $workflowStatus = 'Error'; + $error = 'Failed to create workflow log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $this->logger->error($error); + $this->generateDocLog('E',$error); + } + $this->createWorkflowLog($action, $workflowStatus, $error); + break; + default: + throw new \Exception('Function '.key($action).' unknown.'); + } + } + } + } + } + } + } catch (\Exception $e) { + $this->logger->error('Failed to create workflow log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $this->generateDocLog('E','Failed to create workflow log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + } + + // Generate a document using the rule id and search parameters + protected function generateDocument($ruleId, $searchValue = null, $searchField = 'id', $rerun = true, $action = null) + { + try { + // Instantiate the rule + $rule = new RuleManager($this->logger, $this->connection, $this->entityManager, $this->parameterBagInterface, $this->formulaManager, $this->solutionManager, clone $this); + $rule->setRule($ruleId); + $rule->setJobId($this->jobId); + + if (empty($searchValue)) { + $searchValue = $this->sourceId; + } + + // Generate the documents depending on the search parameter + $documents = $rule->generateDocuments($searchValue, true, '', $searchField); + if (!empty($documents->error)) { + throw new \Exception($documents->error); + } + // Run documents + if ( + !empty($documents) + and $rerun + ) { + foreach ($documents as $doc) { + $errors = $rule->actionDocument($doc->id, 'rerun'); + // Check errors + if (!empty($errors)) { + $this->message .= 'Document ' . $doc->id . ' in error (rule ' . $ruleId . ' : ' . $errors[0] . '. '; + } + // Generate the workflow log for each document if it has been generated by a workflow + if (!empty($action['id'])) { + $error = ''; + if (!empty($errors)) { + $error = $this->message; + $status = 'Error'; + } else { + $status = 'Success'; + } + $this->createWorkflowLog($action, $status, $error, $doc->id); + } + } + } + } catch (\Exception $e) { + $this->logger->error('Error : ' . $e->getMessage() . ' ' . $e->getFile() . ' Line : ( ' . $e->getLine() . ' )'); + $this->generateDocLog('E',$this->message); + } + } + + // Create a workflow log + protected function createWorkflowLog($action, $status, $error=null, $generateDocumentId=null) { + try { + // Generate the workflow log + $workflowLog = new WorkflowLog(); + // Set the current document + $triggerDocumentEntity = $this->entityManager->getRepository(Document::class)->find($this->id); + $workflowLog->setTriggerDocument($triggerDocumentEntity); + // Set the current action + $workflowActionEntity = $this->entityManager->getRepository(WorkflowAction::class)->find($action['id']); + $workflowLog->setAction($workflowActionEntity); + // Set the generated document if the action has generated a document + if (!empty($generateDocumentId)) { + $generateDocumentEntity = $this->entityManager->getRepository(Document::class)->find($generateDocumentId); + $this->docIdRefError = $generateDocumentId; + $workflowLog->setGenerateDocument($generateDocumentEntity); + } + // Set the workflow + $workflowEntity = $this->entityManager->getRepository(Workflow::class)->find($action['workflow_id']); + $workflowLog->setWorkflow($workflowEntity); + // Set the job + $jobEntity = $this->entityManager->getRepository(Job::class)->find($this->jobId); + $workflowLog->setJob($jobEntity); + // Set the creation date + $workflowLog->setDateCreated(new \DateTime()); + // Set the status depending on the error message + if (!empty($errors)) { + $workflowLog->setMessage($error); + $workflowLog->setStatus($status); + } else { + $workflowLog->setStatus('Success');; + } + $this->entityManager->persist($workflowLog); + $this->entityManager->flush(); + // Generate a document log. + $this->generateDocLog('S','Action '.$action['id'].' executed. '.(!empty($generateDocumentId) ? 'The document '.$generateDocumentId.' has been generated. ' : '')); + } catch (\Exception $e) { + $this->logger->error('Error : ' . $e->getMessage() . ' ' . $e->getFile() . ' Line : ( ' . $e->getLine() . ' )'); + $this->generateDocLog('E','Error : ' . $e->getMessage() . ' ' . $e->getFile() . ' Line : ( ' . $e->getLine() . ' )'); + } + } + /** * @throws \Doctrine\DBAL\Exception * Les id de la soluton, de la règle et du document * $type peut contenir : I (info;), W(warning), E(erreur), S(succès) @@ -2527,6 +2708,7 @@ protected function createDocLog() $stmt->bindValue(':job_id', $this->jobId); $result = $stmt->executeQuery(); $this->message = ''; + $this->docIdRefError = ''; } catch (\Exception $e) { $this->logger->error('Failed to create log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } @@ -2542,42 +2724,6 @@ public function generateDocLog($errorType, $message) $this->createDocLog(); } - // Generate a document using the rule id and search parameters - protected function generateDocument($ruleId, $searchValue = null, $searchField = 'id', $rerun = true) - { - try { - // Instantiate the rule - $rule = new RuleManager($this->logger, $this->connection, $this->entityManager, $this->parameterBagInterface, $this->formulaManager, $this->solutionManager, clone $this); - $rule->setRule($ruleId); - $rule->setJobId($this->jobId); - - if (empty($searchValue)) { - $searchValue = $this->sourceId; - } - // $this->sourceId = engagé ID - // Cherche tous les pôles de l'enregistrement correspondant à la règle - $documents = $rule->generateDocuments($searchValue, true, '', $searchField); - if (!empty($documents->error)) { - throw new \Exception($documents->error); - } - // Run documents - if ( - !empty($documents) - and $rerun - ) { - foreach ($documents as $doc) { - $errors = $rule->actionDocument($doc->id, 'rerun'); - // Check errors - if (!empty($errors)) { - $this->message .= 'Document ' . $doc->id . ' in error (rule ' . $ruleId . ' : ' . $errors[0] . '. '; - } - } - } - } catch (\Exception $e) { - $this->message .= 'Error : ' . $e->getMessage(); - $this->logger->error('Error : ' . $e->getMessage() . ' ' . $e->getFile() . ' Line : ( ' . $e->getLine() . ' )'); - } - } } class DocumentManager extends documentcore diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 55d4721d2..bbd96068b 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -61,6 +61,7 @@ class rulecore protected $sourceFields; protected $targetFields; protected $ruleRelationships; + protected $ruleWorkflows; protected $ruleFilters; protected $solutionSource; protected $solutionTarget; @@ -153,6 +154,7 @@ public function setRule($idRule) $this->setRuleParam(); $this->setLimit(); $this->setRuleRelationships(); + $this->setRuleWorkflows(); // Set the rule fields (we use the name_slug in $this->rule) $this->setRuleField(); } @@ -270,6 +272,7 @@ public function generateDocuments($idSource, $readSource = true, $param = '', $i $docParam['rule'] = $this->rule; $docParam['ruleFields'] = $this->ruleFields; $docParam['ruleRelationships'] = $this->ruleRelationships; + $docParam['ruleWorkflows'] = $this->ruleWorkflows; $docParam['data'] = $docData; $docParam['jobId'] = $this->jobId; $docParam['api'] = $this->api; @@ -401,6 +404,7 @@ public function createDocuments() $param['rule'] = $this->rule; $param['ruleFields'] = $this->ruleFields; $param['ruleRelationships'] = $this->ruleRelationships; + $param['ruleWorkflows'] = $this->ruleWorkflows; // Set the param of the rule one time for all $this->documentManager->setRuleId($this->ruleId); $this->documentManager->setRuleParam(); @@ -725,6 +729,7 @@ public function checkPredecessorDocuments($documents = null): array $param['jobId'] = $this->jobId; $param['api'] = $this->api; $param['ruleRelationships'] = $this->ruleRelationships; + $param['ruleWorkflows'] = $this->ruleWorkflows; // Set the param values and clear all document attributes if($this->documentManager->setParam($param, true)) { $response[$document['id']] = $this->documentManager->checkPredecessorDocument(); @@ -758,6 +763,7 @@ public function checkParentDocuments($documents = null): array if (!empty($documents)) { $param['jobId'] = $this->jobId; $param['ruleRelationships'] = $this->ruleRelationships; + $param['ruleWorkflows'] = $this->ruleWorkflows; // Set all config parameters $this->setConfigParam(); // If migration mode, we select all documents to improve performance. For example, we won't execute queries is method document->getTargetId @@ -782,6 +788,7 @@ public function checkParentDocuments($documents = null): array $param['jobId'] = $this->jobId; $param['api'] = $this->api; $param['ruleRelationships'] = $this->ruleRelationships; + $param['ruleWorkflows'] = $this->ruleWorkflows; // Set the param values and clear all document attributes if($this->documentManager->setParam($param, true)) { $response[$document['id']] = $this->documentManager->checkParentDocument(); @@ -814,6 +821,7 @@ public function transformDocuments($documents = null): array if (!empty($documents)) { $param['ruleFields'] = $this->ruleFields; $param['ruleRelationships'] = $this->ruleRelationships; + $param['ruleWorkflows'] = $this->ruleWorkflows; $param['jobId'] = $this->jobId; $param['api'] = $this->api; // Set all config parameters @@ -885,6 +893,7 @@ public function getTargetDataDocuments($documents = null): array $param['solutionTarget'] = $this->solutionTarget; $param['ruleFields'] = $this->ruleFields; $param['ruleRelationships'] = $this->ruleRelationships; + $param['ruleWorkflows'] = $this->ruleWorkflows; $param['jobId'] = $this->jobId; $param['api'] = $this->api; // Set the param values and clear all document attributes @@ -1484,6 +1493,7 @@ protected function sendTarget($type, $documentId = null): array $send['ruleFields'] = $this->ruleFields; $send['ruleParams'] = $this->ruleParams; $send['ruleRelationships'] = $this->ruleRelationships; + $send['ruleWorkflows'] = $this->ruleWorkflows; $send['jobId'] = $this->jobId; // Si des données sont prêtes à être créées if (!empty($send['data'])) { @@ -1609,6 +1619,7 @@ protected function massSendTarget($type, $documentId = null) $send['ruleFields'] = $this->ruleFields; $send['ruleParams'] = $this->ruleParams; $send['ruleRelationships'] = $this->ruleRelationships; + $send['ruleWorkflows'] = $this->ruleWorkflows; $send['jobId'] = $this->jobId; // Si des données sont prêtes à être créées if (!empty($send['data'])) { @@ -2119,6 +2130,29 @@ protected function setRuleRelationships() } } + // Permet de charger toutes les relations de la règle + protected function setRuleWorkflows() + { + try { + $sqlWorkflows = 'SELECT * FROM workflow WHERE rule_id = :ruleId'; + $stmt = $this->connection->prepare($sqlWorkflows); + $stmt->bindValue(':ruleId', $this->ruleId); + $result = $stmt->executeQuery(); + $this->ruleWorkflows = $result->fetchAllAssociative(); + if (!empty($this->ruleWorkflows)) { + foreach($this->ruleWorkflows as $key => $workflow) { + $sqlActions = 'SELECT * FROM workflowaction WHERE workflow_id = :workflowid ORDER BY `order` ASC'; + $stmt = $this->connection->prepare($sqlActions); + $stmt->bindValue(':workflowid', $workflow['id']); + $result = $stmt->executeQuery(); + $this->ruleWorkflows[$key]['actions'] = $result->fetchAllAssociative(); + } + } + } catch (\Exception $e) { + $this->logger->error('Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + } + // Set the limit rule protected function setLimit() { diff --git a/src/Manager/ToolsManager.php b/src/Manager/ToolsManager.php index 47acd3559..a247e4192 100644 --- a/src/Manager/ToolsManager.php +++ b/src/Manager/ToolsManager.php @@ -32,6 +32,9 @@ use Symfony\Component\HttpKernel\KernelInterface; use Symfony\Component\Process\PhpExecutableFinder; use Symfony\Component\Yaml\Yaml; +use Swift_Mailer; +use Swift_Message; +use Swift_SmtpTransport; class toolscore { @@ -215,6 +218,37 @@ public function getParamValue($paramName) } return null; } + + // Send a message using Brevo or SMTP parameters + public function sendMessage($to, $subject, $message) { + // Use Brevo if the key is set + if (!empty($_ENV['SENDINBLUE_APIKEY'])) { + try { + $sendinblue = \SendinBlue\Client\Configuration::getDefaultConfiguration()->setApiKey('api-key', $_ENV['SENDINBLUE_APIKEY']); + $apiInstance = new \SendinBlue\Client\Api\TransactionalEmailsApi(new \GuzzleHttp\Client(), $sendinblue); + $sendSmtpEmail = new \SendinBlue\Client\Model\SendSmtpEmail(); // \SendinBlue\Client\Model\SendSmtpEmail | Values to send a transactional email + $sendSmtpEmail['to'] = array(array('email' => $to)); + $sendSmtpEmail['subject'] = $subject; + $sendSmtpEmail['htmlContent'] = $message; + $sendSmtpEmail['sender'] = array('email' => $this->configParams['email_from'] ?? 'no-reply@myddleware.com'); + $result = $apiInstance->sendTransacEmail($sendSmtpEmail); + } catch (Exception $e) { + throw new Exception('Exception when calling TransactionalEmailsApi->sendTransacEmail: '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + } else { + $swiftMessage = + (new Swift_Message($subject)) + ->setFrom($this->configParams['email_from'] ?? 'no-reply@myddleware.com') + ->setBody($message); + // Send message + $message->setTo($to); + $send = $this->mailer->send($message); + if (!$send) { + $this->logger->error('Failed to send alert email : '.$message.' to '.$to); + throw new Exception('Failed to send alert email : '.$message.' to '.$to); + } + } + } } class ToolsManager extends toolscore diff --git a/src/Repository/WorkflowActionRepository.php b/src/Repository/WorkflowActionRepository.php new file mode 100644 index 000000000..08ec6af83 --- /dev/null +++ b/src/Repository/WorkflowActionRepository.php @@ -0,0 +1,38 @@ +. +*********************************************************************************/ + +namespace App\Repository; + +use App\Entity\WorkflowAction; +use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; +use Doctrine\Persistence\ManagerRegistry; + +class WorkflowActionRepository extends ServiceEntityRepository +{ + public function __construct(ManagerRegistry $registry) + { + parent::__construct($registry, WorkflowAction::class); + } +} diff --git a/src/Repository/WorkflowLogRepository.php b/src/Repository/WorkflowLogRepository.php new file mode 100644 index 000000000..b0c097096 --- /dev/null +++ b/src/Repository/WorkflowLogRepository.php @@ -0,0 +1,38 @@ +. +*********************************************************************************/ + +namespace App\Repository; + +use App\Entity\WorkflowLog; +use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; +use Doctrine\Persistence\ManagerRegistry; + +class WorkflowLogRepository extends ServiceEntityRepository +{ + public function __construct(ManagerRegistry $registry) + { + parent::__construct($registry, WorkflowLog::class); + } +} diff --git a/src/Repository/WorkflowRepository.php b/src/Repository/WorkflowRepository.php new file mode 100644 index 000000000..20469fe0d --- /dev/null +++ b/src/Repository/WorkflowRepository.php @@ -0,0 +1,38 @@ +. +*********************************************************************************/ + +namespace App\Repository; + +use App\Entity\Workflow; +use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; +use Doctrine\Persistence\ManagerRegistry; + +class WorkflowRepository extends ServiceEntityRepository +{ + public function __construct(ManagerRegistry $registry) + { + parent::__construct($registry, Workflow::class); + } +} diff --git a/src/Utils/workflowVariables.php b/src/Utils/workflowVariables.php new file mode 100644 index 000000000..557a8182c --- /dev/null +++ b/src/Utils/workflowVariables.php @@ -0,0 +1,3 @@ +status; + From c19ee6a42dd4fc9a96e95d937be4ea2cf7be42c9 Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 4 Apr 2024 23:09:39 +0200 Subject: [PATCH 21/88] Compare field : manage date field Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 1875d94ea..36f8c2ae7 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1259,7 +1259,14 @@ protected function checkNoChange($history): bool ) { continue; } - + // In case of date with different format (2024-03-14T11:04:16+00:00 == 2024-03-14T12:04:16+01:00) + if ( + !empty(strtotime($history[$field['target_field_name']])) + AND !empty(strtotime($target[$field['target_field_name']])) + AND strtotime($history[$field['target_field_name']]) == strtotime($target[$field['target_field_name']]) + ) { + continue; + } return false; } } From d2bec31bb18c51b6bb928745c3a5a1cd0c8d7e51 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 5 Apr 2024 18:07:27 +0200 Subject: [PATCH 22/88] Avoid to have several result after the serach duplicates Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 7 ++- src/Solutions/salesforce.php | 104 +++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 29 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 36f8c2ae7..25ffa4717 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1329,7 +1329,11 @@ protected function getDocumentHistory($searchFields) } // If read method returns a result else { - // Select the first result + // Error il multiple duplicate records found + if (count($dataTarget['values']) > 1) { + $this->message .= count($dataTarget['values']).' duplicates found. Only one duplicate maximum can be found.'; + return -1; + } $record = current($dataTarget['values']); $updateHistory = $this->updateHistoryTable($record); if (true === $updateHistory) { @@ -1338,7 +1342,6 @@ protected function getDocumentHistory($searchFields) // Erreur dans la mise à jour de la table historique else { $this->message .= $dataTarget['error']; - return -1; } } diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index 7548138a5..74e5b52fd 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -73,6 +73,8 @@ class salesforcecore extends solution { ); protected string $versionApi = 'v38.0'; + + protected bool $sendDeletion = true; // Connexion à Salesforce - Instancie la classe salesforce et affecte access_token et instance_url public function login($paramConnexion) { @@ -656,10 +658,12 @@ public function updateData($param): array ) { $parameters['Pricebook2Id'] = $param['ruleParams']['Pricebook2Id']; } - + if (empty($target_id)) { + throw new \Exception ('The target id is requiered for an update.'); + } $parameters = json_encode($parameters); // Appel de la requête - $query_request_data = $this->call($query_url, $parameters, true); + $query_request_data = $this->call($query_url, $parameters, 'PATCH'); if ($query_request_data === true) { $result[$idDoc] = array( @@ -687,6 +691,54 @@ public function updateData($param): array return $result; } + /** + * @throws \Doctrine\DBAL\Exception + * @throws \Exception + */ + public function deleteData($param): array + { + if(!(isset($param['data']))) { + throw new \Exception ('Data missing for update'); + } + foreach($param['data'] as $idDoc => $data) { + try{ + // Check control before update + $data = $this->checkDataBeforeDelete($param, $data, $idDoc); + // Instanciation de l'URL d'appel + if (empty($data['target_id'])) { + throw new \Exception ('The target id is requiered for a deletion.'); + } + $query_url = $this->instance_url."/services/data/".$this->versionApi."/sobjects/".$param['module'].'/'.$data['target_id']; + + // Appel de la requête + $query_request_data = $this->call($query_url, true, 'DELETE'); + + if ($query_request_data === true) { + $result[$idDoc] = array( + 'id' => $data['target_id'], + 'error' => false + ); + } + else { + $result[$idDoc] = array( + 'id' => '-1', + 'error' => 'Failed to update Data in Salesforce : '.print_r($query_request_data['errors'],true), + ); + } + } + catch (\Exception $e) { + $error = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $result[$idDoc] = array( + 'id' => '-1', + 'error' => $error + ); + } + // Modification du statut du flux + $this->updateDocumentStatus($idDoc,$result[$idDoc],$param); + } + return $result; + } + // Permet de formater la réponse si besoin protected function formatResponse($param,$query_request_data) { return $query_request_data; @@ -718,8 +770,13 @@ protected function getFrom($param): string protected function getWhere($param): string { if (!empty($param['query'])) { + $queryWhere = "+WHERE+"; foreach ($param['query'] as $key => $value) { - $queryWhere = "+WHERE+".$key."+=+'".$value."'"; + $queryWhere .= $key."+=+'".$value."'"; + // Add the AND if not the last entry of the array + if ($key !== array_key_last($param['query'])) { + $queryWhere .= "+AND+"; + } } } else { // On va chercher le nom du champ pour la date de référence: Création ou Modification @@ -796,44 +853,37 @@ public function getRefFieldName($param): string } return ""; } - - // Fonction permettant de faire l'appel REST /** * @throws \Exception */ - protected function call($url, $parameters, $update = false){ + protected function call($url, $parameters, $method = null){ ob_start(); $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); // important (testé en Local wamp) afin de ne pas vérifier le certificat SSL if($parameters === false){ // Si l'appel ne possède pas de paramètres, on exécute un GET en curl - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); curl_setopt($ch, CURLOPT_HTTPHEADER, array('Authorization: OAuth '.$this->access_token)); - } - elseif ($update === false) { // Si l'appel en revanche possède des paramètres dans $parameters, on exécute un POST en curl - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); - if(!isset($parameters['grant_type'])) // A ne pas ajouter pour la connexion - curl_setopt($ch, CURLOPT_HTTPHEADER, array("Authorization: OAuth " . $this->access_token, "Content-type: application/json")); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); // important (testé en Local wamp) afin de ne pas vérifier le certificat SSL - curl_setopt($ch, CURLOPT_POST, TRUE); - curl_setopt($ch, CURLOPT_POSTFIELDS, $parameters); - } - else { - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); + } else { + // No Authorization in case of login action + if(!isset($parameters['grant_type'])) { + curl_setopt($ch, CURLOPT_HTTPHEADER, array("Authorization: OAuth " . $this->access_token, "Content-type: application/json")); + } + // PATCH or DELETE + if (!empty($method)) { + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); + // POST + } else { + curl_setopt($ch, CURLOPT_POST, TRUE); + } curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); // important (testé en Local wamp) afin de ne pas vérifier le certificat SSL - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PATCH'); - curl_setopt($ch, CURLOPT_HTTPHEADER, array("Authorization: OAuth " . $this->access_token, "Content-type: application/json")); curl_setopt($ch, CURLOPT_POSTFIELDS, $parameters); } $query_request_body = curl_exec($ch); // Si on est sur un update et que l'on a un retour 204 on renvoie true - if ($update === true) { + if (!empty($method)) { if(curl_getinfo($ch, CURLINFO_HTTP_CODE) == '204') { return true; } From 6a7e3185f0a5e233a9ee41771680033eb41e7fe2 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 5 Apr 2024 18:08:33 +0200 Subject: [PATCH 23/88] Salesforce connector : add delete action Signed-off-by: myddleware --- src/Solutions/salesforce.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index 74e5b52fd..d583c4063 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -699,7 +699,7 @@ public function deleteData($param): array { if(!(isset($param['data']))) { throw new \Exception ('Data missing for update'); - } + } foreach($param['data'] as $idDoc => $data) { try{ // Check control before update From 58d27e5c51243603219fce30bd47ede2ff5a9fe5 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 10 Apr 2024 16:21:49 +0200 Subject: [PATCH 24/88] Database connector : manage reference field when it is a numeric Signed-off-by: myddleware --- src/Solutions/database.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Solutions/database.php b/src/Solutions/database.php index cc55443ad..6b4d118a7 100644 --- a/src/Solutions/database.php +++ b/src/Solutions/database.php @@ -215,7 +215,7 @@ public function readData($param) $param['date_ref'] = 0; } $result['date_ref'] = $param['date_ref']; - + if (empty($param['limit'])) { $param['limit'] = 100; } @@ -347,7 +347,10 @@ public function readData($param) // If the reference isn't a valid date (it could be an ID in case there is no date in the table) we set the current date if ((bool) strtotime($value)) { $row['date_modified'] = $value; - } else { + // If the ref field is a numeric (increment), we transform it to a date (timestamp) to be able to save the corresponding date to the document + } elseif (is_numeric($value)) { + $row['date_modified'] = date('Y-m-d H:i:s', $value); + } else { $row['date_modified'] = date('Y-m-d H:i:s'); } $result['date_ref'] = $row['date_modified']; @@ -379,7 +382,6 @@ public function readData($param) } catch (Exception $e) { $result['error'] = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; } - return $result; } From 119e5e5f097ac82f1d8d7ceb59738eb551696182 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 10 Apr 2024 16:30:40 +0200 Subject: [PATCH 25/88] Database connector : manage ref rule field in case of the reference is an id Signed-off-by: myddleware --- src/Solutions/database.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Solutions/database.php b/src/Solutions/database.php index 6b4d118a7..06cc5f1ab 100644 --- a/src/Solutions/database.php +++ b/src/Solutions/database.php @@ -347,13 +347,15 @@ public function readData($param) // If the reference isn't a valid date (it could be an ID in case there is no date in the table) we set the current date if ((bool) strtotime($value)) { $row['date_modified'] = $value; + $result['date_ref'] = $row['date_modified']; // If the ref field is a numeric (increment), we transform it to a date (timestamp) to be able to save the corresponding date to the document } elseif (is_numeric($value)) { $row['date_modified'] = date('Y-m-d H:i:s', $value); + $result['date_ref'] = $value; } else { $row['date_modified'] = date('Y-m-d H:i:s'); + $result['date_ref'] = $row['date_modified']; } - $result['date_ref'] = $row['date_modified']; } } elseif ('history' == $param['call_type']) { // Id is fieldId for a history action if ($key === $param['ruleParams']['targetFieldId']) { From e86740868321443d2bb82e5c3c2acef55f6a1f6a Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 10 Apr 2024 16:33:29 +0200 Subject: [PATCH 26/88] Change comment Signed-off-by: myddleware --- src/Solutions/database.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Solutions/database.php b/src/Solutions/database.php index 06cc5f1ab..523169215 100644 --- a/src/Solutions/database.php +++ b/src/Solutions/database.php @@ -344,7 +344,7 @@ public function readData($param) $row['id'] = $value; } if ($key === $param['ruleParams']['fieldDateRef']) { - // If the reference isn't a valid date (it could be an ID in case there is no date in the table) we set the current date + // If the reference is a valid date, we save it if ((bool) strtotime($value)) { $row['date_modified'] = $value; $result['date_ref'] = $row['date_modified']; @@ -352,6 +352,7 @@ public function readData($param) } elseif (is_numeric($value)) { $row['date_modified'] = date('Y-m-d H:i:s', $value); $result['date_ref'] = $value; + // In all other cases, we set the current date } else { $row['date_modified'] = date('Y-m-d H:i:s'); $result['date_ref'] = $row['date_modified']; From 0daeda9703a7cad4a4262a7581080129495d0d24 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 00:39:04 +0200 Subject: [PATCH 27/88] Fix workflow process when the workflow is executed after send Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 10 +++++++--- src/Manager/RuleManager.php | 18 ++++++++++++++++-- src/Solutions/salesforce.php | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 25ffa4717..f7c795fcb 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -2130,7 +2130,10 @@ public function updateStatus($new_status) $this->status = $new_status; $this->afterStatusChange($new_status); $this->createDocLog(); - $this->runWorkflow(); + // runWorkflow can't be executed if updateStatus is called from the solution class + if ($new_status!='Send') { + $this->runWorkflow(); + } } catch (\Exception $e) { $this->message .= 'Error status update : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; @@ -2462,7 +2465,8 @@ protected function getTargetId($ruleRelationship, $record_id) !empty($result['document_id']) AND strpos($result['document_id'], ',') ) { - $result['document_id'] = end(explode(',',$result['document_id'])); + $documentList = explode(',',$result['document_id']); + $result['document_id'] = end($documentList); } } if (!empty($result['record_id'])) { @@ -2526,7 +2530,7 @@ public function getStatus() return $this->status; } - protected function runWorkflow() { + public function runWorkflow() { try { // Check if at least on workflow exist for the rule if (!empty($this->ruleWorkflows)) { diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index bbd96068b..3f71ea585 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -1551,7 +1551,22 @@ protected function sendTarget($type, $documentId = null): array throw new \Exception('Failed to connect to the target application.'); } } - + // Run workflow after send + if ( + !empty($response) + AND empty($response['error']) + AND !empty($this->ruleWorkflows) + ) { + foreach($response as $docId => $value) { + $param['id_doc_myddleware'] = $docId; + $param['jobId'] = $this->jobId; + $param['api'] = $this->api; + $param['ruleWorkflows'] = $this->ruleWorkflows; + // Set the param values and clear all document attributes + $this->documentManager->setParam($param, true); + $this->documentManager->runWorkflow(); + } + } } catch (\Exception $e) { $response['error'] = 'Error : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; if (!$this->api) { @@ -1559,7 +1574,6 @@ protected function sendTarget($type, $documentId = null): array } $this->logger->error($response['error']); } - return $response; } diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index d583c4063..1217699fc 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -772,7 +772,7 @@ protected function getWhere($param): string if (!empty($param['query'])) { $queryWhere = "+WHERE+"; foreach ($param['query'] as $key => $value) { - $queryWhere .= $key."+=+'".$value."'"; + $queryWhere .= $key."+=+'".str_replace(' ', '+', $value)."'"; // Add the AND if not the last entry of the array if ($key !== array_key_last($param['query'])) { $queryWhere .= "+AND+"; From 5f57060c4f3e7eeca4477b696549985a688e5528 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 00:48:56 +0200 Subject: [PATCH 28/88] Add parameterBagInterface Signed-off-by: myddleware --- src/Solutions/solution.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Solutions/solution.php b/src/Solutions/solution.php index e065906eb..6068350ce 100644 --- a/src/Solutions/solution.php +++ b/src/Solutions/solution.php @@ -153,7 +153,7 @@ protected function updateDocumentStatus($idDoc, $value, $param, $forceStatus = n try { $param['id_doc_myddleware'] = $idDoc; $param['api'] = $this->api; - $documentManager = new DocumentManager($this->logger, $this->connection, $this->entityManager, $this->documentRepository, $this->ruleRelationshipsRepository, $this->formulaManager); + $documentManager = new DocumentManager($this->logger, $this->connection, $this->entityManager, $this->documentRepository, $this->ruleRelationshipsRepository, $this->formulaManager, null, $this->parameterBagInterface); $documentManager->setParam($param); // If a message exist, we add it to the document logs if (!empty($value['error'])) { From 054f5ff731dd5cbb9f96f0371e606269972d7216 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 00:53:51 +0200 Subject: [PATCH 29/88] Catch all formula error Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index f7c795fcb..d9ad8a1ad 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1598,7 +1598,7 @@ public function getTransformValue($source, $ruleField) // Trigger to redefine formula $f = $this->changeFormula($f); eval($f.';'); // exec - } catch (\ParseError $e) { + } catch (\Throwable $e) { throw new \Exception('FATAL error because of Invalid formula "'.$ruleField['formula'].';" : '.$e->getMessage()); } // Execute eval only if formula is valid From b4f31ad88326493cd283de875d7076e4d8551dfe Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 01:11:47 +0200 Subject: [PATCH 30/88] Add column active and deleted on workflow tables Signed-off-by: myddleware --- src/Entity/Workflow.php | 17 ++++++++++++++++- src/Entity/WorkflowAction.php | 33 ++++++++++++++++++++++++++++++++- src/Manager/RuleManager.php | 4 ++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/Entity/Workflow.php b/src/Entity/Workflow.php index 79a9d4d8c..7f01af2b9 100644 --- a/src/Entity/Workflow.php +++ b/src/Entity/Workflow.php @@ -86,6 +86,11 @@ class Workflow * @ORM\Column(name="condition", type="text", nullable=false) */ private string $condition; + + /** + * @ORM\Column(name="active", type="boolean", options={"default":1}) + */ + private int $active; /** * @ORM\Column(name="deleted", type="boolean", options={"default":0}) @@ -208,7 +213,6 @@ public function setCondition($condition): self public function setDeleted($deleted): self { $this->deleted = $deleted; - return $this; } @@ -217,6 +221,17 @@ public function getDeleted(): int return $this->deleted; } + public function setActive($active): self + { + $this->active = $active; + return $this; + } + + public function getActive(): int + { + return $this->active; + } + /** * @return Collection|WorkflowAction[] */ diff --git a/src/Entity/WorkflowAction.php b/src/Entity/WorkflowAction.php index 12f2595c7..7cbfa5668 100644 --- a/src/Entity/WorkflowAction.php +++ b/src/Entity/WorkflowAction.php @@ -94,7 +94,16 @@ class WorkflowAction */ private int $order; - + /** + * @ORM\Column(name="active", type="boolean", options={"default":1}) + */ + private int $active; + + /** + * @ORM\Column(name="deleted", type="boolean", options={"default":0}) + */ + private int $deleted; + public function getId(): string { return $this->id; @@ -212,4 +221,26 @@ public function getOrder(): string { return $this->order; } + + public function setActive($active): self + { + $this->active = $active; + return $this; + } + + public function getActive(): int + { + return $this->active; + } + + public function setDeleted($deleted): self + { + $this->deleted = $deleted; + return $this; + } + + public function getDeleted(): int + { + return $this->deleted; + } } diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 3f71ea585..13809b2ba 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -2148,14 +2148,14 @@ protected function setRuleRelationships() protected function setRuleWorkflows() { try { - $sqlWorkflows = 'SELECT * FROM workflow WHERE rule_id = :ruleId'; + $sqlWorkflows = 'SELECT * FROM workflow WHERE rule_id = :ruleId AND deleted = 0 AND active = 1'; $stmt = $this->connection->prepare($sqlWorkflows); $stmt->bindValue(':ruleId', $this->ruleId); $result = $stmt->executeQuery(); $this->ruleWorkflows = $result->fetchAllAssociative(); if (!empty($this->ruleWorkflows)) { foreach($this->ruleWorkflows as $key => $workflow) { - $sqlActions = 'SELECT * FROM workflowaction WHERE workflow_id = :workflowid ORDER BY `order` ASC'; + $sqlActions = 'SELECT * FROM workflowaction WHERE workflow_id = :workflowid AND deleted = 0 ORDER BY `order` ASC'; $stmt = $this->connection->prepare($sqlActions); $stmt->bindValue(':workflowid', $workflow['id']); $result = $stmt->executeQuery(); From 76ef5e1341f610f8f8846554e33aae74e35af215 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 01:12:25 +0200 Subject: [PATCH 31/88] Add active condition for workflow action Signed-off-by: myddleware --- src/Manager/RuleManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 13809b2ba..3b22e52cf 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -2155,7 +2155,7 @@ protected function setRuleWorkflows() $this->ruleWorkflows = $result->fetchAllAssociative(); if (!empty($this->ruleWorkflows)) { foreach($this->ruleWorkflows as $key => $workflow) { - $sqlActions = 'SELECT * FROM workflowaction WHERE workflow_id = :workflowid AND deleted = 0 ORDER BY `order` ASC'; + $sqlActions = 'SELECT * FROM workflowaction WHERE workflow_id = :workflowid AND deleted = 0 AND active = 1 ORDER BY `order` ASC'; $stmt = $this->connection->prepare($sqlActions); $stmt->bindValue(':workflowid', $workflow['id']); $result = $stmt->executeQuery(); From 31e04cf53c1e44864b75c2f7f12b2f8518590d2a Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 23:34:15 +0200 Subject: [PATCH 32/88] Add lookup formula to manage relationship Signed-off-by: myddleware --- src/Controller/DefaultController.php | 4 +- src/Manager/DocumentManager.php | 35 ++++--- src/Manager/FormulaFunctionManager.php | 124 ++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 18 deletions(-) diff --git a/src/Controller/DefaultController.php b/src/Controller/DefaultController.php index c5261966f..1a1bd2292 100644 --- a/src/Controller/DefaultController.php +++ b/src/Controller/DefaultController.php @@ -1408,8 +1408,10 @@ public function ruleSimulation(Request $request): Response 'related_rule' => '', ]; + // Add rule id for simulation purpose when using lookup function + $this->documentManager->setRuleId($ruleKey); // Transformation - $response = $this->documentManager->getTransformValue($record, $target_fields); + $response = $this->documentManager->getTransformValue($record, $target_fields); if (!isset($response['message'])) { $r['after'][$name_fields_target] = $this->documentManager->getTransformValue($record, $target_fields); } diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index d9ad8a1ad..06d5552d3 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1589,25 +1589,30 @@ public function getTransformValue($source, $ruleField) $this->formulaManager->init($ruleField['formula']); // mise en place de la règle dans la classe $this->formulaManager->generateFormule(); // Genère la nouvelle formule à la forme PhP - // Exécute la règle si pas d'erreur de syntaxe - if ( - $f = $this->formulaManager->execFormule() - ) { - // Try the formula first + // Execute formula + if ($f = $this->formulaManager->execFormule()) { + // Manage lookup formula by adding the current rule as the first parameter + if (strpos($f, 'lookup') !== false ) { + $currentRule = $this->ruleId; + $connection = $this->connection; + $entityManager = $this->entityManager; + $myddlewareUserId = $this->userId; + $sourceFieldName = $ruleField['source_field_name']; + $docId = $this->id; + $f = str_replace('lookup(', 'lookup($entityManager, $connection, $currentRule, $docId, $myddlewareUserId, $sourceFieldName, ', $f); + } try { // Trigger to redefine formula $f = $this->changeFormula($f); - eval($f.';'); // exec + eval('$rFormula = '.$f.';'); // exec + if (isset($rFormula)) { + // Return result + return $rFormula; + } else { + throw new \Exception('Invalid formula (failed to retrieve formula) : '.$ruleField['formula']); + } } catch (\Throwable $e) { - throw new \Exception('FATAL error because of Invalid formula "'.$ruleField['formula'].';" : '.$e->getMessage()); - } - // Execute eval only if formula is valid - eval('$rFormula = '.$f.';'); // exec - if (isset($rFormula)) { - // affectation du résultat - return $rFormula; - } else { - throw new \Exception('Invalid formula (failed to retrieve formula) : '.$ruleField['formula']); + throw new \Exception('Failed to execute the formula "'.$ruleField['formula'].';" : '.$e->getMessage()); } } else { throw new \Exception('Invalid formula (failed to execute) : '.$ruleField['formula']); diff --git a/src/Manager/FormulaFunctionManager.php b/src/Manager/FormulaFunctionManager.php index 2cf5d599f..6f0df2275 100644 --- a/src/Manager/FormulaFunctionManager.php +++ b/src/Manager/FormulaFunctionManager.php @@ -25,11 +25,13 @@ namespace App\Manager; +use App\Entity\DocumentRelationship as DocumentRelationship; + class formulafunctioncore { - protected array $names = ['changeTimeZone', 'changeFormatDate', 'changeValue', 'changeMultiValue', 'getValueFromArray']; + protected array $names = ['changeTimeZone', 'changeFormatDate', 'changeValue', 'changeMultiValue', 'getValueFromArray','lookup']; protected string $path = "App\Manager\FormulaFunctionManager::"; - + public function getNamesFunctions(): array { return $this->names; @@ -119,6 +121,124 @@ public static function getValueFromArray($key, $array) return $array[$key]; } } + + public static function lookup($entityManager, $connection, $currentRule, $docId, $myddlewareUserId, $sourceFieldName, $fieldValue, $rule, $errorIfEmpty=false, $errodIfNoFound=true, $parent=false) + { + // Manage error if empty + if (empty($fieldValue)) { + if ($errorIfEmpty) { + throw new \Exception('The field '.$sourceFieldName.' is empty. Failed to find the relate value. '); + } else { + return ''; + } + } + // In case of simulation during rule creation (not edition), we don't have the current rule id. + // We set direction = 1 by default + if ($currentRule === 0) { + $direction = 1; + } else { + // Get rules detail + $ruleQuery = "SELECT * FROM rule WHERE id = :ruleId"; + $stmt = $connection->prepare($ruleQuery); + $stmt->bindValue(':ruleId', $currentRule); + $result = $stmt->executeQuery(); + $ruleRef = $result->fetchAssociative(); + $stmt->bindValue(':ruleId', $rule); + $result = $stmt->executeQuery(); + $ruleLink = $result->fetchAssociative(); + } + + // Query to search the relate record id is different depending on the direction of the relationship + if ( + ( + !empty($ruleRef) + AND $ruleRef['conn_id_source'] == $ruleLink['conn_id_source'] + AND $ruleRef['conn_id_target'] == $ruleLink['conn_id_target'] + ) + OR (!empty($direction)) // Manage simulation + ){ + $sqlParams = " SELECT + target_id record_id, + GROUP_CONCAT(DISTINCT document.id) document_id, + GROUP_CONCAT(DISTINCT document.type) types + FROM document + WHERE + document.rule_id = :ruleRelateId + AND document.source_id = :record_id + AND document.deleted = 0 + AND document.target_id != '' + AND ( + document.global_status = 'Close' + OR document.status = 'No_send' + ) + GROUP BY target_id + HAVING types NOT LIKE '%D%' + LIMIT 1"; + $direction = 1; + } elseif ( + $ruleRef['conn_id_source'] == $ruleLink['conn_id_target'] + AND $ruleRef['conn_id_target'] == $ruleLink['conn_id_source'] + ){ + $sqlParams = " SELECT + source_id record_id, + GROUP_CONCAT(DISTINCT document.id) document_id, + GROUP_CONCAT(DISTINCT document.type) types + FROM document + WHERE + document.rule_id = :ruleRelateId + AND document.source_id != '' + AND document.deleted = 0 + AND document.target_id = :record_id + AND ( + document.global_status = 'Close' + OR document.status = 'No_send' + ) + GROUP BY source_id + HAVING types NOT LIKE '%D%' + LIMIT 1"; + $direction = -1; + } else { + throw new \Exception('The connectors do not match between rule '.$currentRule.' and rule '.$rule.'. '); + } + // Get the record id + $stmt = $connection->prepare($sqlParams); + $stmt->bindValue(':ruleRelateId', $rule); + $stmt->bindValue(':record_id', $fieldValue); + $result = $stmt->executeQuery(); + $result = $result->fetchAssociative(); + // Manage error if no result found + if (empty($result['record_id'])) { + if ($errodIfNoFound) { + throw new \Exception('Failed to retrieve a related document. No data for the field '.$sourceFieldName.'. There is not record with the ID '.('1' == $direction ? 'source' : 'target').' '.$fieldValue.' in the rule '.$ruleLink['name'].'. This document is queued. '); + } else { + return ''; + } + } + // In cas of several document found we get only the last one + if ( + !empty($result['document_id']) + AND strpos($result['document_id'], ',') + ) { + $documentList = explode(',',$result['document_id']); + $result['document_id'] = end($documentList); + } + // No doc id in case of simulation + if (!empty($docId)) { + // Add the relationship in the table document Relationship + try { + $documentRelationship = new DocumentRelationship(); + $documentRelationship->setDocId($docId); + $documentRelationship->setDocRelId($result['document_id']); + $documentRelationship->setDateCreated(new \DateTime()); + $documentRelationship->setCreatedBy((int) $myddlewareUserId); + $documentRelationship->setSourceField($sourceFieldName); + $entityManager->persist($documentRelationship); + } catch (\Exception $e) { + throw new \Exception('Failed to save the document relationship for the field '.$sourceFieldName.' : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + } + } + return $result['record_id']; + } } class FormulaFunctionManager extends formulafunctioncore From b3961fef72532f70906d8dc47811ff932be4c68e Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 17 Apr 2024 23:51:23 +0200 Subject: [PATCH 33/88] Add translation Signed-off-by: myddleware --- src/DataFixtures/LoadFunctionData.php | 2 +- src/Manager/FormulaFunctionManager.php | 6 +++--- translations/messages.en.yml | 1 + translations/messages.fr.yml | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/DataFixtures/LoadFunctionData.php b/src/DataFixtures/LoadFunctionData.php index e3a46f2b4..2e230d6e5 100644 --- a/src/DataFixtures/LoadFunctionData.php +++ b/src/DataFixtures/LoadFunctionData.php @@ -35,7 +35,7 @@ class LoadFunctionData implements FixtureInterface private $manager; protected $functionData = [ 'mathematical' => ['round', 'ceil', 'abs'], - 'text' => ['trim', 'ltrim', 'rtrim', 'lower', 'upper', 'substr', 'striptags', 'changeValue', 'htmlEntityDecode', 'replace', 'utf8encode', 'utf8decode', 'htmlentities', 'htmlspecialchars', 'strlen', 'urlencode', 'chr', 'json_decode', 'json_encode', 'getValueFromArray'], + 'text' => ['trim', 'ltrim', 'rtrim', 'lower', 'upper', 'substr', 'striptags', 'changeValue', 'htmlEntityDecode', 'replace', 'utf8encode', 'utf8decode', 'htmlentities', 'htmlspecialchars', 'strlen', 'urlencode', 'chr', 'json_decode', 'json_encode', 'getValueFromArray','lookup'], 'date' => ['date', 'microtime', 'changeTimeZone', 'changeFormatDate'], 'constant' => ['mdw_no_send_field'], ]; diff --git a/src/Manager/FormulaFunctionManager.php b/src/Manager/FormulaFunctionManager.php index 6f0df2275..24293615d 100644 --- a/src/Manager/FormulaFunctionManager.php +++ b/src/Manager/FormulaFunctionManager.php @@ -122,10 +122,10 @@ public static function getValueFromArray($key, $array) } } - public static function lookup($entityManager, $connection, $currentRule, $docId, $myddlewareUserId, $sourceFieldName, $fieldValue, $rule, $errorIfEmpty=false, $errodIfNoFound=true, $parent=false) + public static function lookup($entityManager, $connection, $currentRule, $docId, $myddlewareUserId, $sourceFieldName, $field, $rule, $errorIfEmpty=false, $errodIfNoFound=true, $parent=false) { // Manage error if empty - if (empty($fieldValue)) { + if (empty($field)) { if ($errorIfEmpty) { throw new \Exception('The field '.$sourceFieldName.' is empty. Failed to find the relate value. '); } else { @@ -203,7 +203,7 @@ public static function lookup($entityManager, $connection, $currentRule, $docId, // Get the record id $stmt = $connection->prepare($sqlParams); $stmt->bindValue(':ruleRelateId', $rule); - $stmt->bindValue(':record_id', $fieldValue); + $stmt->bindValue(':record_id', $field); $result = $stmt->executeQuery(); $result = $result->fetchAssociative(); // Manage error if no result found diff --git a/translations/messages.en.yml b/translations/messages.en.yml index 4bcce1d04..58ad6d913 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -491,6 +491,7 @@ function: htmlentities: htmlentities ( string $string [, int $flags = ENT_COMPAT | ENT_HTML401 [, string $encoding = ini_get("default_charset") [, bool $double_encode = true ]]] ) / Convert all applicable characters to HTML entities htmlspecialchars: htmlspecialchars ( string $string [, int $flags = ENT_COMPAT | ENT_HTML401 [, string $encoding = ini_get("default_charset") [, bool $double_encode = true ]]] ) / Convert special characters to HTML entities chr: chr ( int $bytevalue ) Generate a single-byte string from a number + lookup: lookup ( string $field, string $rule, errorIfEmpty (default = false), errodIfNoFound (default = true)). Search the corresponding value of the $field in the rule $rule. mathematical: round: Rounding a floating point number ceil: Rounding up diff --git a/translations/messages.fr.yml b/translations/messages.fr.yml index 914c8cebb..6bd61b2b6 100644 --- a/translations/messages.fr.yml +++ b/translations/messages.fr.yml @@ -489,6 +489,7 @@ function: htmlentities: htmlentities ( string $string [, int $flags = ENT_COMPAT | ENT_HTML401 [, string $encoding = ini_get("default_charset") [, bool $double_encode = true ]]] ) / Convertit tous les caractères éligibles en entités HTML htmlspecialchars: htmlspecialchars ( string $string [, int $flags = ENT_COMPAT | ENT_HTML401 [, string $encoding = ini_get("default_charset") [, bool $double_encode = true ]]] ) / Convertit les caractères spéciaux en entités HTML chr: chr ( int $bytevalue ) Générer une chaîne d'un octet à partir d'un nombre + lookup: lookup ( string $field, string $rule, errorIfEmpty (default = false), errodIfNoFound (default = true)). Cherche la correspondance du champ $field dans la règle $rule. mathematical: round: Arrondit un nombre à virgule flottante ceil: Arrondit au nombre supérieur From 592bc2349690d8ed0a5dfbd7321c844cc060970a Mon Sep 17 00:00:00 2001 From: myddleware Date: Thu, 18 Apr 2024 16:42:13 +0200 Subject: [PATCH 34/88] PostgreSQL : add views in the module list Signed-off-by: myddleware --- src/Solutions/postgresql.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Solutions/postgresql.php b/src/Solutions/postgresql.php index 29256c9b7..37b49cbac 100644 --- a/src/Solutions/postgresql.php +++ b/src/Solutions/postgresql.php @@ -43,10 +43,17 @@ protected function generatePdo(): \PDO // Generate query protected function get_query_show_tables(): string { - return "SELECT * + // Read tables and views + return "SELECT schemaname, tablename FROM pg_catalog.pg_tables WHERE schemaname != 'pg_catalog' - AND schemaname != 'information_schema'"; + AND schemaname != 'information_schema' + UNION + SELECT schemaname, viewname tablename + FROM pg_catalog.pg_views + WHERE schemaname != 'pg_catalog' + AND schemaname != 'information_schema' + "; } // Get all tables from the database From 29a0d5cc48f7c5a2a9c48451f9c38ac17863bf67 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 24 Apr 2024 22:43:23 +0200 Subject: [PATCH 35/88] Add workflow function : changeStatus Add workflow variables Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 30 +++++++++++++++++++++++++----- src/Manager/RuleManager.php | 17 ++++++++++------- src/Utils/workflowVariables.php | 5 ++++- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 06d5552d3..701f7acee 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -2134,11 +2134,14 @@ public function updateStatus($new_status) $this->message .= 'Status : '.$new_status; $this->status = $new_status; $this->afterStatusChange($new_status); - $this->createDocLog(); + // We don't clear the message because we could need it in the workflow, we clear it after the workflow execution + $this->createDocLog(false); // runWorkflow can't be executed if updateStatus is called from the solution class if ($new_status!='Send') { $this->runWorkflow(); } + $this->message = ''; + $this->docIdRefError = ''; } catch (\Exception $e) { $this->message .= 'Error status update : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->typeError = 'E'; @@ -2595,7 +2598,22 @@ public function runWorkflow() { $this->tools->sendMessage($arguments['to'],$arguments['subject'],$arguments['message']); } catch (\Exception $e) { $workflowStatus = 'Error'; - $error = 'Failed to create workflow log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $error = 'Failed to send notification : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $this->logger->error($error); + $this->generateDocLog('E',$error); + } + $this->createWorkflowLog($action, $workflowStatus, $error); + break; + case 'updateStatus': + try { + $workflowStatus = 'Success'; + $error = ''; + $this->typeError = 'W'; + $this->message = 'Status change using workflow. '; + $this->updateStatus($arguments['status']); + } catch (\Exception $e) { + $workflowStatus = 'Error'; + $error = 'Failed change status : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->logger->error($error); $this->generateDocLog('E',$error); } @@ -2712,7 +2730,7 @@ protected function createWorkflowLog($action, $status, $error=null, $generateDoc * $message contient le message de l'erreur avec potentiellement des variable &1, &2... * $data contient les varables du message de type array('id_contact', 'nom_contact') */ - protected function createDocLog() + protected function createDocLog($clearMessage=true) { try { $now = gmdate('Y-m-d H:i:s'); @@ -2726,8 +2744,10 @@ protected function createDocLog() $stmt->bindValue(':ref_doc_id', $this->docIdRefError); $stmt->bindValue(':job_id', $this->jobId); $result = $stmt->executeQuery(); - $this->message = ''; - $this->docIdRefError = ''; + if ($clearMessage) { + $this->message = ''; + $this->docIdRefError = ''; + } } catch (\Exception $e) { $this->logger->error('Failed to create log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } diff --git a/src/Manager/RuleManager.php b/src/Manager/RuleManager.php index 3b22e52cf..18f3ed5e8 100644 --- a/src/Manager/RuleManager.php +++ b/src/Manager/RuleManager.php @@ -1551,6 +1551,7 @@ protected function sendTarget($type, $documentId = null): array throw new \Exception('Failed to connect to the target application.'); } } + // Run workflow after send if ( !empty($response) @@ -1558,13 +1559,15 @@ protected function sendTarget($type, $documentId = null): array AND !empty($this->ruleWorkflows) ) { foreach($response as $docId => $value) { - $param['id_doc_myddleware'] = $docId; - $param['jobId'] = $this->jobId; - $param['api'] = $this->api; - $param['ruleWorkflows'] = $this->ruleWorkflows; - // Set the param values and clear all document attributes - $this->documentManager->setParam($param, true); - $this->documentManager->runWorkflow(); + if (!empty($value)) { + $param['id_doc_myddleware'] = $docId; + $param['jobId'] = $this->jobId; + $param['api'] = $this->api; + $param['ruleWorkflows'] = $this->ruleWorkflows; + // Set the param values and clear all document attributes + $this->documentManager->setParam($param, true); + $this->documentManager->runWorkflow(); + } } } } catch (\Exception $e) { diff --git a/src/Utils/workflowVariables.php b/src/Utils/workflowVariables.php index 557a8182c..bcef0e6c8 100644 --- a/src/Utils/workflowVariables.php +++ b/src/Utils/workflowVariables.php @@ -1,3 +1,6 @@ status; - +$documentType = $this->documentType; +$attempt = $this->attempt; +$message = $this->message; +$typeError = $this->typeError; From ac6080c9eb8d420729f5eda49f5f6fe888c814c2 Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 24 Apr 2024 22:48:40 +0200 Subject: [PATCH 36/88] Add parameter for lookup function : createDocRelationship=true) Signed-off-by: myddleware --- src/Manager/FormulaFunctionManager.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Manager/FormulaFunctionManager.php b/src/Manager/FormulaFunctionManager.php index 24293615d..e2420c54f 100644 --- a/src/Manager/FormulaFunctionManager.php +++ b/src/Manager/FormulaFunctionManager.php @@ -122,7 +122,7 @@ public static function getValueFromArray($key, $array) } } - public static function lookup($entityManager, $connection, $currentRule, $docId, $myddlewareUserId, $sourceFieldName, $field, $rule, $errorIfEmpty=false, $errodIfNoFound=true, $parent=false) + public static function lookup($entityManager, $connection, $currentRule, $docId, $myddlewareUserId, $sourceFieldName, $field, $rule, $errorIfEmpty=false, $errodIfNoFound=true, $createDocRelationship=true) { // Manage error if empty if (empty($field)) { @@ -223,7 +223,10 @@ public static function lookup($entityManager, $connection, $currentRule, $docId, $result['document_id'] = end($documentList); } // No doc id in case of simulation - if (!empty($docId)) { + if ( + !empty($docId) + AND $createDocRelationship + ) { // Add the relationship in the table document Relationship try { $documentRelationship = new DocumentRelationship(); From 3228ffe4e2bcdd085d31e19c7dbd77c96317d48f Mon Sep 17 00:00:00 2001 From: myddleware Date: Wed, 24 Apr 2024 23:35:59 +0200 Subject: [PATCH 37/88] Formula : Manage dot into a variable Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 5 +++++ src/Manager/FormulaFunctionManager.php | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 701f7acee..7dbb10543 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1582,6 +1582,11 @@ public function getTransformValue($source, $ruleField) // We skip my_value because it is a constante if ('my_value' != $ruleField['source_field_name']) { $fieldNameDyn = $ruleField['source_field_name']; // value : variable name + // Replace dot by a string because dot can't be into a variable in php, the formula can't work + if (strpos($ruleField['source_field_name'], '.') !== false) { + $fieldNameDyn = str_replace('.', '___dot___', $ruleField['source_field_name']); + $ruleField['formula'] = str_replace($ruleField['source_field_name'], $fieldNameDyn, $ruleField['formula']); + } $$fieldNameDyn = $source[$ruleField['source_field_name']]; // Dynamic variable (e.g $name = name) } } diff --git a/src/Manager/FormulaFunctionManager.php b/src/Manager/FormulaFunctionManager.php index e2420c54f..0ee262d2b 100644 --- a/src/Manager/FormulaFunctionManager.php +++ b/src/Manager/FormulaFunctionManager.php @@ -209,7 +209,7 @@ public static function lookup($entityManager, $connection, $currentRule, $docId, // Manage error if no result found if (empty($result['record_id'])) { if ($errodIfNoFound) { - throw new \Exception('Failed to retrieve a related document. No data for the field '.$sourceFieldName.'. There is not record with the ID '.('1' == $direction ? 'source' : 'target').' '.$fieldValue.' in the rule '.$ruleLink['name'].'. This document is queued. '); + throw new \Exception('Failed to retrieve a related document. No data for the field '.$sourceFieldName.'. There is not record with the ID '.('1' == $direction ? 'source' : 'target').' '.$field.' in the rule '.$ruleLink['name'].'. This document is queued. '); } else { return ''; } From a73552c8bb99996bd3b42bf6940a80c2a74271c7 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 26 Apr 2024 10:24:27 +0200 Subject: [PATCH 38/88] Workflow add source data in variable Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 143 +++++++++++++++++--------------- translations/messages.en.yml | 1 + translations/messages.fr.yml | 1 + 3 files changed, 80 insertions(+), 65 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 7dbb10543..70d5b1f4c 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -107,6 +107,7 @@ class documentcore 'Error_checking' => 'Error', 'Error_sending' => 'Error', 'Not_found' => 'Error', + 'Error_workflow' => 'Error', ]; private array $notSentFields = []; @@ -164,6 +165,7 @@ public static function lstStatus(): array 'Error_transformed' => 'flux.status.error_transformed', 'Error_checking' => 'flux.status.error_checking', 'Error_sending' => 'flux.status.error_sending', + 'Error_workflow' => 'flux.status.Error_workflow', ]; } @@ -2547,7 +2549,12 @@ public function runWorkflow() { try { // Check if at least on workflow exist for the rule if (!empty($this->ruleWorkflows)) { - // includ variables used in the formula + // Add all source data in variables + foreach($this->sourceData as $key => $value) { + $fieldName = 'source_'.$key; + $$fieldName = $value; + } + // include variables used in the formula include __DIR__.'/../Utils/workflowVariables.php'; if (file_exists( __DIR__.'/../Custom/Utils/workflowVariables.php')) { include __DIR__.'/../Custom/Utils/workflowVariables.php'; @@ -2561,80 +2568,86 @@ public function runWorkflow() { eval('$condition = '.$f.';'); // exec // Execute the action if the condition is met if ($condition == 1) { - // Execute all actions - if (!empty($ruleWorkflow['actions'])) { - // Call each actions - foreach($ruleWorkflow['actions'] as $action) { - // Check if the action has already been executed for the current document - // Only if attempt > 0, if it is the first attempt then the action has never been executed - if ($this->attempt > 0) { - // Search action for the current document - $workflowLogEntity = $this->entityManager->getRepository(WorkflowLog::class) - ->findOneBy([ - 'triggerDocument' => $this->id, - 'action' => $action['id'], - ] - ); - // If the current action has been found for the current document, we don't execute the current action - if ( - !empty($workflowLogEntity) - AND $workflowLogEntity->getStatus() == 'Success' - ) { - // GenerateDocument can be empty depending the action - if (!empty($workflowLogEntity->getGenerateDocument())) { - $this->docIdRefError = $workflowLogEntity->getGenerateDocument()->getId(); + try { + // Execute all actions + if (!empty($ruleWorkflow['actions'])) { + // Call each actions + foreach($ruleWorkflow['actions'] as $action) { + // Check if the action has already been executed for the current document + // Only if attempt > 0, if it is the first attempt then the action has never been executed + if ($this->attempt > 0) { + // Search action for the current document + $workflowLogEntity = $this->entityManager->getRepository(WorkflowLog::class) + ->findOneBy([ + 'triggerDocument' => $this->id, + 'action' => $action['id'], + ] + ); + // If the current action has been found for the current document, we don't execute the current action + if ( + !empty($workflowLogEntity) + AND $workflowLogEntity->getStatus() == 'Success' + ) { + // GenerateDocument can be empty depending the action + if (!empty($workflowLogEntity->getGenerateDocument())) { + $this->docIdRefError = $workflowLogEntity->getGenerateDocument()->getId(); + } + $this->generateDocLog('W','Action ' . $action['id'] . ' already executed for this document. '); + continue; } - $this->generateDocLog('W','Action ' . $action['id'] . ' already executed for this document. '); - continue; } - } - // Execute action depending of the function in the workflow - $arguments = unserialize($action['arguments']); - switch ($action['action']) { - case 'generateDocument': - $this->generateDocument($arguments['ruleId'],$this->sourceData[$arguments['searchValue']],$arguments['searchField'],$arguments['rerun'], $action); - break; - case 'sendNotification': - try { - $workflowStatus = 'Success'; - $error = ''; - // Method sendMessage throws an exception if it fails - $this->tools->sendMessage($arguments['to'],$arguments['subject'],$arguments['message']); - } catch (\Exception $e) { - $workflowStatus = 'Error'; - $error = 'Failed to send notification : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; - $this->logger->error($error); - $this->generateDocLog('E',$error); - } - $this->createWorkflowLog($action, $workflowStatus, $error); - break; - case 'updateStatus': - try { - $workflowStatus = 'Success'; - $error = ''; - $this->typeError = 'W'; - $this->message = 'Status change using workflow. '; - $this->updateStatus($arguments['status']); - } catch (\Exception $e) { - $workflowStatus = 'Error'; - $error = 'Failed change status : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; - $this->logger->error($error); - $this->generateDocLog('E',$error); - } - $this->createWorkflowLog($action, $workflowStatus, $error); - break; - default: - throw new \Exception('Function '.key($action).' unknown.'); + // Execute action depending of the function in the workflow + $arguments = unserialize($action['arguments']); + switch ($action['action']) { + case 'generateDocument': + $this->generateDocument($arguments['ruleId'],$this->sourceData[$arguments['searchValue']],$arguments['searchField'],$arguments['rerun'], $action); + break; + case 'sendNotification': + try { + $workflowStatus = 'Success'; + $error = ''; + // Method sendMessage throws an exception if it fails + $this->tools->sendMessage($arguments['to'],$arguments['subject'],$arguments['message']); + } catch (\Exception $e) { + $workflowStatus = 'Error'; + $error = 'Failed to send notification : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $this->logger->error($error); + $this->generateDocLog('E',$error); + } + $this->createWorkflowLog($action, $workflowStatus, $error); + break; + case 'updateStatus': + try { + $workflowStatus = 'Success'; + $error = ''; + $this->typeError = 'W'; + $this->message = 'Status change using workflow. '; + $this->updateStatus($arguments['status']); + } catch (\Exception $e) { + $workflowStatus = 'Error'; + $error = 'Failed change status : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; + $this->logger->error($error); + $this->generateDocLog('E',$error); + } + $this->createWorkflowLog($action, $workflowStatus, $error); + break; + default: + throw new \Exception('Function '.key($action).' unknown.'); + } } } + } catch (\Exception $e) { + $this->logger->error('Failed to run the workflow '.$ruleWorkflow['name'].' : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $this->generateDocLog('E','Failed to run the workflow '.$ruleWorkflow['name'].' : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $this->updateStatus('Error_workflow'); } } } } } catch (\Exception $e) { - $this->logger->error('Failed to create workflow log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); - $this->generateDocLog('E','Failed to create workflow log : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $this->logger->error('Failed to run all workflows : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); + $this->generateDocLog('E','Failed to run all workflows : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'); } } diff --git a/translations/messages.en.yml b/translations/messages.en.yml index 58ad6d913..b95a27027 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -377,6 +377,7 @@ flux: error_checking: Checking error error_sending: Sending error not_found: Not found + Error_workflow: Workflow error type: update: Update create: Create diff --git a/translations/messages.fr.yml b/translations/messages.fr.yml index 6bd61b2b6..131252fe8 100644 --- a/translations/messages.fr.yml +++ b/translations/messages.fr.yml @@ -375,6 +375,7 @@ flux: error_checking: Vérification en erreur error_sending: Envoi en erreur Not_found: Non trouvée + Error_workflow: Workflow en erreur type: update: Update create: Create From c8e16c119d5121ff429f6076a7848710aed483d1 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 26 Apr 2024 11:32:34 +0200 Subject: [PATCH 39/88] Lookup : manage rule order with this new function Signed-off-by: myddleware --- src/Manager/JobManager.php | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Manager/JobManager.php b/src/Manager/JobManager.php index 0598812ab..1b57e3cea 100644 --- a/src/Manager/JobManager.php +++ b/src/Manager/JobManager.php @@ -865,8 +865,11 @@ public function orderRules(): bool // Si la règle n'a pas de relation on initialise l'ordre à 1 sinon on met 99 $sql = "SELECT rule.id, - GROUP_CONCAT(rulerelationship.field_id SEPARATOR ';') field_id + GROUP_CONCAT(rulerelationship.field_id SEPARATOR ';') field_id, + GROUP_CONCAT(rulefield.formula SEPARATOR ';') formula FROM rule + LEFT OUTER JOIN rulefield + ON rule.id = rulefield.rule_id LEFT OUTER JOIN rulerelationship ON rule.id = rulerelationship.rule_id WHERE @@ -877,6 +880,15 @@ public function orderRules(): bool $rules = $result->fetchAllAssociative(); if (!empty($rules)) { + // Add the rule in formula to the list of rules in relationship + foreach ($rules as $key => $rule) { + $matches = array(); + // Get the second parameters (rule id) of the lookup functions + preg_match_all('/lookup\(\{[^}]+\},"([^"]+)"/', $rule['formula'], $matches); + // Transform result and add it to the other rules (old relationships) + $rules[$key]['field_id'] .= ';'.implode(';', array_unique($matches[1])); + $rules[$key]['field_id'] = trim($rules[$key]['field_id'],';'); + } // Création d'un tableau en clé valeur et sauvegarde d'un tableau de référence $ruleKeyValue = []; foreach ($rules as $key => $rule) { @@ -922,7 +934,6 @@ public function orderRules(): bool } $rules = $rulesRef; } - // On vide la table RuleOrder $sql = 'DELETE FROM ruleorder'; $stmt = $this->connection->prepare($sql); @@ -943,7 +954,6 @@ public function orderRules(): bool $this->connection->rollBack(); // -- ROLLBACK TRANSACTION $this->message .= 'Failed to update table RuleOrder : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; $this->logger->error($this->message); - return false; } From 1fe1774c3b166ae12c735712ad3b17f4a153a417 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 26 Apr 2024 11:59:58 +0200 Subject: [PATCH 40/88] Add target and history fields Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 70d5b1f4c..6b2e3afe9 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -2549,6 +2549,8 @@ public function runWorkflow() { try { // Check if at least on workflow exist for the rule if (!empty($this->ruleWorkflows)) { + $targetFields = false; + $historyFields = false; // Add all source data in variables foreach($this->sourceData as $key => $value) { $fieldName = 'source_'.$key; @@ -2561,6 +2563,37 @@ public function runWorkflow() { } // Execute every workflow of the rule foreach ($this->ruleWorkflows as $ruleWorkflow) { + // Add target fields if requested and if not already calculated + if ( + strpos($ruleWorkflow['condition'], 'target_') !== false + AND !$targetFields + ) { + $targetFields = true; + $target = $this->getDocumentData('T'); + // Add all source data in variables + if (!empty($target)) { + foreach($target as $key => $value) { + $fieldName = 'target_'.$key; + $$fieldName = $value; + } + } + } + // Add history fields if requested and if not already calculated + if ( + strpos($ruleWorkflow['condition'], 'history_') !== false + AND !$historyFields + ) { + $historyFields = true; + $history = $this->getDocumentData('H'); + // Add all source data in variables + if (!empty($history)) { + foreach($history as $key => $value) { + $fieldName = 'history_'.$key; + $$fieldName = $value; + } + } + } + // Check the condition $this->formulaManager->init($ruleWorkflow['condition']); // mise en place de la règle dans la classe $this->formulaManager->generateFormule(); // Genère la nouvelle formule à la forme PhP From 41dcf647ba5560911adde075f74fdf70d1201632 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 3 May 2024 11:21:54 +0200 Subject: [PATCH 41/88] Workflow : error management Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 6b2e3afe9..d87b9d352 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -2637,32 +2637,18 @@ public function runWorkflow() { $this->generateDocument($arguments['ruleId'],$this->sourceData[$arguments['searchValue']],$arguments['searchField'],$arguments['rerun'], $action); break; case 'sendNotification': - try { - $workflowStatus = 'Success'; - $error = ''; - // Method sendMessage throws an exception if it fails - $this->tools->sendMessage($arguments['to'],$arguments['subject'],$arguments['message']); - } catch (\Exception $e) { - $workflowStatus = 'Error'; - $error = 'Failed to send notification : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; - $this->logger->error($error); - $this->generateDocLog('E',$error); - } + $workflowStatus = 'Success'; + $error = ''; + // Method sendMessage throws an exception if it fails + $this->tools->sendMessage($arguments['to'],$arguments['subject'],$arguments['message']); $this->createWorkflowLog($action, $workflowStatus, $error); break; case 'updateStatus': - try { - $workflowStatus = 'Success'; - $error = ''; - $this->typeError = 'W'; - $this->message = 'Status change using workflow. '; - $this->updateStatus($arguments['status']); - } catch (\Exception $e) { - $workflowStatus = 'Error'; - $error = 'Failed change status : '.$e->getMessage().' '.$e->getFile().' Line : ( '.$e->getLine().' )'; - $this->logger->error($error); - $this->generateDocLog('E',$error); - } + $workflowStatus = 'Success'; + $error = ''; + $this->typeError = 'W'; + $this->message = 'Status change using workflow. '; + $this->updateStatus($arguments['status']); $this->createWorkflowLog($action, $workflowStatus, $error); break; default: From 88c598d73f3ccff6b2adbd55d67a38424242ccf8 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 3 May 2024 15:08:44 +0200 Subject: [PATCH 42/88] Workflow : fix bug in case sourceData is empty Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index d87b9d352..7a87dd51b 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -2551,6 +2551,10 @@ public function runWorkflow() { if (!empty($this->ruleWorkflows)) { $targetFields = false; $historyFields = false; + // Can be empty depending on the context of the workflow call + if (empty($this->sourceData)) { + $this->sourceData = $this->getDocumentData('S'); + } // Add all source data in variables foreach($this->sourceData as $key => $value) { $fieldName = 'source_'.$key; From 93fec8b6b9125c38f930641676fe37c4e91c8b62 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 3 May 2024 15:50:00 +0200 Subject: [PATCH 43/88] Formula : remove strip_tags Signed-off-by: myddleware --- src/Manager/FormulaManager.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Manager/FormulaManager.php b/src/Manager/FormulaManager.php index ed7ddfc7f..a906fddab 100644 --- a/src/Manager/FormulaManager.php +++ b/src/Manager/FormulaManager.php @@ -270,7 +270,6 @@ private function secureFormule() // ----------- secure $string = trim($string); - $string = strip_tags($string); // ----- remove control characters ----- $string = str_replace("\r", '', $string); // --- replace with empty space From b4f7c92db7768fe1dd411fa634d6275a80940a28 Mon Sep 17 00:00:00 2001 From: myddleware Date: Tue, 7 May 2024 09:59:49 +0200 Subject: [PATCH 44/88] Bug fix : secure mdw_cancel_document search Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 7a87dd51b..05f2a5749 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -953,7 +953,7 @@ public function transformDocument(): bool $transformed = $this->updateTargetTable(); if (!empty($transformed)) { // If the value mdw_cancel_document is found in the target data of the document after transformation we cancel the document - if (array_search('mdw_cancel_document',$transformed) !== false) { + if (array_search('mdw_cancel_document',$transformed, true) !== false) { $this->message .= 'The document contains the value mdw_cancel_document. '; $this->typeError = 'W'; $this->updateStatus('Cancel'); From a3fcfb0c38072935ea9ea016bd0942bcc83f0d82 Mon Sep 17 00:00:00 2001 From: myddleware Date: Fri, 10 May 2024 13:25:02 +0200 Subject: [PATCH 45/88] Bugfix : when too much doc related, the id of the document found was truncated Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index 05f2a5749..d3aaff931 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -2395,7 +2395,7 @@ protected function getTargetId($ruleRelationship, $record_id) if ('-1' == $direction) { $sqlParams = " SELECT source_id record_id, - GROUP_CONCAT(DISTINCT document.id) document_id, + GROUP_CONCAT(DISTINCT document.id ORDER BY document.source_date_modified DESC) document_id, GROUP_CONCAT(DISTINCT document.type) types FROM document WHERE @@ -2413,7 +2413,7 @@ protected function getTargetId($ruleRelationship, $record_id) } elseif ('1' == $direction) { $sqlParams = " SELECT target_id record_id, - GROUP_CONCAT(DISTINCT document.id) document_id, + GROUP_CONCAT(DISTINCT document.id ORDER BY document.source_date_modified DESC) document_id, GROUP_CONCAT(DISTINCT document.type) types FROM document WHERE @@ -2475,13 +2475,13 @@ protected function getTargetId($ruleRelationship, $record_id) $result = $stmt->executeQuery(); $result = $result->fetchAssociative(); - // In cas of several document found we get only the last one + // In cas of several document found we get only the first one (which is the most recent one) if ( !empty($result['document_id']) AND strpos($result['document_id'], ',') ) { $documentList = explode(',',$result['document_id']); - $result['document_id'] = end($documentList); + $result['document_id'] = $documentList[0]; } } if (!empty($result['record_id'])) { From baa6c95ae55e528026f3b4cd05e9085713901a69 Mon Sep 17 00:00:00 2001 From: myddleware Date: Tue, 14 May 2024 01:15:28 +0200 Subject: [PATCH 46/88] Salesforce : manage empty relationship Signed-off-by: myddleware --- src/Solutions/salesforce.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index 1217699fc..eafa55a1b 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -394,7 +394,6 @@ public function readData($param): array $query = $baseQuery.$querySelect.$queryFrom.$queryWhere.$queryOrder.$queryLimit.$queryOffset; $query_request_data = $this->call($query, false); $query_request_data = $this->formatResponse($param,$query_request_data); - // Affectation du nombre de résultat à $result['count'] if (isset($query_request_data['totalSize'])){ $currentCount = $query_request_data['totalSize']; @@ -414,7 +413,10 @@ public function readData($param): array // Don't save attributes if($fieldKey != 'attributes'){ // In case there are 2 levels of relationship (think about a recursive function here) - if(substr($fieldKey,-3) == '__r') { + if( + substr($fieldKey,-3) == '__r' + AND !empty($fieldValue) + ) { foreach($fieldValue as $fieldKeyLevel2 => $fieldValueLevel2) { if($fieldKeyLevel2 != 'attributes'){ $row[mb_strtolower($fieldKeyLevel2)] = $fieldValueLevel2; From cfa8fc5e6574cd1730831b505580707cdcb50daa Mon Sep 17 00:00:00 2001 From: myddleware Date: Tue, 14 May 2024 02:18:19 +0200 Subject: [PATCH 47/88] Salesforce : Manage 3rd level of relationship Signed-off-by: myddleware --- src/Solutions/salesforce.php | 45 +++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/Solutions/salesforce.php b/src/Solutions/salesforce.php index eafa55a1b..01750ebe0 100644 --- a/src/Solutions/salesforce.php +++ b/src/Solutions/salesforce.php @@ -413,14 +413,39 @@ public function readData($param): array // Don't save attributes if($fieldKey != 'attributes'){ // In case there are 2 levels of relationship (think about a recursive function here) - if( - substr($fieldKey,-3) == '__r' - AND !empty($fieldValue) - ) { - foreach($fieldValue as $fieldKeyLevel2 => $fieldValueLevel2) { - if($fieldKeyLevel2 != 'attributes'){ - $row[mb_strtolower($fieldKeyLevel2)] = $fieldValueLevel2; - $row[$param['module'].'.'.$key.'.'.$fieldKey.'.'.$fieldKeyLevel2] = $fieldValueLevel2; + if(substr($fieldKey,-3) == '__r') { + if (!empty($fieldValue)) { + foreach($fieldValue as $fieldKeyLevel2 => $fieldValueLevel2) { + if($fieldKeyLevel2 != 'attributes'){ + // In case there are 3 levels of relationship (think about a recursive function here) + if(substr($fieldKeyLevel2,-3) == '__r') { + if(!empty($fieldValueLevel2)) { + foreach($fieldValueLevel2 as $fieldKeyLevel3 => $fieldValueLevel3) { + if($fieldKeyLevel3 != 'attributes'){ + $row[mb_strtolower($fieldKeyLevel3)] = $fieldValueLevel3; + $row[$param['module'].'.'.$key.'.'.$fieldKey.'.'.$fieldKeyLevel2.'.'.$fieldKeyLevel3] = $fieldValueLevel3; + } + } + // If a relationship is empty, we set all field under this relationship to empty + } else { + foreach($param['fields'] as $field) { + if (str_starts_with($field, $param['module'].'.'.$key.'.'.$fieldKey.'.'.$fieldKeyLevel2)) { + $row[$field] = ''; + } + } + } + } + else { + $row[$param['module'].'.'.$key.'.'.$fieldKey.'.'.$fieldKeyLevel2] = $fieldValueLevel2; + } + } + } + // If a relationship is empty, we set all field under this relationship to empty + } else { + foreach($param['fields'] as $field) { + if (str_starts_with($field, $param['module'].'.'.$key.'.'.$fieldKey)) { + $row[$field] = ''; + } } } } @@ -434,6 +459,10 @@ public function readData($param): array elseif(!($key == 'attributes')){ if($key == 'Id') $row[mb_strtolower($key)] = $record[$key]; + // If Id is requested in the field mapping + if (!empty($param['fields']['Id'])) { + $row[$key] = $record[$key]; + } else { if($key == 'CreatedDate') { $record[$key] = $this->dateTimeToMyddleware($record[$key]); From 79d8cea5bfb69268a4a3266773b84f3796e17740 Mon Sep 17 00:00:00 2001 From: myddleware Date: Tue, 14 May 2024 02:20:36 +0200 Subject: [PATCH 48/88] Manage dot in field for formula Signed-off-by: myddleware --- src/Manager/DocumentManager.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Manager/DocumentManager.php b/src/Manager/DocumentManager.php index d3aaff931..692e05596 100644 --- a/src/Manager/DocumentManager.php +++ b/src/Manager/DocumentManager.php @@ -1572,6 +1572,11 @@ public function getTransformValue($source, $ruleField) // We skip my_value because it is a constant if ('my_value' != $listFields) { $fieldNameDyn = $listFields; // value : variable name + // Replace dot by a string because dot can't be into a variable in php, the formula can't work + if (strpos($ruleField['source_field_name'], '.') !== false) { + $fieldNameDyn = str_replace('.', '___dot___', $listFields); + $ruleField['formula'] = str_replace($listFields, $fieldNameDyn, $ruleField['formula']); + } if (array_key_exists($listFields, $source)) { $$fieldNameDyn = (!empty($source[$listFields]) ? $source[$listFields] : ''); // Dynamic variable (e.g $name = name) } else { From 9e4d983ac661ed8748798fb1d4bc17f1fed9bdf8 Mon Sep 17 00:00:00 2001 From: AlexMyddleware <106162060+AlexMyddleware@users.noreply.github.com> Date: Tue, 14 May 2024 07:10:27 +0200 Subject: [PATCH 49/88] feat: https://github.com/Myddleware/myddleware/issues/1113 (#1125) * feat: issue 1097 add an empty search function and change the doc list button address * fix issue 1113 https://github.com/Myddleware/myddleware/issues/1113 * feat: change label to search instead of Save * feat: add php function to remove one filter 1067 https://github.com/Myddleware/myddleware/issues/1067 * feat: function js to send a request to the server, clear the form and the local storage 1067 https://github.com/Myddleware/myddleware/issues/1067 * feat: add the twig variable of the function path for ajax request 1067 https://github.com/Myddleware/myddleware/issues/1067 * style: comment console.logs * issue 1114: remove jobscheduler from main menu https://github.com/Myddleware/myddleware/issues/1114 * 1082 add the result to the function https://github.com/Myddleware/myddleware/issues/1082 * 1082 add the twig results with the dynamic show hide * also get the same for the tasks * translation for the crontab * 1133 use fetch all associative https://github.com/Myddleware/myddleware/issues/1133 --- assets/js/crontab.js | 4 +- assets/js/regle.js | 81 +++++++++++++++---- src/Controller/FilterController.php | 53 ++++++++++++ src/Controller/JobSchedulerController.php | 6 +- src/DataFixtures/LoadCronJobData.php | 9 +++ src/Form/Filter/CombinedFilterType.php | 1 + templates/JobScheduler/crontab_list.html.twig | 43 ++++++++++ templates/Task/list.html.twig | 25 ++++-- templates/base.html.twig | 11 +-- templates/testFilter.html.twig | 3 +- translations/messages.en.yml | 1 + translations/messages.fr.yml | 1 + 12 files changed, 201 insertions(+), 37 deletions(-) diff --git a/assets/js/crontab.js b/assets/js/crontab.js index 4d29dc2a4..d6733f89b 100644 --- a/assets/js/crontab.js +++ b/assets/js/crontab.js @@ -1,7 +1,7 @@ // Creates a function that will be called when a button with the class crontab_class // is clicked. The function will send a request to the server for the function with the following name crontab_show -console.log('crontabbou.js'); +// console.log('crontabbou.js'); // if an element with the class table-head-result is clicked, then call the function sortTable with the id of the element as an argument $(document).on('click', '.table-head-result', function() { @@ -11,7 +11,7 @@ $(document).on('click', '.table-head-result', function() { sortTable(id); }); -console.log('coucou camille'); +// console.log('coucou camille'); function sortTable(n) { console.log('inside function sortTable'); var table, rows, switching, i, x, y, shouldSwitch, dir, switchcount = 0; diff --git a/assets/js/regle.js b/assets/js/regle.js index b87a7df9d..ef6998f83 100755 --- a/assets/js/regle.js +++ b/assets/js/regle.js @@ -306,7 +306,7 @@ $(function () { var open = pairTypes[t]; var close = map[open]; - console.log('Checking pair type:', open+close); + // console.log('Checking pair type:', open+close); for (var i = 0; i < formula.length; i++) { if (formula[i] === open) { @@ -316,18 +316,18 @@ $(function () { 'position': i, 'pairNum': currentPair }); - console.log('Found opening symbol at', i, 'Pair Number:', currentPair); + // console.log('Found opening symbol at', i, 'Pair Number:', currentPair); } else if (formula[i] === close) { var last = stack.pop(); if (!last) { - console.log('Found closing symbol without matching opening symbol at', i); + // console.log('Found closing symbol without matching opening symbol at', i); errorAt = i; currentPair++; break; } else { - console.log('Found matching closing symbol for pair', last.pairNum, 'at', i); + // console.log('Found matching closing symbol for pair', last.pairNum, 'at', i); if (!pairs.includes(last.pairNum)) { pairs.push(last.pairNum); } @@ -337,7 +337,7 @@ $(function () { // If we still have unclosed brackets at the end of parsing, record an error if (stack.length > 0) { - console.log('Found unbalanced pair at the end of the formula'); + // console.log('Found unbalanced pair at the end of the formula'); var lastUnbalanced = stack.pop(); errorAt = lastUnbalanced.position; currentPair = lastUnbalanced.pairNum; @@ -351,11 +351,11 @@ $(function () { pairs.splice(index, 1); } - console.log('Pair Type:', open + close); - console.log('Status:', status); - console.log('Error Position:', errorAt); - console.log('Unbalanced Pair:', unbalancedPair); - console.log('Balanced Pairs:', pairs); + // console.log('Pair Type:', open + close); + // console.log('Status:', status); + // console.log('Error Position:', errorAt); + // console.log('Unbalanced Pair:', unbalancedPair); + // console.log('Balanced Pairs:', pairs); if (!status) { return { @@ -391,7 +391,7 @@ $(function () { }); - console.log('these are the values', values); + // console.log('these are the values', values); var bracketError = false; var emptyBracketError = false; @@ -505,7 +505,7 @@ $(function () { // empty the values array missingFieldList = []; values = []; - console.log('these are the values', values); + // console.log('these are the values', values); $.ajax({ @@ -824,13 +824,13 @@ $(function () { if ($(this).attr('disabled') != 'disabled') { if ($(this).is(":checked")) { if (remove) { - id = $(this).parent().parent().attr('data-id'); + id = $(this).attr('name'); massAddFlux(id, true, massFluxTab); $(this).prop("checked", false); } } else { if (remove == false) { - id = $(this).parent().parent().attr('data-id'); + id = $(this).attr('name'); massAddFlux(id, false, massFluxTab); $(this).prop("checked", true); } @@ -842,7 +842,6 @@ $(function () { }); $('input', '.listepagerflux td').on('change', function () { - // id = $(this).parent().parent().attr('data-id'); if ($(this).is(":checked")) { massAddFlux($(this).attr('name'), false, massFluxTab); } else { @@ -1977,6 +1976,58 @@ function massAddFlux(id, cond, massFluxTab) { $('#cancelreloadflux').find('span').html(massFluxTab.length); } +$(document).ready(function() { + $('.removeFilters').click(function() { + // Get the class list of the clicked element + var classList = $(this).attr('class').split(/\s+/); + // console.log(classList); + + // Find the filter class (it's the last class in the list) + var filterClass = classList[classList.length - 1]; + // console.log(filterClass); + + // Get the stored filters from local storage + var storedFilters = JSON.parse(localStorage.getItem('storedFilters')); + // console.log(storedFilters); + + // Remove the filter from the stored filters + delete storedFilters[filterClass]; + // console.log(storedFilters); + + // Save the updated filters back to local storage + localStorage.setItem('storedFilters', JSON.stringify(storedFilters)); + // console.log(localStorage.getItem('storedFilters')); + + // Make an AJAX request to the server to remove the filter from the session + $.ajax({ + url: path_remove_filter, + method: 'POST', + data: { filterName: 'FluxFilter' + toCamelCase(filterClass) }, + success: function(response) { + if (response.status === 'success') { + // console.log('Filter successfully removed Argonien'); + + // Clear the form field + var formFieldName = 'combined_filter[document][' + filterClass + ']'; + $('input[name="' + formFieldName + '"]').val(''); + // console.log('Filter input cleared'); + } else { + // console.log('Error removing filter: ' + response.message); + } + } + }); + }); +}); + +function toCamelCase(str) { + // Split the string into words + var words = str.split('_'); + + // Convert each word to title case (except for the first one), and join them back together + return words[0] + words.slice(1).map(function(word) { + return word.charAt(0).toUpperCase() + word.slice(1); + }).join(''); +} // Save the modified field data by using an ajax request function saveInputFlux(div, link) { diff --git a/src/Controller/FilterController.php b/src/Controller/FilterController.php index afccfab1c..914d81b48 100644 --- a/src/Controller/FilterController.php +++ b/src/Controller/FilterController.php @@ -50,6 +50,7 @@ use Symfony\Component\HttpFoundation\Response; use Symfony\Component\Routing\Annotation\Route; use Symfony\Component\HttpKernel\KernelInterface; +use Symfony\Component\HttpFoundation\JsonResponse; use Pagerfanta\Exception\NotValidCurrentPageException; use Symfony\Contracts\Translation\TranslatorInterface; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; @@ -127,6 +128,58 @@ public function __construct( } } + /** + * @Route("/document/list/empty-search", name="document_empty_search") + */ +public function emptySearchAction(Request $request): Response +{ + $formFilter = $this->createForm(FilterType::class, null); + $form = $this->createForm(CombinedFilterType::class, null, [ + 'entityManager' => $this->entityManager, + ]); + + // set the timezone + $timezone = !empty($timezone) ? $this->getUser()->getTimezone() : 'UTC'; + + // Return an empty array so that there will be no documents to display + $documents = array(); + + // default pagination + $compact = $this->nav_pagination([ + 'adapter_em_repository' => $documents, + 'maxPerPage' => 25, + 'page' => 1, + ], false); + + return $this->render('testFilter.html.twig', [ + 'form' => $form->createView(), + 'formFilter'=> $formFilter->createView(), + 'documents' => $documents, + 'nb' => $compact['nb'], + 'entities' => $compact['entities'], + 'pager' => $compact['pager'], + 'condition' => 0, + 'timezone' => $timezone, + 'csvdocumentids' => '', + 'nbDocuments' => 0, + ]); +} + +/** + * @Route("/remove-filter", name="remove_filter", methods={"POST"}) + */ +public function removeFilter(Request $request): JsonResponse +{ + $filterName = $request->request->get('filterName'); + + if ($filterName) { + $this->sessionService->{'remove'.$filterName}(); + return new JsonResponse(['status' => 'success']); + } + + return new JsonResponse(['status' => 'error', 'message' => 'No filter name provided']); +} + // Function to disylay the documents with filters diff --git a/src/Controller/JobSchedulerController.php b/src/Controller/JobSchedulerController.php index 032c5a175..8bc87e298 100644 --- a/src/Controller/JobSchedulerController.php +++ b/src/Controller/JobSchedulerController.php @@ -322,10 +322,14 @@ public function crontabList(): Response $entity = $this->entityManager->getRepository(CronJob::class)->findAll(); + // Fetch the cron_job_result data + $cronJobResults = $this->entityManager->getRepository(CronJobResult::class)->findAll(); + return $this->render('JobScheduler/crontab_list.html.twig', [ 'entity' => $entity, 'timezone' => $timezone, - 'entitiesCron' => $entitiesCron + 'entitiesCron' => $entitiesCron, + 'cronJobResults' => $cronJobResults, ]); } diff --git a/src/DataFixtures/LoadCronJobData.php b/src/DataFixtures/LoadCronJobData.php index 41463464d..b72c65a11 100644 --- a/src/DataFixtures/LoadCronJobData.php +++ b/src/DataFixtures/LoadCronJobData.php @@ -64,6 +64,15 @@ private function generateEntities() } } + if (!$foundCronJob) { + $sql = "SELECT * FROM cron_job LIMIT 1"; + $stmt = $this->manager->getConnection()->executeQuery($sql); + $result = $stmt->fetchAllAssociative(); + if (!empty($result)) { + $foundCronJob = true; + } + } + // If we didn't found the solution we create a new one, otherwise we update it if (!$foundCronJob) { $crontab = CronJob::create($cronJobData['command'], $cronJobData['period']); diff --git a/src/Form/Filter/CombinedFilterType.php b/src/Form/Filter/CombinedFilterType.php index 12e57b716..58dda9d20 100644 --- a/src/Form/Filter/CombinedFilterType.php +++ b/src/Form/Filter/CombinedFilterType.php @@ -27,6 +27,7 @@ public function buildForm(FormBuilderInterface $builder, array $options) ]); // add save button to builder $builder->add('save', SubmitType::class, [ + 'label' => 'Search', 'attr' => ['class' => 'btn btn-primary mb-2'], ]); } diff --git a/templates/JobScheduler/crontab_list.html.twig b/templates/JobScheduler/crontab_list.html.twig index bc7580b92..c95e10d8b 100644 --- a/templates/JobScheduler/crontab_list.html.twig +++ b/templates/JobScheduler/crontab_list.html.twig @@ -95,6 +95,49 @@
+ +
+
+
{{ 'crontab.title_cronjobresult'|trans }}
+ + + + + + + + + + + + + {% for result in cronJobResults %} + + + + + + + + + + + + {% endfor %} + +
{{ result.id }}{{ result.cronJob.id }}{{ result.runAt|date("d/m/Y H:i:s", timezone) }}{{ result.runTime }}{{ result.statusCode }} +
+ {{ result.output|slice(0, 100) ~ '...' }} + +
+ + 		{{ result.output|slice(0, 300) ~ '...' }}{{ result.createdAt|date("d/m/Y H:i:s", timezone) }}{{ result.updatedAt|date("d/m/Y H:i:s", timezone) }}
+
+
+

{{'help.title'|trans}} diff --git a/templates/Task/list.html.twig b/templates/Task/list.html.twig index 86d4e35d5..d5fa150bd 100644 --- a/templates/Task/list.html.twig +++ b/templates/Task/list.html.twig @@ -67,24 +67,33 @@ {% else %} - {% endif %} + {% endif %} {{ task.param }} - + {% if task.status|lower == 'end' %} {{ task.status }} {% else %} {{ task.status }} {% endif %} - + {{ task.begin|date("d/m/Y H:i:s", timezone) }} {{ task.end|date("d/m/Y H:i:s", timezone) }} - {{ task.open }} + {{ task.open }} {{ task.close }} {{ task.cancel }} - {{ task.error }} - {{ task.message|sensor }} - - {% endfor %} + {{ task.error }} + +
+ {{ task.message|slice(0, 100) ~ '...' }} + +
+ + + + {% endfor %} {{'list_task.th.id'|trans}} {{'list_task.th.status'|trans}} diff --git a/templates/base.html.twig b/templates/base.html.twig index c684628a5..8398c07f8 100644 --- a/templates/base.html.twig +++ b/templates/base.html.twig @@ -66,15 +66,6 @@ {{'menu_user.management_smtp'|trans}} - -
  • - - - - - - {{'menu_user.jobscheduler'|trans}} -
  • @@ -160,7 +151,7 @@