Speed up ncr:reindex-nodes command by allowing larger batch size

gdbots · Mar 25, 2019 · 21bab98 · 21bab98
1 parent 8aedccb
commit 21bab98
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/CHANGELOG-0.x.md b/CHANGELOG-0.x.md
@@ -2,6 +2,10 @@
 This changelog references the relevant changes done in 0.x versions.
 
 
+## v0.3.14
+* Speed up `ncr:reindex-nodes` command by allowing larger batch size and smaller batch delay.
+
+
 ## v0.3.13
 * Add `NcrExtension::derefNodes` and twig function `ncr_deref_nodes`.
 

diff --git a/src/Command/ExportNodesCommand.php b/src/Command/ExportNodesCommand.php
@@ -5,6 +5,7 @@
 
 use Gdbots\Common\Util\NumberUtils;
 use Gdbots\Ncr\Ncr;
+use Gdbots\Pbj\SchemaQName;
 use Gdbots\Schemas\Ncr\Mixin\Node\Node;
 use Gdbots\Schemas\Ncr\Mixin\Node\NodeV1Mixin;
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
@@ -91,10 +92,13 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $output->setVerbosity(OutputInterface::VERBOSITY_QUIET);
         $errOutput->setVerbosity(OutputInterface::VERBOSITY_NORMAL);
 
-        $batchSize = NumberUtils::bound($input->getOption('batch-size'), 1, 1000);
-        $batchDelay = NumberUtils::bound($input->getOption('batch-delay'), 100, 600000);
+        $batchSize = NumberUtils::bound($input->getOption('batch-size'), 1, 2000);
+        $batchDelay = NumberUtils::bound($input->getOption('batch-delay'), 10, 600000);
         $context = json_decode($input->getOption('context') ?: '{}', true);
         $context['tenant_id'] = (string)$input->getOption('tenant-id');
+        $context['exporting'] = true;
+        $qname = $input->getArgument('qname') ? SchemaQName::fromString($input->getArgument('qname')) : null;
+        $context['exporting_all'] = null === $qname;
 
         $i = 0;
 
@@ -114,7 +118,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             }
         };
 
-        foreach ($this->getSchemasUsingMixin(NodeV1Mixin::create(), $input->getArgument('qname')) as $schema) {
+        foreach ($this->getSchemasUsingMixin(NodeV1Mixin::create(), (string)$qname ?: null) as $schema) {
             $this->ncr->pipeNodes($schema->getQName(), $receiver, $context);
         }
     }

diff --git a/src/Command/ReindexNodesCommand.php b/src/Command/ReindexNodesCommand.php
@@ -107,13 +107,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
     {
         $dryRun = $input->getOption('dry-run');
         $skipErrors = $input->getOption('skip-errors');
-        $batchSize = NumberUtils::bound($input->getOption('batch-size'), 1, 1000);
-        $batchDelay = NumberUtils::bound($input->getOption('batch-delay'), 100, 600000);
+        $batchSize = NumberUtils::bound($input->getOption('batch-size'), 1, 2000);
+        $batchDelay = NumberUtils::bound($input->getOption('batch-delay'), 10, 600000);
         $context = json_decode($input->getOption('context') ?: '{}', true);
         $context['tenant_id'] = (string)$input->getOption('tenant-id');
         $context['skip_errors'] = $skipErrors;
         $context['reindexing'] = true;
         $qname = $input->getArgument('qname') ? SchemaQName::fromString($input->getArgument('qname')) : null;
+        $context['reindex_all'] = null === $qname;
 
         $io = new SymfonyStyle($input, $output);
         $io->title(sprintf('Reindexing nodes for qname "%s"', $qname ?? 'ALL'));