From d7beec6c4accafc1587d8550659fce990294a348 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20Mu=CC=88ller?= <christian@flownative.com>
Date: Tue, 19 Sep 2017 11:40:11 +0200
Subject: [PATCH] FEATURE: Make batch size configurable

To control the memory usage of single indexing jobs better the
batch size can now be configured via the:

    Flowpack:
      ElasticSearch:
        ContentRepositoryQueueIndexer:
          batchSize: 50

configuration setting. For backwards compatibility reasons a fallback
of 500 (the old hardcoded default) exists if the setting was not
configured at all.
---
 Classes/Command/NodeIndexQueueCommandController.php | 9 ++++++++-
 Configuration/Settings.yaml                         | 3 ++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Classes/Command/NodeIndexQueueCommandController.php b/Classes/Command/NodeIndexQueueCommandController.php
index d1b77ff..35246e7 100644
--- a/Classes/Command/NodeIndexQueueCommandController.php
+++ b/Classes/Command/NodeIndexQueueCommandController.php
@@ -26,6 +26,7 @@ class NodeIndexQueueCommandController extends CommandController
 
     const BATCH_QUEUE_NAME = 'Flowpack.ElasticSearch.ContentRepositoryQueueIndexer';
     const LIVE_QUEUE_NAME = 'Flowpack.ElasticSearch.ContentRepositoryQueueIndexer.Live';
+    const DEFAULT_BATCH_SIZE = 500;
 
     /**
      * @var JobManager
@@ -69,6 +70,12 @@ class NodeIndexQueueCommandController extends CommandController
      */
     protected $nodeIndexer;
 
+    /**
+     * @Flow\InjectConfiguration(package="Flowpack.ElasticSearch.ContentRepositoryQueueIndexer")
+     * @var array
+     */
+    protected $settings;
+
     /**
      * Index all nodes by creating a new index and when everything was completed, switch the index alias.
      *
@@ -210,7 +217,7 @@ protected function indexWorkspace($workspaceName, $indexPostfix)
         $this->outputLine('<info>++</info> Indexing %s workspace', [$workspaceName]);
         $nodeCounter = 0;
         $offset = 0;
-        $batchSize = 500;
+        $batchSize = $this->settings['batchSize'] ?? static::DEFAULT_BATCH_SIZE;
         while (true) {
             $iterator = $this->nodeDataRepository->findAllBySiteAndWorkspace($workspaceName, $offset, $batchSize);
 
diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml
index 479243b..1aeca68 100644
--- a/Configuration/Settings.yaml
+++ b/Configuration/Settings.yaml
@@ -2,7 +2,8 @@ Flowpack:
   ElasticSearch:
     ContentRepositoryQueueIndexer:
       enableLiveAsyncIndexing: true
-
+      # Change size of single batch jobs via this setting (fallback default is 500)
+      # batchSize: 50
   JobQueue:
     Common:
       queues: