Update ElasticSearch and Elastica to 6.x (#1880)

* Update ElasticSearch and Elastica dependencies

Update ElasticSearch to 6.8.23 in the Dockerfile and update Elastica to
6.x in composer.

* Remove arElasticSearchIndexDecorator.

- 'inline' scripts are deprecated, changed to 'source'

* Update index for ES 6.x

Update arElasticSearchPlugin to use multiple indices instead of multiple
types since ES 6.x removed being able to add multiple types. Also update
mapping to remove include_in_all and add that to the _all field using
copy_to instead since include_in_all was also removed in ES 6.x

* Fix autocompleteAction for ES 6.x

Update autocompleteAction to use multiple indices

* Add index types for ES Queries without a type

Add the appropriate index types for ES requests that did not need to
specify explicit types in ES 5.x

* Fix updateByQuery calls for ElasticSearch 6.x

* Fix broken publication status update

Add missing document type to partialUpdate in arElasticSearchPlugin

* Fix linting errors

* Switch elasticsearch to the oss docker image

* Update mapping condition for _all fields

Update condition in arElasticSearchPluginUtil to only get string
fields included in _all if copy_to is explicitly set to _all in the
mappping.

* Update ES mapping to copy relevant fields to _all

* Rename arElasticSearchIndexDecorator

Renamed arElasticSearchIndexDecorator as
arElasticSearchMultiIndexWrapper for clarity and added a few comments
about changes needed for ElasticSearch/Elastica 7.x.

* Rename getType function from ESMultiIndexWrapper

Rename getType from arElasticSearchMultiIndexWrapper to getIndex for
clarity.

* Add dummy type for ES index

Add a dummy ElasticSearch index type since one is still required for
ES 6.x, instead of having a seperate type name for each of the
multiple indices.

* Change references to index type for ElasticSearch

Update any references to ElasticSearch index type to index name.
Also add getType as an alias for getIndex for backwards compatibility
with custom themes that still might be making references to
ElasticSearch's getType.

* Fix copy to _all fields for ES mapping

Removed copy_to _all from all fields that previously had include_in_all
set to false.

* Remove unnecessary ElasticSearch type variables

Remove unnecessary $type ElasticSearch variables in QubitLftSyncer,
arElasticSearchPlugin, and updatePublicationStatusTask

* Remove ES index prefix from MultiIndexWrapper

Remove the index prefix from arElasticSearchMultiIndexWrapper and use
AtoM class names internally instead of using prefix to refer to them.

* Fix autocompleteAction bug, address CR feedback

Fix bug in search autocompleteAction which was showing duplicate results
for actors under repositories. Also address some of the CR feedback and
update/add methods for add, update, and deleteById in the
arElasticSearchMultiIndexWrapper.

* Remove non text fields from _all ES mapping

* Remove unnecessary loop in arElasticSearchPlugin

Remove unnecessary foreach loop in arElasticSearchPlugin. Also move code
for adding filter for stripping md tags into a seperate method to make
the initialization code more readable.

* Update ES index refresh to only refresh one index

Update the index refresh method to accept an optional name param that
only refreshes that particular index in
arElasticSearchMultiIndexWrapper. Also address some small CR feedback.

* Refactor ES initialize and populate methods (WIP)

* Tweak index recreation

* Initialize on populate

* Tweak flush/refresh operation

* Only ignore 404 errors when deleting indexes

* Prevent flush if current index name is not set

* Make loadDiacriticsMappings a private function

* Improve populate logging

* Configure filters once before indices creation

* Normalize loadDiacriticsMappings function

Check setting and update config global variable there.

* Create indices on update if they don't exist

* Trust recreate option on index creation

* Remove unused function

---------

Co-authored-by: melaniekung <mkung@artefactual.com>
Co-authored-by: José Raddaoui Marín <raddaouimarin@gmail.com>

apps/qubit/modules/actor/actions/browseAction.class.php

@@ -499,7 +499,7 @@ protected function doSearch($request)
 
         $this->search->query->setQuery($this->search->queryBool);
 
-        return QubitSearch::getInstance()->index->getType('QubitActor')->search($this->search->getQuery(false));
+        return QubitSearch::getInstance()->index->getIndex('QubitActor')->search($this->search->getQuery(false));
     }
 
     private function getRelatedAuthorityUsingSlug($slug)

apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php

@@ -108,7 +108,7 @@ public static function getRelatedInformationObjects($actorId, $page, $limit, $ev
         $query->setSize($limit);
         $query->setFrom($limit * ($page - 1));
 
-        return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query);
+        return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query);
     }
 
     public static function nestedActorAndEventTypeQuery($actorId, $eventTypeId)

apps/qubit/modules/clipboard/actions/viewAction.class.php

@@ -51,7 +51,7 @@ public function execute($request)
 
             $this->search->query->setQuery($this->search->queryBool);
 
-            $resultSet = QubitSearch::getInstance()->index->getType($this->entityType)->search($this->search->query);
+            $resultSet = QubitSearch::getInstance()->index->getIndex($this->entityType)->search($this->search->query);
         }
 
         // Page results

apps/qubit/modules/default/actions/browseAction.class.php

@@ -146,7 +146,7 @@ protected function populateAggs($resultSet)
 
                     $this->search->query->setRawQuery($queryParams);
 
-                    $resultSetWithoutLanguageFilter = QubitSearch::getInstance()->index->getType($this::INDEX_TYPE)->search($this->search->query);
+                    $resultSetWithoutLanguageFilter = QubitSearch::getInstance()->index->getIndex($this::INDEX_TYPE)->search($this->search->query);
 
                     $count = $resultSetWithoutLanguageFilter->getTotalHits();
                 }

apps/qubit/modules/default/actions/fullTreeViewAction.class.php

@@ -175,7 +175,7 @@ protected function doElasticsearchQuery($term, $options = [])
         // Get results, with drafts filtered when appropriate
         return QubitSearch::getInstance()
             ->index
-            ->getType('QubitInformationObject')
+            ->getIndex('QubitInformationObject')
             ->search($query->getQuery(false, false));
     }
 
@@ -237,7 +237,7 @@ protected function countChildren($id, $options = [])
         // Return a count of the results found
         return QubitSearch::getInstance()
             ->index
-            ->getType('QubitInformationObject')
+            ->getIndex('QubitInformationObject')
             ->count($query->getQuery(false, false));
     }
 

apps/qubit/modules/default/actions/moveAction.class.php

@@ -140,10 +140,10 @@ public function execute($request)
         $this->query->setQuery($this->queryBool);
 
         if ($this->resource instanceof QubitInformationObject) {
-            $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->query);
+            $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->query);
         } elseif ($this->resource instanceof QubitTerm) {
             // TODO: Add parent_id for terms in ES, add move button
-            $resultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($this->query);
+            $resultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($this->query);
         }
 
         // Page results

apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php

@@ -125,7 +125,7 @@ protected function getDescendantDigitalObjectCount()
 
         $results = QubitSearch::getInstance()
             ->index
-            ->getType('QubitInformationObject')
+            ->getIndex('QubitInformationObject')
             ->search($search->getQuery(false, true));
 
         return $results->getTotalHits();

apps/qubit/modules/informationobject/actions/autocompleteAction.class.php

@@ -94,7 +94,7 @@ public function execute($request)
 
         $this->query->setQuery($this->queryBool);
 
-        $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->query);
+        $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->query);
 
         // Page results
         $this->pager = new QubitSearchPager($resultSet);

apps/qubit/modules/informationobject/actions/browseAction.class.php

@@ -244,7 +244,7 @@ public function execute($request)
 
         $this->setView($request);
 
-        $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->getQuery(false, true));
+        $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->getQuery(false, true));
 
         // Page results
         $this->pager = new QubitSearchPager($resultSet);

apps/qubit/modules/informationobject/actions/inventoryAction.class.php

@@ -175,6 +175,6 @@ private static function getResults($resource, $limit = 10, $page = 1, $sort = nu
         QubitAclSearch::filterDrafts($queryBool);
         $query->setQuery($queryBool);
 
-        return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query);
+        return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query);
     }
 }

apps/qubit/modules/repository/actions/browseAction.class.php

@@ -142,7 +142,7 @@ public function execute($request)
 
         $this->search->query->setQuery($this->search->queryBool);
 
-        $resultSet = QubitSearch::getInstance()->index->getType('QubitRepository')->search($this->search->query);
+        $resultSet = QubitSearch::getInstance()->index->getIndex('QubitRepository')->search($this->search->query);
 
         $this->pager = new QubitSearchPager($resultSet);
         $this->pager->setPage($request->page ? $request->page : 1);
@@ -201,7 +201,7 @@ private function getAdvancedFilterTerms()
         $query = new \Elastica\Query(new \Elastica\Query\MatchAll());
         $query->setSize($limit);
 
-        $this->repositories = QubitSearch::getInstance()->index->getType('QubitRepository')->search($query);
+        $this->repositories = QubitSearch::getInstance()->index->getIndex('QubitRepository')->search($query);
     }
 
     /**

apps/qubit/modules/repository/actions/holdingsAction.class.php

@@ -91,6 +91,6 @@ public static function getHoldings($id, $page, $limit)
         $title = sprintf('i18n.%s.title.alphasort', sfContext::getInstance()->user->getCulture());
         $query->setSort([$title => 'asc']);
 
-        return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query);
+        return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query);
     }
 }

apps/qubit/modules/repository/actions/maintainedActorsAction.class.php

@@ -82,6 +82,6 @@ public static function getActors($repositoryId, $page, $limit)
         $field = sprintf('i18n.%s.authorizedFormOfName.alphasort', sfContext::getInstance()->user->getCulture());
         $query->setSort([$field => 'asc']);
 
-        return QubitSearch::getInstance()->index->getType('QubitActor')->search($query);
+        return QubitSearch::getInstance()->index->getIndex('QubitActor')->search($query);
     }
 }

apps/qubit/modules/search/actions/autocompleteAction.class.php

@@ -39,7 +39,6 @@ public function execute($request)
         $culture = $this->context->user->getCulture();
 
         $client = QubitSearch::getInstance()->client;
-        $index = QubitSearch::getInstance()->index->getInstance();
 
         // Multisearch object
         $mSearch = new \Elastica\Multi\Search($client);
@@ -74,9 +73,12 @@ public function execute($request)
             ],
         ];
 
+        // Wrapper to access ElasticSearch indices
+        $indexWrapper = QubitSearch::getInstance()->index;
+
         foreach ($items as $item) {
             $search = new \Elastica\Search($client);
-            $search->addIndex($index)->addType($index->getType($item['type']));
+            $search->addIndex($indexWrapper->getIndex($item['type']));
 
             $query = new \Elastica\Query();
             $query->setSize(3)->setSource($item['fields']);

apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php

@@ -187,7 +187,7 @@ public function doSearch()
         $query->setFrom($limit * ($page - 1));
         $query->setSort(['createdAt' => 'desc']);
 
-        $resultSet = QubitSearch::getInstance()->index->getType($this->form->getValue('className'))->search($query);
+        $resultSet = QubitSearch::getInstance()->index->getIndex($this->form->getValue('className'))->search($query);
 
         // Page results
         $this->pager = new QubitSearchPager($resultSet);

apps/qubit/modules/search/actions/indexAction.class.php

@@ -48,7 +48,7 @@ public function execute($request)
         QubitAclSearch::filterDrafts($this->search->queryBool);
         $this->search->query->setQuery($this->search->queryBool);
 
-        $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->query);
+        $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->query);
 
         $total = $resultSet->getTotalHits();
         if (1 > $total) {

apps/qubit/modules/taxonomy/actions/indexAction.class.php

@@ -224,7 +224,7 @@ public function execute($request)
                 $this->query->setSort(['updatedAt' => $request->sortDir]);
         }
 
-        $resultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($this->query);
+        $resultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($this->query);
 
         // Return special response in JSON for XHR requests
         if ($request->isXmlHttpRequest()) {

apps/qubit/modules/term/actions/indexAction.class.php

@@ -248,7 +248,7 @@ public function execute($request)
             QubitAclSearch::filterDrafts($this->search->queryBool);
             $this->search->query->setQuery($this->search->queryBool);
 
-            $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->query);
+            $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->query);
 
             // Page results
             $this->pager = new QubitSearchPager($resultSet);
@@ -328,7 +328,7 @@ protected function loadListTerms($request)
         $listQueryBool->addMust(new \Elastica\Query\Term(['taxonomyId' => $this->resource->taxonomyId]));
 
         $listQuery->setQuery($listQueryBool);
-        $this->listResultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($listQuery);
+        $this->listResultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($listQuery);
 
         // Page list results
         $this->listPager = new QubitSearchPager($this->listResultSet);

apps/qubit/modules/term/actions/navigateRelatedComponent.class.php

@@ -62,7 +62,7 @@ public static function getEsDocsRelatedToTerm($relatedModelClass, $term, $option
             QubitAclSearch::filterDrafts($search->queryBool);
         }
 
-        return QubitSearch::getInstance()->index->getType($relatedModelClass)->search($search->getQuery(false));
+        return QubitSearch::getInstance()->index->getIndex($relatedModelClass)->search($search->getQuery(false));
     }
 
     public static function getEsDocsRelatedToTermCount($relatedModelClass, $termId, $search = null)

composer.json

@@ -20,7 +20,7 @@
         "league/csv": "^9.4",
         "apereo/phpcas": "^1.3.8",
         "ezyang/htmlpurifier": "^4.13",
-        "ruflin/elastica": "5.*",
+        "ruflin/elastica": "6.*",
         "jumbojett/openid-connect-php": "^1.0"
     },
     "autoload-dev": {