diff --git a/apps/qubit/modules/actor/actions/browseAction.class.php b/apps/qubit/modules/actor/actions/browseAction.class.php index 4394ef6e53..11569c1131 100644 --- a/apps/qubit/modules/actor/actions/browseAction.class.php +++ b/apps/qubit/modules/actor/actions/browseAction.class.php @@ -499,7 +499,7 @@ protected function doSearch($request) $this->search->query->setQuery($this->search->queryBool); - return QubitSearch::getInstance()->index->getType('QubitActor')->search($this->search->getQuery(false)); + return QubitSearch::getInstance()->index->getIndex('QubitActor')->search($this->search->getQuery(false)); } private function getRelatedAuthorityUsingSlug($slug) diff --git a/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php b/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php index 9318d5f3d4..043d5cc3da 100644 --- a/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php +++ b/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php @@ -108,7 +108,7 @@ public static function getRelatedInformationObjects($actorId, $page, $limit, $ev $query->setSize($limit); $query->setFrom($limit * ($page - 1)); - return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); } public static function nestedActorAndEventTypeQuery($actorId, $eventTypeId) diff --git a/apps/qubit/modules/clipboard/actions/viewAction.class.php b/apps/qubit/modules/clipboard/actions/viewAction.class.php index 64c84ade89..d418b5db38 100644 --- a/apps/qubit/modules/clipboard/actions/viewAction.class.php +++ b/apps/qubit/modules/clipboard/actions/viewAction.class.php @@ -51,7 +51,7 @@ public function execute($request) $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType($this->entityType)->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex($this->entityType)->search($this->search->query); } // Page results diff --git a/apps/qubit/modules/default/actions/browseAction.class.php b/apps/qubit/modules/default/actions/browseAction.class.php index 2aa7a95dba..86a95bb297 100644 --- a/apps/qubit/modules/default/actions/browseAction.class.php +++ b/apps/qubit/modules/default/actions/browseAction.class.php @@ -146,7 +146,7 @@ protected function populateAggs($resultSet) $this->search->query->setRawQuery($queryParams); - $resultSetWithoutLanguageFilter = QubitSearch::getInstance()->index->getType($this::INDEX_TYPE)->search($this->search->query); + $resultSetWithoutLanguageFilter = QubitSearch::getInstance()->index->getIndex($this::INDEX_TYPE)->search($this->search->query); $count = $resultSetWithoutLanguageFilter->getTotalHits(); } diff --git a/apps/qubit/modules/default/actions/fullTreeViewAction.class.php b/apps/qubit/modules/default/actions/fullTreeViewAction.class.php index afda1afb1a..21ea57f501 100644 --- a/apps/qubit/modules/default/actions/fullTreeViewAction.class.php +++ b/apps/qubit/modules/default/actions/fullTreeViewAction.class.php @@ -175,7 +175,7 @@ protected function doElasticsearchQuery($term, $options = []) // Get results, with drafts filtered when appropriate return QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->search($query->getQuery(false, false)); } @@ -237,7 +237,7 @@ protected function countChildren($id, $options = []) // Return a count of the results found return QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->count($query->getQuery(false, false)); } diff --git a/apps/qubit/modules/default/actions/moveAction.class.php b/apps/qubit/modules/default/actions/moveAction.class.php index 793016f1de..7601729cb2 100644 --- a/apps/qubit/modules/default/actions/moveAction.class.php +++ b/apps/qubit/modules/default/actions/moveAction.class.php @@ -140,10 +140,10 @@ public function execute($request) $this->query->setQuery($this->queryBool); if ($this->resource instanceof QubitInformationObject) { - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->query); } elseif ($this->resource instanceof QubitTerm) { // TODO: Add parent_id for terms in ES, add move button - $resultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($this->query); } // Page results diff --git a/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php b/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php index 87f3121d63..a517cf4913 100644 --- a/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php +++ b/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php @@ -125,7 +125,7 @@ protected function getDescendantDigitalObjectCount() $results = QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->search($search->getQuery(false, true)); return $results->getTotalHits(); diff --git a/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php b/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php index 95bdb95a4d..7b5823ac33 100644 --- a/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php @@ -94,7 +94,7 @@ public function execute($request) $this->query->setQuery($this->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->query); // Page results $this->pager = new QubitSearchPager($resultSet); diff --git a/apps/qubit/modules/informationobject/actions/browseAction.class.php b/apps/qubit/modules/informationobject/actions/browseAction.class.php index 4167cc5644..97a227af4e 100644 --- a/apps/qubit/modules/informationobject/actions/browseAction.class.php +++ b/apps/qubit/modules/informationobject/actions/browseAction.class.php @@ -244,7 +244,7 @@ public function execute($request) $this->setView($request); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->getQuery(false, true)); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->getQuery(false, true)); // Page results $this->pager = new QubitSearchPager($resultSet); diff --git a/apps/qubit/modules/informationobject/actions/inventoryAction.class.php b/apps/qubit/modules/informationobject/actions/inventoryAction.class.php index 5f7c1591e3..5832fb9e99 100644 --- a/apps/qubit/modules/informationobject/actions/inventoryAction.class.php +++ b/apps/qubit/modules/informationobject/actions/inventoryAction.class.php @@ -175,6 +175,6 @@ private static function getResults($resource, $limit = 10, $page = 1, $sort = nu QubitAclSearch::filterDrafts($queryBool); $query->setQuery($queryBool); - return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); } } diff --git a/apps/qubit/modules/repository/actions/browseAction.class.php b/apps/qubit/modules/repository/actions/browseAction.class.php index b17764ba72..6be51aaf5b 100644 --- a/apps/qubit/modules/repository/actions/browseAction.class.php +++ b/apps/qubit/modules/repository/actions/browseAction.class.php @@ -142,7 +142,7 @@ public function execute($request) $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitRepository')->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitRepository')->search($this->search->query); $this->pager = new QubitSearchPager($resultSet); $this->pager->setPage($request->page ? $request->page : 1); @@ -201,7 +201,7 @@ private function getAdvancedFilterTerms() $query = new \Elastica\Query(new \Elastica\Query\MatchAll()); $query->setSize($limit); - $this->repositories = QubitSearch::getInstance()->index->getType('QubitRepository')->search($query); + $this->repositories = QubitSearch::getInstance()->index->getIndex('QubitRepository')->search($query); } /** diff --git a/apps/qubit/modules/repository/actions/holdingsAction.class.php b/apps/qubit/modules/repository/actions/holdingsAction.class.php index 48d4d45618..23ed4d883a 100644 --- a/apps/qubit/modules/repository/actions/holdingsAction.class.php +++ b/apps/qubit/modules/repository/actions/holdingsAction.class.php @@ -91,6 +91,6 @@ public static function getHoldings($id, $page, $limit) $title = sprintf('i18n.%s.title.alphasort', sfContext::getInstance()->user->getCulture()); $query->setSort([$title => 'asc']); - return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); } } diff --git a/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php b/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php index d7d3c3b3cb..7ae3f6ee9f 100644 --- a/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php +++ b/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php @@ -82,6 +82,6 @@ public static function getActors($repositoryId, $page, $limit) $field = sprintf('i18n.%s.authorizedFormOfName.alphasort', sfContext::getInstance()->user->getCulture()); $query->setSort([$field => 'asc']); - return QubitSearch::getInstance()->index->getType('QubitActor')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitActor')->search($query); } } diff --git a/apps/qubit/modules/search/actions/autocompleteAction.class.php b/apps/qubit/modules/search/actions/autocompleteAction.class.php index ec91ccf6ed..02e9447828 100644 --- a/apps/qubit/modules/search/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/search/actions/autocompleteAction.class.php @@ -39,7 +39,6 @@ public function execute($request) $culture = $this->context->user->getCulture(); $client = QubitSearch::getInstance()->client; - $index = QubitSearch::getInstance()->index->getInstance(); // Multisearch object $mSearch = new \Elastica\Multi\Search($client); @@ -74,9 +73,12 @@ public function execute($request) ], ]; + // Wrapper to access ElasticSearch indices + $indexWrapper = QubitSearch::getInstance()->index; + foreach ($items as $item) { $search = new \Elastica\Search($client); - $search->addIndex($index)->addType($index->getType($item['type'])); + $search->addIndex($indexWrapper->getIndex($item['type'])); $query = new \Elastica\Query(); $query->setSize(3)->setSource($item['fields']); diff --git a/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php b/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php index 209edbdce6..4716538646 100644 --- a/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php +++ b/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php @@ -187,7 +187,7 @@ public function doSearch() $query->setFrom($limit * ($page - 1)); $query->setSort(['createdAt' => 'desc']); - $resultSet = QubitSearch::getInstance()->index->getType($this->form->getValue('className'))->search($query); + $resultSet = QubitSearch::getInstance()->index->getIndex($this->form->getValue('className'))->search($query); // Page results $this->pager = new QubitSearchPager($resultSet); diff --git a/apps/qubit/modules/search/actions/indexAction.class.php b/apps/qubit/modules/search/actions/indexAction.class.php index 9bb7d20782..bbd0143ed4 100644 --- a/apps/qubit/modules/search/actions/indexAction.class.php +++ b/apps/qubit/modules/search/actions/indexAction.class.php @@ -48,7 +48,7 @@ public function execute($request) QubitAclSearch::filterDrafts($this->search->queryBool); $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->query); $total = $resultSet->getTotalHits(); if (1 > $total) { diff --git a/apps/qubit/modules/taxonomy/actions/indexAction.class.php b/apps/qubit/modules/taxonomy/actions/indexAction.class.php index 1a9604327e..f17d3650cc 100644 --- a/apps/qubit/modules/taxonomy/actions/indexAction.class.php +++ b/apps/qubit/modules/taxonomy/actions/indexAction.class.php @@ -224,7 +224,7 @@ public function execute($request) $this->query->setSort(['updatedAt' => $request->sortDir]); } - $resultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($this->query); // Return special response in JSON for XHR requests if ($request->isXmlHttpRequest()) { diff --git a/apps/qubit/modules/term/actions/indexAction.class.php b/apps/qubit/modules/term/actions/indexAction.class.php index 3773825054..45ae9c7d9c 100644 --- a/apps/qubit/modules/term/actions/indexAction.class.php +++ b/apps/qubit/modules/term/actions/indexAction.class.php @@ -248,7 +248,7 @@ public function execute($request) QubitAclSearch::filterDrafts($this->search->queryBool); $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->query); // Page results $this->pager = new QubitSearchPager($resultSet); @@ -328,7 +328,7 @@ protected function loadListTerms($request) $listQueryBool->addMust(new \Elastica\Query\Term(['taxonomyId' => $this->resource->taxonomyId])); $listQuery->setQuery($listQueryBool); - $this->listResultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($listQuery); + $this->listResultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($listQuery); // Page list results $this->listPager = new QubitSearchPager($this->listResultSet); diff --git a/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php b/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php index 2d9ba45c82..1000797866 100644 --- a/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php +++ b/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php @@ -62,7 +62,7 @@ public static function getEsDocsRelatedToTerm($relatedModelClass, $term, $option QubitAclSearch::filterDrafts($search->queryBool); } - return QubitSearch::getInstance()->index->getType($relatedModelClass)->search($search->getQuery(false)); + return QubitSearch::getInstance()->index->getIndex($relatedModelClass)->search($search->getQuery(false)); } public static function getEsDocsRelatedToTermCount($relatedModelClass, $termId, $search = null) diff --git a/composer.json b/composer.json index 912f1d0976..8c82b6bd6a 100644 --- a/composer.json +++ b/composer.json @@ -20,7 +20,7 @@ "league/csv": "^9.4", "apereo/phpcas": "^1.3.8", "ezyang/htmlpurifier": "^4.13", - "ruflin/elastica": "5.*", + "ruflin/elastica": "6.*", "jumbojett/openid-connect-php": "^1.0" }, "autoload-dev": { diff --git a/composer.lock b/composer.lock index 6afe91680f..98ffdf5fd8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "035f0da316e6b2ff0e564d5a70f3567a", + "content-hash": "95c831656c031654f6504bb3f06c6b5a", "packages": [ { "name": "apereo/phpcas", @@ -79,31 +79,33 @@ }, { "name": "elasticsearch/elasticsearch", - "version": "v5.4.0", + "version": "v6.8.3", "source": { "type": "git", "url": "https://github.com/elastic/elasticsearch-php.git", - "reference": "d3c5b55ad94f5053ca76c48585b4cde2cdc6bc59" + "reference": "a48d84b133453136b9bfab4e561147129dde3b2d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/d3c5b55ad94f5053ca76c48585b4cde2cdc6bc59", - "reference": "d3c5b55ad94f5053ca76c48585b4cde2cdc6bc59", + "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/a48d84b133453136b9bfab4e561147129dde3b2d", + "reference": "a48d84b133453136b9bfab4e561147129dde3b2d", "shasum": "" }, "require": { - "guzzlehttp/ringphp": "~1.0", - "php": "^5.6|^7.0", - "psr/log": "~1.0" + "ext-json": ">=1.3.7", + "ezimuel/ringphp": "^1.1.2", + "php": "^7.3 || ^8.0", + "psr/log": "^1|^2" }, "require-dev": { - "cpliakas/git-wrapper": "~1.0", - "doctrine/inflector": "^1.1", - "mockery/mockery": "0.9.4", - "phpunit/phpunit": "^4.7|^5.4", - "sami/sami": "~3.2", - "symfony/finder": "^2.8", - "symfony/yaml": "^2.8" + "doctrine/inflector": "^1.3", + "mockery/mockery": "^1.2", + "phpstan/phpstan": "^0.12", + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.4", + "symfony/finder": "~4.0", + "symfony/yaml": "~4.0", + "symplify/git-wrapper": ">=9.0 <9.3.27" }, "suggest": { "ext-curl": "*", @@ -111,6 +113,9 @@ }, "type": "library", "autoload": { + "files": [ + "src/autoload.php" + ], "psr-4": { "Elasticsearch\\": "src/Elasticsearch/" } @@ -122,6 +127,9 @@ "authors": [ { "name": "Zachary Tong" + }, + { + "name": "Enrico Zimuel" } ], "description": "PHP Client for Elasticsearch", @@ -132,93 +140,88 @@ ], "support": { "issues": "https://github.com/elastic/elasticsearch-php/issues", - "source": "https://github.com/elastic/elasticsearch-php/tree/v5.4.0" + "source": "https://github.com/elastic/elasticsearch-php/tree/v6.8.3" }, - "time": "2019-01-08T18:57:00+00:00" + "time": "2024-04-12T13:12:26+00:00" }, { - "name": "ezyang/htmlpurifier", - "version": "v4.18.0", + "name": "ezimuel/guzzlestreams", + "version": "3.1.0", "source": { "type": "git", - "url": "https://github.com/ezyang/htmlpurifier.git", - "reference": "cb56001e54359df7ae76dc522d08845dc741621b" + "url": "https://github.com/ezimuel/guzzlestreams.git", + "reference": "b4b5a025dfee70d6cd34c780e07330eb93d5b997" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ezyang/htmlpurifier/zipball/cb56001e54359df7ae76dc522d08845dc741621b", - "reference": "cb56001e54359df7ae76dc522d08845dc741621b", + "url": "https://api.github.com/repos/ezimuel/guzzlestreams/zipball/b4b5a025dfee70d6cd34c780e07330eb93d5b997", + "reference": "b4b5a025dfee70d6cd34c780e07330eb93d5b997", "shasum": "" }, "require": { - "php": "~5.6.0 || ~7.0.0 || ~7.1.0 || ~7.2.0 || ~7.3.0 || ~7.4.0 || ~8.0.0 || ~8.1.0 || ~8.2.0 || ~8.3.0 || ~8.4.0" + "php": ">=5.4.0" }, "require-dev": { - "cerdic/css-tidy": "^1.7 || ^2.0", - "simpletest/simpletest": "dev-master" - }, - "suggest": { - "cerdic/css-tidy": "If you want to use the filter 'Filter.ExtractStyleBlocks'.", - "ext-bcmath": "Used for unit conversion and imagecrash protection", - "ext-iconv": "Converts text to and from non-UTF-8 encodings", - "ext-tidy": "Used for pretty-printing HTML" + "phpunit/phpunit": "~9.0" }, "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, "autoload": { - "files": [ - "library/HTMLPurifier.composer.php" - ], - "psr-0": { - "HTMLPurifier": "library/" - }, - "exclude-from-classmap": [ - "/library/HTMLPurifier/Language/" - ] + "psr-4": { + "GuzzleHttp\\Stream\\": "src/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ - "LGPL-2.1-or-later" + "MIT" ], "authors": [ { - "name": "Edward Z. Yang", - "email": "admin@htmlpurifier.org", - "homepage": "http://ezyang.com" + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" } ], - "description": "Standards compliant HTML filter written in PHP", - "homepage": "http://htmlpurifier.org/", + "description": "Fork of guzzle/streams (abandoned) to be used with elasticsearch-php", + "homepage": "http://guzzlephp.org/", "keywords": [ - "html" + "Guzzle", + "stream" ], "support": { - "issues": "https://github.com/ezyang/htmlpurifier/issues", - "source": "https://github.com/ezyang/htmlpurifier/tree/v4.18.0" + "source": "https://github.com/ezimuel/guzzlestreams/tree/3.1.0" }, - "time": "2024-11-01T03:51:45+00:00" + "time": "2022-10-24T12:58:50+00:00" }, { - "name": "guzzlehttp/ringphp", - "version": "1.1.1", + "name": "ezimuel/ringphp", + "version": "1.2.2", "source": { "type": "git", - "url": "https://github.com/guzzle/RingPHP.git", - "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b" + "url": "https://github.com/ezimuel/ringphp.git", + "reference": "7887fc8488013065f72f977dcb281994f5fde9f4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/5e2a174052995663dd68e6b5ad838afd47dd615b", - "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b", + "url": "https://api.github.com/repos/ezimuel/ringphp/zipball/7887fc8488013065f72f977dcb281994f5fde9f4", + "reference": "7887fc8488013065f72f977dcb281994f5fde9f4", "shasum": "" }, "require": { - "guzzlehttp/streams": "~3.0", + "ezimuel/guzzlestreams": "^3.0.1", "php": ">=5.4.0", "react/promise": "~2.0" }, + "replace": { + "guzzlehttp/ringphp": "self.version" + }, "require-dev": { "ext-curl": "*", - "phpunit/phpunit": "~4.0" + "phpunit/phpunit": "~9.0" }, "suggest": { "ext-curl": "Guzzle will use specific adapters if cURL is present" @@ -245,68 +248,72 @@ "homepage": "https://github.com/mtdowling" } ], - "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", + "description": "Fork of guzzle/RingPHP (abandoned) to be used with elasticsearch-php", "support": { - "issues": "https://github.com/guzzle/RingPHP/issues", - "source": "https://github.com/guzzle/RingPHP/tree/1.1.1" + "source": "https://github.com/ezimuel/ringphp/tree/1.2.2" }, - "abandoned": true, - "time": "2018-07-31T13:22:33+00:00" + "time": "2022-12-07T11:28:53+00:00" }, { - "name": "guzzlehttp/streams", - "version": "3.0.0", + "name": "ezyang/htmlpurifier", + "version": "v4.18.0", "source": { "type": "git", - "url": "https://github.com/guzzle/streams.git", - "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5" + "url": "https://github.com/ezyang/htmlpurifier.git", + "reference": "cb56001e54359df7ae76dc522d08845dc741621b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/streams/zipball/47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", - "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", + "url": "https://api.github.com/repos/ezyang/htmlpurifier/zipball/cb56001e54359df7ae76dc522d08845dc741621b", + "reference": "cb56001e54359df7ae76dc522d08845dc741621b", "shasum": "" }, "require": { - "php": ">=5.4.0" + "php": "~5.6.0 || ~7.0.0 || ~7.1.0 || ~7.2.0 || ~7.3.0 || ~7.4.0 || ~8.0.0 || ~8.1.0 || ~8.2.0 || ~8.3.0 || ~8.4.0" }, "require-dev": { - "phpunit/phpunit": "~4.0" + "cerdic/css-tidy": "^1.7 || ^2.0", + "simpletest/simpletest": "dev-master" }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.0-dev" - } + "suggest": { + "cerdic/css-tidy": "If you want to use the filter 'Filter.ExtractStyleBlocks'.", + "ext-bcmath": "Used for unit conversion and imagecrash protection", + "ext-iconv": "Converts text to and from non-UTF-8 encodings", + "ext-tidy": "Used for pretty-printing HTML" }, + "type": "library", "autoload": { - "psr-4": { - "GuzzleHttp\\Stream\\": "src/" - } + "files": [ + "library/HTMLPurifier.composer.php" + ], + "psr-0": { + "HTMLPurifier": "library/" + }, + "exclude-from-classmap": [ + "/library/HTMLPurifier/Language/" + ] }, "notification-url": "https://packagist.org/downloads/", "license": [ - "MIT" + "LGPL-2.1-or-later" ], "authors": [ { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" + "name": "Edward Z. Yang", + "email": "admin@htmlpurifier.org", + "homepage": "http://ezyang.com" } ], - "description": "Provides a simple abstraction over streams of data", - "homepage": "http://guzzlephp.org/", + "description": "Standards compliant HTML filter written in PHP", + "homepage": "http://htmlpurifier.org/", "keywords": [ - "Guzzle", - "stream" + "html" ], "support": { - "issues": "https://github.com/guzzle/streams/issues", - "source": "https://github.com/guzzle/streams/tree/master" + "issues": "https://github.com/ezyang/htmlpurifier/issues", + "source": "https://github.com/ezyang/htmlpurifier/tree/v4.18.0" }, - "abandoned": true, - "time": "2014-10-12T19:18:40+00:00" + "time": "2024-11-01T03:51:45+00:00" }, { "name": "jumbojett/openid-connect-php", @@ -785,26 +792,32 @@ }, { "name": "ruflin/elastica", - "version": "5.3.6", + "version": "6.2.1", "source": { "type": "git", "url": "https://github.com/ruflin/Elastica.git", - "reference": "1ed6f587a38d4d6455b688ffc9a9d370d4ae60e9" + "reference": "f95cf646f884a8e7b73bd12eda3e5753994b24d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ruflin/Elastica/zipball/1ed6f587a38d4d6455b688ffc9a9d370d4ae60e9", - "reference": "1ed6f587a38d4d6455b688ffc9a9d370d4ae60e9", + "url": "https://api.github.com/repos/ruflin/Elastica/zipball/f95cf646f884a8e7b73bd12eda3e5753994b24d3", + "reference": "f95cf646f884a8e7b73bd12eda3e5753994b24d3", "shasum": "" }, "require": { - "elasticsearch/elasticsearch": "5.4.*", - "php": ">=5.6.0", + "elasticsearch/elasticsearch": "^6.0", + "ext-json": "*", + "php": "^7.0||^8.0", "psr/log": "~1.0" }, + "conflict": { + "elasticsearch/elasticsearch": "6.8.0 || 6.8.1" + }, "require-dev": { "aws/aws-sdk-php": "~3.0", - "guzzlehttp/guzzle": "~6.0" + "guzzlehttp/guzzle": "~6.0", + "phpunit/phpunit": "^6.0||^8.5.20", + "yoast/phpunit-polyfills": "1.0.1" }, "suggest": { "aws/aws-sdk-php": "Allow using IAM authentication with Amazon ElasticSearch Service", @@ -815,7 +828,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "5.3.x-dev" + "dev-master": "6.0.x-dev" } }, "autoload": { @@ -841,9 +854,9 @@ ], "support": { "issues": "https://github.com/ruflin/Elastica/issues", - "source": "https://github.com/ruflin/Elastica/tree/5.3.6" + "source": "https://github.com/ruflin/Elastica/tree/6.2.1" }, - "time": "2019-08-29T06:34:21+00:00" + "time": "2022-04-14T06:57:28+00:00" } ], "packages-dev": [ diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 92ed06742c..fee1e3fcbb 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -43,7 +43,7 @@ services: - "63001:80" elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16 + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23 env_file: etc/environment ulimits: memlock: diff --git a/docker/etc/environment b/docker/etc/environment index 104549117b..a7ec4cc84f 100644 --- a/docker/etc/environment +++ b/docker/etc/environment @@ -1,6 +1,5 @@ # Elasticsearch bootstrap.memory_lock=true -xpack.security.enabled=false cluster.routing.allocation.disk.threshold_enabled=false ES_JAVA_OPTS=-Xms640m -Xmx640m diff --git a/lib/QubitLftSyncer.class.php b/lib/QubitLftSyncer.class.php index 4b1dc5ab44..bb0d3c171d 100644 --- a/lib/QubitLftSyncer.class.php +++ b/lib/QubitLftSyncer.class.php @@ -68,7 +68,7 @@ private function getChildLftChecksumForElasticsearch() // Get results $result = QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->search($query->getQuery(false, false)); // Amalgamate lft values in array @@ -92,8 +92,8 @@ private function repairEsChildrenLftValues() $results = QubitPdo::fetchAll($sql, $params, ['fetchMode' => PDO::FETCH_ASSOC]); $bulk = new Elastica\Bulk(QubitSearch::getInstance()->client); - $bulk->setIndex(QubitSearch::getInstance()->index->getName()); - $bulk->setType('QubitInformationObject'); + $bulk->setIndex(QubitSearch::getInstance()->index->getIndex('QubitInformationObject')); + $bulk->setType(QubitSearch::getInstance()::ES_TYPE); foreach ($results as $row) { $bulk->addAction( diff --git a/lib/job/arActorExportJob.class.php b/lib/job/arActorExportJob.class.php index 2abafc151a..5e7e71aaa9 100644 --- a/lib/job/arActorExportJob.class.php +++ b/lib/job/arActorExportJob.class.php @@ -46,7 +46,7 @@ public static function findExportRecords($parameters) return QubitSearch::getInstance() ->index - ->getType('QubitActor') + ->getIndex('QubitActor') ->createSearch($query->getQuery(false, false)); } diff --git a/lib/job/arInformationObjectExportJob.class.php b/lib/job/arInformationObjectExportJob.class.php index ab2a6d8e70..ee2c8fb710 100644 --- a/lib/job/arInformationObjectExportJob.class.php +++ b/lib/job/arInformationObjectExportJob.class.php @@ -64,7 +64,7 @@ public static function findExportRecords($parameters) return QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->createSearch($query->getQuery(false, false)); } diff --git a/lib/job/arRepositoryCsvExportJob.class.php b/lib/job/arRepositoryCsvExportJob.class.php index ecd987862a..0f141385c4 100644 --- a/lib/job/arRepositoryCsvExportJob.class.php +++ b/lib/job/arRepositoryCsvExportJob.class.php @@ -85,7 +85,7 @@ protected function exportResults($path) { $itemsExported = 0; - $search = QubitSearch::getInstance()->index->getType('QubitRepository')->createSearch($this->search->getQuery(false, false)); + $search = QubitSearch::getInstance()->index->getIndex('QubitRepository')->createSearch($this->search->getQuery(false, false)); $writer = new csvRepositoryExport($path, null, 10000); $writer->loadResourceSpecificConfiguration('QubitRepository'); diff --git a/lib/job/arUpdateEsActorRelationsJob.class.php b/lib/job/arUpdateEsActorRelationsJob.class.php index d768a80199..941678cbc4 100644 --- a/lib/job/arUpdateEsActorRelationsJob.class.php +++ b/lib/job/arUpdateEsActorRelationsJob.class.php @@ -102,7 +102,7 @@ public static function previousRelationActorIds($actorId) { try { // Get actor's previously indexed relations from Elasticsearch - $doc = QubitSearch::getInstance()->index->getType('QubitActor')->getDocument($actorId); + $doc = QubitSearch::getInstance()->index->getIndex('QubitActor')->getDocument($actorId); return self::uniqueIdsFromRelationData($doc->getData()['actorRelations']); } catch (\Elastica\Exception\NotFoundException $e) { diff --git a/lib/job/arUpdatePublicationStatusJob.class.php b/lib/job/arUpdatePublicationStatusJob.class.php index f3f524b4cc..4d2198aa12 100644 --- a/lib/job/arUpdatePublicationStatusJob.class.php +++ b/lib/job/arUpdatePublicationStatusJob.class.php @@ -78,7 +78,7 @@ public function runJob($parameters) $queryScript = \Elastica\Script\AbstractScript::create([ 'script' => [ - 'inline' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, + 'source' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, 'lang' => 'painless', ], ]); @@ -91,7 +91,7 @@ public function runJob($parameters) 'conflicts' => 'proceed', ]; - $response = QubitSearch::getInstance()->index->updateByQuery($query, $queryScript, $options)->getData(); + $response = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->updateByQuery($query, $queryScript, $options)->getData(); $message = $this->i18n->__( 'Index update completed in %1 ms.', diff --git a/lib/model/QubitInformationObject.php b/lib/model/QubitInformationObject.php index 65e6fef4a8..17da4ff220 100644 --- a/lib/model/QubitInformationObject.php +++ b/lib/model/QubitInformationObject.php @@ -2184,7 +2184,7 @@ public static function getByTitleIdentifierAndRepo($identifier, $title, $repoNam $query = new \Elastica\Query($queryBool); $query->setSize(1); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); if ($resultSet->count()) { return $resultSet[0]->getId(); diff --git a/lib/model/QubitTerm.php b/lib/model/QubitTerm.php index 8e5e0cfa85..c54464220d 100644 --- a/lib/model/QubitTerm.php +++ b/lib/model/QubitTerm.php @@ -896,7 +896,7 @@ public static function getEsTermsByTaxonomyId($taxonomyId, $limit = 10) $query = new \Elastica\Query($queryBool); $query->setSize($limit); - return QubitSearch::getInstance()->index->getType('QubitTerm')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($query); } /** diff --git a/lib/task/propel/propelGenerateSlugsTask.class.php b/lib/task/propel/propelGenerateSlugsTask.class.php index b12bd4d5b0..8f068d1226 100644 --- a/lib/task/propel/propelGenerateSlugsTask.class.php +++ b/lib/task/propel/propelGenerateSlugsTask.class.php @@ -311,7 +311,7 @@ private function getSlugStringFromES($id, $property) $query->setQuery($queryBool); $query->setSize(1); - $results = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + $results = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); if (!$results->count()) { return null; diff --git a/lib/task/search/arDocumentTask.class.php b/lib/task/search/arDocumentTask.class.php index 52cac22473..11528b48f7 100644 --- a/lib/task/search/arDocumentTask.class.php +++ b/lib/task/search/arDocumentTask.class.php @@ -37,7 +37,7 @@ public function execute($arguments = [], $options = []) if (null !== $slugObject = QubitObject::getBySlug($arguments[slug])) { $this->log(sprintf("Fetching data for %s ID %d...\n", $slugObject->className, $slugObject->id)); - $doc = QubitSearch::getInstance()->index->getType($slugObject->className)->getDocument($slugObject->id); + $doc = QubitSearch::getInstance()->index->getIndex($slugObject->className)->getDocument($slugObject->id); echo json_encode($doc->getData(), JSON_PRETTY_PRINT)."\n"; } else { diff --git a/lib/task/search/arSearchStatusTask.class.php b/lib/task/search/arSearchStatusTask.class.php index cb62d9831d..8be4b42043 100644 --- a/lib/task/search/arSearchStatusTask.class.php +++ b/lib/task/search/arSearchStatusTask.class.php @@ -78,7 +78,7 @@ private function objectsIndexed($docType) // Determine model class name from document type name $docTypeModelClass = 'Qubit'.ucfirst($docType); - return QubitSearch::getInstance()->index->getType($docTypeModelClass)->count(); + return QubitSearch::getInstance()->index->getIndex($docTypeModelClass)->count(); } private function objectsAvailableToIndex($docType) diff --git a/lib/task/tools/updatePublicationStatusTask.class.php b/lib/task/tools/updatePublicationStatusTask.class.php index eee2a902eb..230140c1c1 100644 --- a/lib/task/tools/updatePublicationStatusTask.class.php +++ b/lib/task/tools/updatePublicationStatusTask.class.php @@ -166,14 +166,14 @@ protected function updatePublicationStatusDescendants($resource, $publicationSta $queryScript = \Elastica\Script\AbstractScript::create([ 'script' => [ - 'inline' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, + 'source' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, 'lang' => 'painless', ], ]); $options = ['conflicts' => 'proceed']; - $response = QubitSearch::getInstance()->index->updateByQuery($query, $queryScript, $options)->getData(); + $response = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->updateByQuery($query, $queryScript, $options)->getData(); if (!empty($response['failures'])) { $this->failures += count($response['failures']); diff --git a/plugins/arElasticSearchPlugin/config/mapping.yml b/plugins/arElasticSearchPlugin/config/mapping.yml index 550e095023..aa0a2d39aa 100644 --- a/plugins/arElasticSearchPlugin/config/mapping.yml +++ b/plugins/arElasticSearchPlugin/config/mapping.yml @@ -34,8 +34,8 @@ mapping: properties: contact_person: { type: keyword } street_address: { type: text, index: false } - postal_code: { type: text, include_in_all: false } - country_code: { type: keyword, include_in_all: false } + postal_code: { type: text } + country_code: { type: keyword } location: { type: geo_point } other_name: @@ -61,12 +61,12 @@ mapping: dynamic: strict type: nested properties: - start_date: { type: date, include_in_all: false } - end_date: { type: date, include_in_all: false } + start_date: { type: date } + end_date: { type: date } start_date_string: { type: keyword } end_date_string: { type: keyword } - type_id: { type: integer, include_in_all: false } - actor_id: { type: integer, include_in_all: false } + type_id: { type: integer } + actor_id: { type: integer } relation: _attributes: @@ -74,9 +74,9 @@ mapping: dynamic: strict type: nested properties: - object_id: { type: integer, include_in_all: false } - subject_id: { type: integer, include_in_all: false } - type_id: { type: integer, include_in_all: false } + object_id: { type: integer } + subject_id: { type: integer } + type_id: { type: integer } accession_event: _attributes: @@ -91,7 +91,7 @@ mapping: dynamic: strict dynamic: strict properties: - date: { type: date, include_in_all: false } + date: { type: date } date_string: { type: keyword } donor: @@ -109,104 +109,104 @@ mapping: nested_only: true dynamic: strict properties: - basis: { type: text, include_in_all: false } - start_date: { type: date, include_in_all: false } - end_date: { type: date, include_in_all: false } - copyright_status: { type: text, include_in_all: false } - rights_holder: { type: text, include_in_all: false } - rights_note: { type: text, include_in_all: false } - license_terms: { type: text, include_in_all: false } + basis: { type: text } + start_date: { type: date } + end_date: { type: date } + copyright_status: { type: text } + rights_holder: { type: text } + rights_note: { type: text } + license_terms: { type: text } act_right: _attributes: nested_only: true dynamic: strict properties: - act: { type: text, include_in_all: false } - restriction: { type: text, include_in_all: false } - start_date: { type: date, include_in_all: false } - end_date: { type: date, include_in_all: false } + act: { type: text } + restriction: { type: text } + start_date: { type: date } + end_date: { type: date } mediainfo_track: _attributes: nested_only: true dynamic: strict properties: - count: { type: integer, include_in_all: false } - video_format_list: { type: keyword, include_in_all: false } - video_format_with_hint_list: { type: keyword, include_in_all: false } - codecs_video: { type: keyword, include_in_all: false } - video_language_list: { type: keyword, include_in_all: false } - audio_format_list: { type: keyword, include_in_all: false } - audio_format_with_hint_list: { type: keyword, include_in_all: false } - audio_codecs: { type: keyword, include_in_all: false } - audio_language_list: { type: keyword, include_in_all: false } - complete_name: { type: keyword, include_in_all: false } - file_name: { type: keyword, include_in_all: false } - file_extension: { type: keyword, include_in_all: false } - format: { type: keyword, include_in_all: false } - format_info: { type: keyword, include_in_all: false } - format_url: { type: keyword, include_in_all: false } - format_profile: { type: keyword, include_in_all: false } - format_settings: { type: keyword, include_in_all: false } - format_settings_cabac: { type: keyword, include_in_all: false } - format_settings_re_frames: { type: integer, include_in_all: false } - format_settings_gop: { type: keyword, include_in_all: false } - format_extensions_usually_used: { type: keyword, include_in_all: false } - commercial_name: { type: keyword, include_in_all: false } - internet_media_type: { type: keyword, include_in_all: false } - codec_id: { type: keyword, include_in_all: false } - codec_id_info: { type: keyword, include_in_all: false } - codec_id_url: { type: keyword, include_in_all: false } - codec: { type: keyword, include_in_all: false } - codec_family: { type: keyword, include_in_all: false } - codec_info: { type: keyword, include_in_all: false } - codec_url: { type: keyword, include_in_all: false } - codec_cc: { type: keyword, include_in_all: false } - codec_profile: { type: keyword, include_in_all: false } - codec_settings: { type: keyword, include_in_all: false } - codec_settings_cabac: { type: keyword, include_in_all: false } - codec_settings_ref_frames: { type: integer, include_in_all: false } - codec_extensions_usually_used: { type: keyword, include_in_all: false } - file_size: { type: long, include_in_all: false } - duration: { type: integer, include_in_all: false } - bit_rate: { type: integer, include_in_all: false } - bit_rate_mode: { type: keyword, include_in_all: false } - overall_bit_rate: { type: integer, include_in_all: false } - channels: { type: integer, include_in_all: false } - channel_positions: { type: keyword, include_in_all: false } - sampling_rate: { type: integer, include_in_all: false } - samples_count: { type: integer, include_in_all: false } - compression_mode: { type: keyword, include_in_all: false } - width: { type: integer, include_in_all: false } - height: { type: integer, include_in_all: false } - pixel_aspect_ratio: { type: float, include_in_all: false } - display_aspect_ratio: { type: keyword, include_in_all: false } - rotation: { type: float, include_in_all: false } - frame_rate_mode: { type: keyword, include_in_all: false } - frame_rate: { type: float, include_in_all: false } - frame_count: { type: integer, include_in_all: false } - resolution: { type: integer, include_in_all: false } - colorimetry: { type: keyword, include_in_all: false } - color_space: { type: keyword, include_in_all: false } - chroma_subsampling: { type: keyword, include_in_all: false } - bit_depth: { type: integer, include_in_all: false } - scan_type: { type: keyword, include_in_all: false } - interlacement: { type: keyword, include_in_all: false } - bits_pixel_frame: { type: float, include_in_all: false } - stream_size: { type: long, include_in_all: false } - proportion_of_this_stream: { type: float, include_in_all: false } - header_size: { type: long, include_in_all: false } - data_size: { type: long, include_in_all: false } - footer_size: { type: long, include_in_all: false } - language: { type: keyword, include_in_all: false } - color_primaries: { type: keyword, include_in_all: false } - transfer_characteristics: { type: keyword, include_in_all: false } - matrix_coefficients: { type: keyword, include_in_all: false } - is_streamable: { type: boolean, include_in_all: false } - writing_application: { type: keyword, include_in_all: false } - file_last_modification_date: { type: date, include_in_all: false } - file_last_modification_date_local: { type: date, include_in_all: false } + count: { type: integer } + video_format_list: { type: keyword } + video_format_with_hint_list: { type: keyword } + codecs_video: { type: keyword } + video_language_list: { type: keyword } + audio_format_list: { type: keyword } + audio_format_with_hint_list: { type: keyword } + audio_codecs: { type: keyword } + audio_language_list: { type: keyword } + complete_name: { type: keyword } + file_name: { type: keyword } + file_extension: { type: keyword } + format: { type: keyword } + format_info: { type: keyword } + format_url: { type: keyword } + format_profile: { type: keyword } + format_settings: { type: keyword } + format_settings_cabac: { type: keyword } + format_settings_re_frames: { type: integer } + format_settings_gop: { type: keyword } + format_extensions_usually_used: { type: keyword } + commercial_name: { type: keyword } + internet_media_type: { type: keyword } + codec_id: { type: keyword } + codec_id_info: { type: keyword } + codec_id_url: { type: keyword } + codec: { type: keyword } + codec_family: { type: keyword } + codec_info: { type: keyword } + codec_url: { type: keyword } + codec_cc: { type: keyword } + codec_profile: { type: keyword } + codec_settings: { type: keyword } + codec_settings_cabac: { type: keyword } + codec_settings_ref_frames: { type: integer } + codec_extensions_usually_used: { type: keyword } + file_size: { type: long } + duration: { type: integer } + bit_rate: { type: integer } + bit_rate_mode: { type: keyword } + overall_bit_rate: { type: integer } + channels: { type: integer } + channel_positions: { type: keyword } + sampling_rate: { type: integer } + samples_count: { type: integer } + compression_mode: { type: keyword } + width: { type: integer } + height: { type: integer } + pixel_aspect_ratio: { type: float } + display_aspect_ratio: { type: keyword } + rotation: { type: float } + frame_rate_mode: { type: keyword } + frame_rate: { type: float } + frame_count: { type: integer } + resolution: { type: integer } + colorimetry: { type: keyword } + color_space: { type: keyword } + chroma_subsampling: { type: keyword } + bit_depth: { type: integer } + scan_type: { type: keyword } + interlacement: { type: keyword } + bits_pixel_frame: { type: float } + stream_size: { type: long } + proportion_of_this_stream: { type: float } + header_size: { type: long } + data_size: { type: long } + footer_size: { type: long } + language: { type: keyword } + color_primaries: { type: keyword } + transfer_characteristics: { type: keyword } + matrix_coefficients: { type: keyword } + is_streamable: { type: boolean } + writing_application: { type: keyword } + file_last_modification_date: { type: date } + file_last_modification_date_local: { type: date } mediainfo: _attributes: @@ -223,84 +223,84 @@ mapping: mediainfo: mediainfo dynamic: strict properties: - puid: { type: keyword, include_in_all: false } - filename: { type: keyword, include_in_all: false } - last_modified: { type: date, include_in_all: false } - date_ingested: { type: date, include_in_all: false } - size: { type: long, include_in_all: false } - mime_type: { type: keyword, include_in_all: false } + puid: { type: keyword } + filename: { type: keyword } + last_modified: { type: date } + date_ingested: { type: date } + size: { type: long } + mime_type: { type: keyword } audio: type: object properties: - bit_depth: { type: integer, include_in_all: false } - sample_rate: { type: keyword, include_in_all: false } - channels: { type: integer, include_in_all: false } - data_encoding: { type: keyword, include_in_all: false } - offset: { type: integer, include_in_all: false } - byte_order: { type: keyword, include_in_all: false } + bit_depth: { type: integer } + sample_rate: { type: keyword } + channels: { type: integer } + data_encoding: { type: keyword } + offset: { type: integer } + byte_order: { type: keyword } document: type: object properties: - title: { type: keyword, include_in_all: false } - author: { type: keyword, include_in_all: false } - page_count: { type: integer, include_in_all: false } - word_count: { type: integer, include_in_all: false } - character_count: { type: integer, include_in_all: false } - language: { type: keyword, include_in_all: false } - is_protected: { type: boolean, include_in_all: false } - is_rights_managed: { type: boolean, include_in_all: false } - is_tagged: { type: boolean, include_in_all: false } - has_outline: { type: boolean, include_in_all: false } - has_annotations: { type: boolean, include_in_all: false } - has_forms: { type: boolean, include_in_all: false } + title: { type: keyword } + author: { type: keyword } + page_count: { type: integer } + word_count: { type: integer } + character_count: { type: integer } + language: { type: keyword } + is_protected: { type: boolean } + is_rights_managed: { type: boolean } + is_tagged: { type: boolean } + has_outline: { type: boolean } + has_annotations: { type: boolean } + has_forms: { type: boolean } text: type: object properties: - linebreak: { type: keyword, include_in_all: false } - charset: { type: keyword, include_in_all: false } - markup_basis: { type: keyword, include_in_all: false } - markup_basis_version: { type: keyword, include_in_all: false } - markup_language: { type: keyword, include_in_all: false } + linebreak: { type: keyword } + charset: { type: keyword } + markup_basis: { type: keyword } + markup_basis_version: { type: keyword } + markup_language: { type: keyword } format_identification_event: type: object properties: - type: { type: keyword, include_in_all: false } - dateTime: { type: date, include_in_all: false } - detail: { type: keyword, include_in_all: false } - outcome: { type: keyword, include_in_all: false } - outcomeDetailNote: { type: keyword, include_in_all: false } + type: { type: keyword } + dateTime: { type: date } + detail: { type: keyword } + outcome: { type: keyword } + outcomeDetailNote: { type: keyword } linkingAgentIdentifier: type: object properties: - type: { type: keyword, include_in_all: false } - value: { type: keyword, include_in_all: false } + type: { type: keyword } + value: { type: keyword } other_events: type: object properties: - type: { type: keyword, include_in_all: false } - dateTime: { type: date, include_in_all: false } - detail: { type: keyword, include_in_all: false } - outcome: { type: keyword, include_in_all: false } - outcomeDetailNote: { type: keyword, include_in_all: false } + type: { type: keyword } + dateTime: { type: date } + detail: { type: keyword } + outcome: { type: keyword } + outcomeDetailNote: { type: keyword } linkingAgentIdentifier: type: object properties: - type: { type: keyword, include_in_all: false } - value: { type: keyword, include_in_all: false } + type: { type: keyword } + value: { type: keyword } agents: type: object properties: - identifier_type: { type: keyword, include_in_all: false } - identifier_value: { type: keyword, include_in_all: false } - name: { type: keyword, include_in_all: false } - type: { type: keyword, include_in_all: false } + identifier_type: { type: keyword } + identifier_value: { type: keyword } + name: { type: keyword } + type: { type: keyword } format: type: object properties: - name: { type: keyword, include_in_all: false } - version: { type: keyword, include_in_all: false } - registry_name: { type: keyword, include_in_all: false } - registry_key: { type: keyword, include_in_all: false } + name: { type: keyword } + version: { type: keyword } + registry_name: { type: keyword } + registry_key: { type: keyword } physical_object: _attributes: @@ -334,11 +334,11 @@ mapping: digital_objects: premis_object dynamic: strict properties: - uuid: { type: keyword, include_in_all: false } - filename: { type: keyword, include_in_all: false } - size_on_disk: { type: long, include_in_all: false } - digital_object_count: { type: integer, include_in_all: false } - created_at: { type: date, include_in_all: false } + uuid: { type: keyword } + filename: { type: keyword } + size_on_disk: { type: long } + digital_object_count: { type: integer } + created_at: { type: date } term: _attributes: @@ -351,9 +351,9 @@ mapping: dynamic: strict properties: slug: { type: keyword } - taxonomy_id: { type: integer, include_in_all: false } - is_protected: { type: boolean, include_in_all: false } - number_of_descendants: { type: integer, include_in_all: false } + taxonomy_id: { type: integer } + is_protected: { type: boolean } + number_of_descendants: { type: integer } actor: _attributes: @@ -385,20 +385,20 @@ mapping: untouched: type: keyword corporate_body_identifiers: { type: keyword } - entity_type_id: { type: integer, include_in_all: false } - maintaining_repository_id: { type: integer, include_in_all: false } - direct_subjects: { type: integer, include_in_all: false } - direct_places: { type: integer, include_in_all: false } - actor_direct_relation_types: { type: integer, include_in_all: false } - has_digital_object: { type: boolean, include_in_all: false } + entity_type_id: { type: integer } + maintaining_repository_id: { type: integer } + direct_subjects: { type: integer } + direct_places: { type: integer } + actor_direct_relation_types: { type: integer } + has_digital_object: { type: boolean } digital_object: type: object properties: - media_type_id: { type: integer, include_in_all: false } - usage_id: { type: integer, include_in_all: false } + media_type_id: { type: integer } + usage_id: { type: integer } thumbnail_path: { type: keyword } - filename: { type: text } - digital_object_alt_text: { type: text } + filename: { type: text, copy_to: _all } + digital_object_alt_text: { type: text, copy_to: _all } accession: _attributes: @@ -428,7 +428,7 @@ mapping: fields: untouched: type: keyword - date: { type: date, include_in_all: false } + date: { type: date } repository: _attributes: @@ -450,10 +450,10 @@ mapping: fields: untouched: type: keyword - types: { type: integer, include_in_all: false } - geographic_subregions: { type: integer, include_in_all: false } - thematic_areas: { type: integer, include_in_all: false } - logo_path: { type: keyword, include_in_all: false } + types: { type: integer } + geographic_subregions: { type: integer } + thematic_areas: { type: integer } + logo_path: { type: keyword } function_object: _attributes: @@ -590,34 +590,34 @@ mapping: level_of_description_id: { type: integer } lft: { type: integer } publication_status_id: { type: integer } - parent_id: { type: integer, include_in_all: false } - ancestors: { type: integer, include_in_all: false } - children: { type: integer, include_in_all: false } - copyright_status_id: { type: integer, include_in_all: false } - material_type_id: { type: integer, include_in_all: false } - transcript: { type: text, include_in_all: true } - direct_subjects: { type: integer, include_in_all: false } - direct_places: { type: integer, include_in_all: false } - direct_genres: { type: integer, include_in_all: false } - has_digital_object: { type: boolean, include_in_all: false } + parent_id: { type: integer } + ancestors: { type: integer } + children: { type: integer } + copyright_status_id: { type: integer } + material_type_id: { type: integer } + transcript: { type: text, copy_to: _all } + direct_subjects: { type: integer } + direct_places: { type: integer } + direct_genres: { type: integer } + has_digital_object: { type: boolean } finding_aid: type: object properties: - transcript: { type: text, include_in_all: true } - status: { type: integer, include_in_all: false } + transcript: { type: text, copy_to: _all } + status: { type: integer } digital_object: type: object properties: - media_type_id: { type: integer, include_in_all: false } - usage_id: { type: integer, include_in_all: false } + media_type_id: { type: integer } + usage_id: { type: integer } thumbnail_path: { type: keyword } - filename: { type: text } - digital_object_alt_text: { type: text } + filename: { type: text, copy_to: _all } + digital_object_alt_text: { type: text, copy_to: _all } alternative_identifiers: type: object properties: - label: { type: text } - identifier: { type: text } + label: { type: text, copy_to: _all } + identifier: { type: text, copy_to: _all } reference_code: type: text fields: @@ -630,5 +630,5 @@ mapping: search_analyzer: standard term_vector: with_positions_offsets # Not nested date fields for sorting - start_date_sort: { type: date, include_in_all: false } - end_date_sort: { type: date, include_in_all: false } + start_date_sort: { type: date } + end_date_sort: { type: date } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php deleted file mode 100644 index 02a45ec56e..0000000000 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php +++ /dev/null @@ -1,83 +0,0 @@ -. - */ - -use Elastica\Query; -use Elastica\Response; -use Elastica\Script\AbstractScript; -use Elasticsearch\Endpoints\UpdateByQuery; - -/** - * arElasticSearchIndexDecorator can be removed when Elastica >= 6.x. It is present to - * provide Elastica/Index::updateByQuery(). - * - * @author sbreker - */ -class arElasticSearchIndexDecorator -{ - protected $_instance; - - public function __construct(Elastica\Index $instance) - { - $this->_instance = $instance; - } - - public function __call($method, $args) - { - return call_user_func_array([$this->_instance, $method], $args); - } - - public function __get($key) - { - return $this->_instance->{$key}; - } - - public function __set($key, $val) - { - return $this->_instance->{$key} = $val; - } - - public function getInstance() - { - return $this->_instance; - } - - /** - * Update entries in the db based on a query. - * - * @param AbstractQuery|array|Collapse|Query|string|Suggest $query Query object or array - * @param AbstractScript $script Script - * @param array $options Optional params - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html - */ - public function updateByQuery($query, AbstractScript $script, array $options = []): Response - { - $endpoint = new UpdateByQuery(); - $q = Query::create($query)->getQuery(); - $body = [ - 'query' => \is_array($q) ? $q : $q->toArray(), - 'script' => $script->toArray()['script'], - ]; - - $endpoint->setBody($body); - $endpoint->setParams($options); - - return $this->requestEndpoint($endpoint); - } -} diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php index a84d143607..f09ac23ece 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php @@ -263,7 +263,7 @@ protected function processPropertyAttributes($typeName, array &$typeProperties) throw new sfException('No i18n_languages in database settings.'); } - $this->setIfNotSet($typeProperties['properties'], 'sourceCulture', ['type' => 'keyword', 'include_in_all' => false]); + $this->setIfNotSet($typeProperties['properties'], 'sourceCulture', ['type' => 'keyword']); // We are using the same mapping for all the i18n fields $nestedI18nFields = []; @@ -342,7 +342,7 @@ protected function processForeignTypes(array &$typeProperties) $mapping = $this->mapping[$foreignTypeNameCamelized]; // Add id of the foreign resource - $mapping['properties']['id'] = ['type' => 'integer', 'include_in_all' => 'false']; + $mapping['properties']['id'] = ['type' => 'integer']; $typeProperties['properties'][$fieldNameCamelized] = $mapping; } @@ -368,7 +368,7 @@ protected function processPartialForeignTypes(array &$typeProperties) } // Add source culture property - $this->setIfNotSet($mapping['properties'], 'sourceCulture', ['type' => 'keyword', 'include_in_all' => false]); + $this->setIfNotSet($mapping['properties'], 'sourceCulture', ['type' => 'keyword']); $nestedI18nFields = []; foreach ($mapping['_i18nFields'] as $i18nFieldName) { @@ -403,7 +403,7 @@ protected function processPartialForeignTypes(array &$typeProperties) } // Add id of the partial foreign resource - $mapping['properties']['id'] = ['type' => 'integer', 'include_in_all' => 'false']; + $mapping['properties']['id'] = ['type' => 'integer']; $typeProperties['properties'][$fieldNameCamelized] = $mapping; } @@ -443,7 +443,7 @@ protected function getI18nFieldMapping($fieldName) { return [ 'type' => 'text', - 'include_in_all' => true, + 'copy_to' => '_all', ]; } @@ -492,7 +492,6 @@ protected function addSortFields($nestedI18nFields, $typeProperties) $nestedI18nFields[$item]['fields']['alphasort'] = [ 'type' => 'keyword', 'normalizer' => 'alphasort', - 'include_in_all' => false, ]; } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php new file mode 100644 index 0000000000..fc4641a51a --- /dev/null +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php @@ -0,0 +1,88 @@ +. + */ + +/** + * arElasticSearchMultiIndexWrapper facilitates handling ElasticSearch indices + * and has methods that match signatures of pre ES 6.x methods that used a + * single index with multiple types instead of multiple index with a single + * type or no type. + * This class has an indices property which is an array of ElasticSearch + * indices in order to keep arElasticSearchPlugin's index property backwards + * compatible with custom themes. + */ +class arElasticSearchMultiIndexWrapper +{ + protected $indices; + + public function __construct() + { + $this->indices = []; + } + + public function addIndex($name, Elastica\Index $index) + { + $this->indices[$name] = $index; + } + + public function addDocuments($name, $documents) + { + $this->indices[$name]->addDocuments($documents); + } + + public function deleteDocuments($name, $documents) + { + $this->indices[$name]->deleteDocuments($documents); + } + + public function updateDocuments($name, $documents) + { + $this->indices[$name]->updateDocuments($documents); + } + + public function deleteById($name, $documentId) + { + $this->indices[$name]->deleteById($documentId); + } + + /** + * Refresh ElasticSearch indices. If an index name is provided, + * only that specific index will be refreshed. + * + * @param string $name Index name to be refreshed (optional) + */ + public function refresh($name) + { + $this->indices[$name]->refresh(); + } + + // Return the index element from the array of indices + // that matches the qualified index name + public function getIndex($name) + { + return $this->indices[$name]; + } + + // Alias for getIndex. Can be safely removed once + // calls to getIndex that are external to the plugin have + // been removed or refactored. + public function getType($name) + { + return $this->getIndex($name); + } +} diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index 07e9179900..0170ba146e 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -28,7 +28,15 @@ class arElasticSearchPlugin extends QubitSearchEngine /** * Minimum version of Elasticsearch supported. */ - public const MIN_VERSION = '1.3.0'; + public const MIN_VERSION = '6.0.0'; + + /** + * Dummy type for the ElasticSearch index. + * This is required in ES 6.x but it is optional in + * ES 7.x and can be removed when ElasticSearch and + * Elastica are upgraded. + */ + public const ES_TYPE = '_doc'; /** * Elastic_Client object. @@ -44,6 +52,13 @@ class arElasticSearchPlugin extends QubitSearchEngine */ public $index; + /** + * Current batch index name, used for batch flush. + * + * @var mixed defaults to null + */ + protected $currentBatchIndexName; + /** * Mappings configuration, mapping.yml. * @@ -90,9 +105,7 @@ public function __construct(array $options = []) // Verify the version running in the server $this->checkVersion(); - // TODO: arElasticSearchIndexDecorator can be removed when Elastica >= 6.x. It is present to - // provide Elastica/Index::updateByQuery(). - $this->index = new arElasticSearchIndexDecorator($this->client->getIndex($this->config['index']['name'])); + $this->index = new arElasticSearchMultiIndexWrapper(); // Load batch mode configuration $this->batchMode = true === $this->config['batch_mode']; @@ -112,7 +125,6 @@ public function __destruct() } $this->flushBatch(); - $this->index->refresh(); } public static function loadMappings() @@ -137,23 +149,6 @@ public static function loadMappings() return $esMapping; } - public function loadDiacriticsMappings() - { - // Find diacritics_mapping.yml - $diacriticsFinder = sfFinder::type('file')->name('diacritics_mapping.yml'); - $diacriticsFiles = array_unique( - array_merge( - $diacriticsFinder->in(sfConfig::get('sf_upload_dir')), - ) - ); - - if (!count($diacriticsFiles)) { - throw new sfException('You must create a diacritics_mapping.yml file.'); - } - - return sfYaml::load(array_shift($diacriticsFiles)); - } - /** * Optimize index. * @@ -164,16 +159,6 @@ public function optimize($args = []) return $this->client->optimizeAll($args); } - public function flush() - { - try { - $this->index->delete(); - } catch (Exception $e) { - } - - $this->initialize(); - } - /* * Flush batch of documents if we're in batch mode. * @@ -182,35 +167,39 @@ public function flush() */ public function flushBatch() { - if ($this->batchMode) { - // Batch add documents, if any - if (count($this->batchAddDocs) > 0) { - try { - $this->index->addDocuments($this->batchAddDocs); - } catch (Exception $e) { - // Clear batchAddDocs if something went wrong too - $this->batchAddDocs = []; - - throw $e; - } + if (!$this->batchMode || !$this->currentBatchIndexName) { + return; + } + // Batch add documents, if any + if (count($this->batchAddDocs) > 0) { + try { + $this->index->addDocuments($this->currentBatchIndexName, $this->batchAddDocs); + } catch (Exception $e) { + // Clear batchAddDocs if something went wrong too $this->batchAddDocs = []; - } - // Batch delete documents, if any - if (count($this->batchDeleteDocs) > 0) { - try { - $this->index->deleteDocuments($this->batchDeleteDocs); - } catch (Exception $e) { - // Clear batchDeleteDocs if something went wrong too - $this->batchDeleteDocs = []; + throw $e; + } - throw $e; - } + $this->batchAddDocs = []; + } + // Batch delete documents, if any + if (count($this->batchDeleteDocs) > 0) { + try { + $this->index->deleteDocuments($this->currentBatchIndexName, $this->batchDeleteDocs); + } catch (Exception $e) { + // Clear batchDeleteDocs if something went wrong too $this->batchDeleteDocs = []; + + throw $e; } + + $this->batchDeleteDocs = []; } + + $this->index->refresh($this->currentBatchIndexName); } /** @@ -223,22 +212,18 @@ public function populate($options = []) $excludeTypes = (!empty($options['excludeTypes'])) ? $options['excludeTypes'] : []; $update = (!empty($options['update'])) ? $options['update'] : false; - // Delete index and initialize again if all document types are to be - // indexed and not updating - if (!count($excludeTypes) && !$update) { - $this->flush(); - $this->log('Index erased.'); - } else { - // Initialize index if necessary - $this->initialize(); + // Make sure it's initialized, QubitSearch::disable() gets an instance + // without initialization and it's used in the install/purgue tasks. + $this->initialize(); + $this->loadAndNormalizeMappings(); + $this->loadDiacriticsMappings(); + $this->configureFilters(); - // Load mappings if index initialization wasn't needed - $this->loadAndNormalizeMappings(); + $indicesCount = $this->countAndDisplayIndices($excludeTypes, $update); + if (0 == $indicesCount) { + return; } - // Display what types will be indexed - $this->displayTypesToIndex($excludeTypes); - // If we're indexing IOs or Actors we'll cache a term id => parent id // array with all terms from the needed taxonomies in sfConfig. This // array will be used to obtain the related terms ancestor ids without @@ -259,7 +244,11 @@ public function populate($options = []) ); } - $this->log('Populating index...'); + if ($update) { + $this->log('Populating indices...'); + } else { + $this->log('Defining and populating indices...'); + } // Document counter, timer and errors $total = 0; @@ -267,33 +256,32 @@ public function populate($options = []) $errors = []; $showErrors = false; - foreach ($this->mappings as $typeName => $typeProperties) { - if (!in_array(strtolower($typeName), $excludeTypes)) { - $camelizedTypeName = sfInflector::camelize($typeName); - $className = 'arElasticSearch'.$camelizedTypeName; + foreach ($this->mappings as $indexName => $indexProperties) { + if (in_array(strtolower($indexName), $excludeTypes)) { + continue; + } - // If excluding types then index as a whole hasn't been flushed: delete - // type's documents if not updating - if (count($excludeTypes) && !$update) { - $this->index->getType('Qubit'.$camelizedTypeName)->deleteByQuery(new \Elastica\Query\MatchAll()); - } + $camelizedTypeName = sfInflector::camelize($indexName); + $className = 'arElasticSearch'.$camelizedTypeName; + $indexName = 'Qubit'.$camelizedTypeName; - $class = new $className(); - $class->setTimer($timer); + $this->recreateIndex($indexName, $indexProperties, $update); - $typeErrors = $class->populate(); - if (count($typeErrors) > 0) { - $showErrors = true; - $errors = array_merge($errors, $typeErrors); - } + $class = new $className(); + $class->setTimer($timer); - $total += $class->getCount(); + $typeErrors = $class->populate(); + if (count($typeErrors) > 0) { + $showErrors = true; + $errors = array_merge($errors, $typeErrors); } + + $total += $class->getCount(); } $this->log( vsprintf( - 'Index populated with %s documents in %s seconds.', + 'Indices populated with %s documents in %s seconds.', [$total, $timer->elapsed()] ) ); @@ -329,10 +317,14 @@ public function disable() * Centralize document addition to keep control of the batch queue. * * @param mixed $data - * @param mixed $type + * @param mixed $indexName */ - public function addDocument($data, $type) + public function addDocument($data, $indexName) { + if (!$this->enabled) { + return; + } + if (!isset($data['id'])) { throw new sfException('Failed to parse id field.'); } @@ -343,19 +335,30 @@ public function addDocument($data, $type) unset($data['id']); $document = new \Elastica\Document($id, $data); - $document->setType($type); + + // Setting a dummy type since it is required in ES 6.x + // but it can be removed in 7.x when it becomes optional + $document->setType(self::ES_TYPE); if ($this->batchMode) { + if (!$this->currentBatchIndexName) { + $this->currentBatchIndexName = $indexName; + } + + if ($this->currentBatchIndexName != $indexName) { + $this->flushBatch(); + $this->currentBatchIndexName = $indexName; + } + // Add this document to the batch add queue $this->batchAddDocs[] = $document; // If we have a full batch, send additions and deletions in bulk if (count($this->batchAddDocs) >= $this->batchSize) { $this->flushBatch(); - $this->index->refresh(); } } else { - $this->index->getType($type)->addDocument($document); + $this->index->addDocuments($indexName, [$document]); } } @@ -379,10 +382,16 @@ public function partialUpdate($object, $data) return; } + $indexName = get_class($object); + $document = new \Elastica\Document($object->id, $data); + // Setting a dummy type since it is required in ES 6.x + // but it can be removed in 7.x when it becomes optional + $document->setType(self::ES_TYPE); + try { - $this->index->getType(get_class($object))->updateDocument($document); + $this->index->updateDocuments($indexName, [$document]); } catch (\Elastica\Exception\NotFoundException $e) { // Create document if it's not found $this->update($object); @@ -402,7 +411,7 @@ public function partialUpdateById(string $className, int $id, array $data) $document = new \Elastica\Document($id, $data); try { - $this->index->getType($className)->updateDocument($document); + $this->index->updateDocuments($className, [$document]); } catch (\Elastica\Exception\ResponseException $e) { // Create document if none exists $modelPdoClassName = self::modelClassFromQubitObjectClass($className).'Pdo'; @@ -426,24 +435,33 @@ public function delete($object) return; } + $indexName = get_class($object); if ($this->batchMode) { + if (!$this->currentBatchIndexName) { + $this->currentBatchIndexName = $indexName; + } + // The document being deleted may not have been added to the index yet (if it's // still queued up in $this->batchAddDocs) so create a document object representing // the document to be deleted and add this document object to the batch delete // queue. $document = new \Elastica\Document($object->id); - $document->setType(get_class($object)); + $document->setType(self::ES_TYPE); + + if ($this->currentBatchIndexName != $indexName) { + $this->flushBatch(); + $this->currentBatchIndexName = $indexName; + } $this->batchDeleteDocs[] = $document; // If we have a full batch, send additions and deletions in bulk if (count($this->batchDeleteDocs) >= $this->batchSize) { $this->flushBatch(); - $this->index->refresh(); } } else { try { - $this->index->getType(get_class($object))->deleteById($object->id); + $this->index->deleteById($indexName, $object->id); } catch (\Elastica\Exception\NotFoundException $e) { // Ignore } @@ -485,65 +503,106 @@ public static function modelClassFromQubitObjectClass($className) } /** - * Initialize ES index if it does not exist. + * Initialize indices wrapper. */ - protected function initialize() + private function initialize() { - if (sfConfig::get('app_diacritics')) { - $this->config['index']['configuration']['analysis']['char_filter']['diacritics_lowercase'] = $this->loadDiacriticsMappings(); + $indices = ['aip', 'term', 'actor', 'accession', 'repository', 'functionObject', 'informationObject']; + foreach ($indices as $indexName) { + $indexName = 'Qubit'.sfInflector::camelize($indexName); + $prefixedIndexName = $this->config['index']['name'].'_'.strtolower($indexName); + $index = $this->client->getIndex($prefixedIndexName); + $this->index->addIndex($indexName, $index); } + } - try { - $this->index->open(); - } catch (Exception $e) { - // If the index has not been initialized, create it - if ($e instanceof \Elastica\Exception\ResponseException) { - // Based on the markdown_enabled setting, add a new filter to strip Markdown tags - if ( - sfConfig::get('app_markdown_enabled', true) - && isset($this->config['index']['configuration']['analysis']['char_filter']['strip_md']) - ) { - foreach ($this->config['index']['configuration']['analysis']['analyzer'] as $key => $analyzer) { - $filters = ['strip_md']; - - if ($this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']) { - $filters = array_merge($filters, $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']); - } - - if (sfConfig::get('app_diacritics')) { - $filters = array_merge($filters, ['diacritics_lowercase']); - } - - $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = $filters; - } - } + private function recreateIndex($indexName, $indexProperties, $update) + { + $index = $this->index->getIndex($indexName); - $this->index->create( - $this->config['index']['configuration'], - ['recreate' => true] - ); - } + // No need to recreate updating an existing index. + if ($update && $index->exists()) { + return; + } + + // In ES 7.x if the mapping type is updated to a dummy type, + // this may need to include a param for include_type_name + // set to false in order to avoid automatically creating a + // type for the index that was just created + $index->create( + $this->config['index']['configuration'], + ['recreate' => true] + ); + + // Define mapping in elasticsearch + $mapping = new \Elastica\Type\Mapping(); + + // Setting a dummy type since it is required in ES 6.x + // but it can be removed in 7.x when it becomes optional + $mapping->setType($index->getType(self::ES_TYPE)); + $mapping->setProperties($indexProperties['properties']); - // Load and normalize mappings - $this->loadAndNormalizeMappings(); + // Parse other parameters + unset($indexProperties['properties']); + foreach ($indexProperties as $key => $value) { + $mapping->setParam($key, $value); + } + + $this->log(sprintf( + 'Defining mapping for index %s...', + $this->config['index']['name'].'_'.strtolower($indexName) + )); + + // In ES 7.x this should be changed to: + // $mapping->send($index, [ 'include_type_name' => false ]) + // which can be removed in 8.x since that is the default behaviour + // and will have be removed by 9.x when it is discontinued + $mapping->send(); + } - // Iterate over types (actor, informationobject, ...) - foreach ($this->mappings as $typeName => $typeProperties) { - $typeName = 'Qubit'.sfInflector::camelize($typeName); + private function loadDiacriticsMappings() + { + if (!sfConfig::get('app_diacritics')) { + return; + } - // Define mapping in elasticsearch - $mapping = new \Elastica\Type\Mapping(); - $mapping->setType($this->index->getType($typeName)); - $mapping->setProperties($typeProperties['properties']); + // Find diacritics_mapping.yml + $diacriticsFinder = sfFinder::type('file')->name('diacritics_mapping.yml'); + $diacriticsFiles = array_unique( + array_merge( + $diacriticsFinder->in(sfConfig::get('sf_upload_dir')), + ) + ); - // Parse other parameters - unset($typeProperties['properties']); - foreach ($typeProperties as $key => $value) { - $mapping->setParam($key, $value); + if (!count($diacriticsFiles)) { + throw new sfException('You must create a diacritics_mapping.yml file.'); + } + + $this->config['index']['configuration']['analysis']['char_filter']['diacritics_lowercase'] = sfYaml::load(array_shift($diacriticsFiles)); + } + + /** + * Set filter configuration params based on markdown settings. + */ + private function configureFilters() + { + // Based on markdown_enabled setting, add a new filter to strip Markdown tags + if ( + sfConfig::get('app_markdown_enabled', true) + && isset($this->config['index']['configuration']['analysis']['char_filter']['strip_md']) + ) { + foreach ($this->config['index']['configuration']['analysis']['analyzer'] as $key => $analyzer) { + $filters = ['strip_md']; + + if ($this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']) { + $filters = array_merge($filters, $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']); + } + + if (sfConfig::get('app_diacritics')) { + $filters = array_merge($filters, ['diacritics_lowercase']); } - $this->log(sprintf('Defining mapping %s...', $typeName)); - $mapping->send(); + $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = $filters; } } } @@ -595,25 +654,31 @@ private function loadAndNormalizeMappings() } /** - * Display types that will be indexed. + * Count and display indices that will be created/updated. + * + * @param array $excludeTypes + * @param bool $update * - * @param mixed $excludeTypes + * @return int Count of indices */ - private function displayTypesToIndex($excludeTypes) + private function countAndDisplayIndices($excludeTypes, $update) { - $typeCount = 0; + $count = 0; - $this->log('Types that will be indexed:'); + $this->log(sprintf('Indices that will be %s:', $update ? 'updated' : 'created')); - foreach ($this->mappings as $typeName => $typeProperties) { - if (!in_array(strtolower($typeName), $excludeTypes)) { - $this->log(' - '.$typeName); - ++$typeCount; + foreach ($this->mappings as $indexName => $indexProperties) { + $indexName = strtolower($indexName); + if (!in_array($indexName, $excludeTypes)) { + $this->log(' - '.$this->config['index']['name'].'_'.$indexName); + ++$count; } } - if (!$typeCount) { + if (!$count) { $this->log(' None'); } + + return $count; } } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php index 9283e41e5c..7ce541e3ee 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php @@ -482,8 +482,8 @@ protected static function getAllObjectStringFields( // Get string fields included in _all elseif ( ( - !isset($propertyProperties['include_in_all']) - || $propertyProperties['include_in_all'] + isset($propertyProperties['copy_to']) + && ('_all' == $propertyProperties['copy_to']) ) && ( isset($propertyProperties['type']) && 'text' == $propertyProperties['type'] diff --git a/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php b/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php index 0861621527..ce196b2367 100644 --- a/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php +++ b/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php @@ -97,7 +97,7 @@ protected function get($request) $this->search->query->setSort([$field => $order]); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->getQuery(false, true)); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->getQuery(false, true)); // Build array from results $results = $lodMapping = []; diff --git a/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php b/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php index 2cccfa1445..dc7040f0db 100644 --- a/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php +++ b/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php @@ -154,7 +154,7 @@ public function execute($request) break; } - $resultSet = QubitSearch::getInstance()->index->getType('QubitAccession')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitAccession')->search($this->query); $this->pager = new QubitSearchPager($resultSet); $this->pager->setPage($request->page ? $request->page : 1); diff --git a/plugins/sfSkosPlugin/test/unit/importTest.php b/plugins/sfSkosPlugin/test/unit/importTest.php index 5f53660911..e2659d5dde 100644 --- a/plugins/sfSkosPlugin/test/unit/importTest.php +++ b/plugins/sfSkosPlugin/test/unit/importTest.php @@ -969,14 +969,14 @@ function withTransaction($callback) foreach ($parent->getDescendants() as $key => $item) { try { - $search->index->getType('QubitTerm')->getDocument($item->id); + $search->index->getIndex('QubitTerm')->getDocument($item->id); $t->pass("Term {$key} is indexed"); } catch (Elastica\Exception\NotFoundException $e) { $t->fail("Term {$key} was not indexed"); } } - $doc = $search->index->getType('QubitTerm')->getDocument($parent->id)->getData(); + $doc = $search->index->getIndex('QubitTerm')->getDocument($parent->id)->getData(); $t->is($doc['numberOfDescendants'], count($importer->getGraph()->allOfType('skos:Concept')), 'Parent term ES document :numberOfDescendants: field is up to date'); });