From b721a616e8ccb14c5338351c2e91ba5716799505 Mon Sep 17 00:00:00 2001 From: Jacques-Henri Berthemet Date: Thu, 18 Jul 2024 15:49:40 +0200 Subject: [PATCH] Support for Cassandra 4.0.x --- README.md | 32 +- pom.xml | 86 +- .../webme/commons/index/CStarUtils.java | 147 +- .../webme/commons/index/ElasticIndex.java | 1794 ++++++++--------- .../webme/commons/index/EsIndexBuilder.java | 61 +- .../webme/commons/index/EsSecondaryIndex.java | 257 +-- .../webme/commons/index/JsonUtils.java | 656 +++--- .../index/config/IndexConfiguration.java | 2 +- .../commons/index/config/OptionReader.java | 209 +- .../index/config/OptionReaderImpl.java | 218 -- .../index/indexers/FakePartitionIterator.java | 68 +- .../index/indexers/SingleRowIterator.java | 26 +- .../indexers/StreamingPartitionIterator.java | 97 +- .../db/filter/EsSimpleExpression.java | 30 + .../commons/index/ElasticIndexConfigTest.java | 182 +- .../commons/index/EsSecondaryIndexTest.java | 342 ++-- .../index/EsSecondaryIndexUnderTest.java | 102 +- .../webme/commons/index/JsonUtilsTest.java | 196 +- 18 files changed, 2217 insertions(+), 2288 deletions(-) delete mode 100644 src/main/java/com/genesyslab/webme/commons/index/config/OptionReaderImpl.java create mode 100644 src/main/java/org/apache/cassandra/db/filter/EsSimpleExpression.java diff --git a/README.md b/README.md index 6467330..650a63c 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ This documentation explains usage and configuration of "ESIndex" that is a Elast This plugin requires an Elasticsearch (ES) cluster already configured. -The plugin will install in a regular Cassandra 3.11.x release downloaded from http://cassandra.apache.org/. +The plugin will install in a regular Cassandra 4.0.x release downloaded from http://cassandra.apache.org/. There is nothing to change in Cassandra configuration files to support the index. Cassandra’s behaviour remains unchanged for applications that do not use the index. @@ -62,17 +62,17 @@ ElasticSearch index for Cassandra using: ![Diag](doc/diagram.png) # Supported Versions -Tested versions are Elasticsearch 5.x, 6.x and Cassandra 3.11.x. However the plugin may also work with different Elasticsearch versions -(1.7, 2.x 5.x, 6.x) if the application provides the corresponding mappings and options. -Other versions of Apache Cassandra like 3.0, 2.2 or 4.0 are not supported as the secondary index interface used by the plugin is different. +Tested versions are Elasticsearch 5.x, 6.x, 7.x and Cassandra 4.0.x. However the plugin may also work with different Elasticsearch versions +(1.7, 2.x 5.x, 6.x, 7.x) if the application provides the corresponding mappings and options. +Other versions of Apache Cassandra like 1.x 2.x, 3.x or 4.1 are not supported as the secondary index interface used by the plugin is different. Other Cassandra vendors are not tested, ScyllaDB is not supported. -| Versions | Elasticsearch 1.x | Elasticsearch 2.x | Elasticsearch 5.x | Elasticsearch 6.x | -|---|---|---|---|---| -| Cassandra 2.x | No | No | No | No | -| Cassandra 3.x | No | No | No | No | -| Cassandra 3.11.x | Limited | Limited | Yes | Yes | -| Cassandra 4.x | No | No | No | No | +| Versions | Elasticsearch 1.x | Elasticsearch 2.x | Elasticsearch 5.x | Elasticsearch 6.x | Elasticsearch 7.x | +|---------------|-------------------|-------------------|-------------------|--------------------|--------------------| +| Cassandra 1.x | No | No | No | No | No | +| Cassandra 2.x | No | No | No | No | No | +| Cassandra 3.x | No | No | No | No | No | +| Cassandra 4.x | Limited | Limited | Limited | Yes | Yes | * **No**: Plugin can't work due to different Cassandra interface. * **Yes**: Plugin works without problem. @@ -91,7 +91,7 @@ This will build a "all in one jar' in `target/distribution/lib4cassandra` com.genesyslab es-index - 9.1.002.00 + 9.2.000.00 ``` See [Github Package](https://github.com/GenesysPureEngagePremise/cassandra-es-index/packages) @@ -99,13 +99,13 @@ See [Github Package](https://github.com/GenesysPureEngagePremise/cassandra-es-in See [Maven repository](https://mvnrepository.com/artifact/com.genesys/es-index/9.1.002.00) ## Installing the plugin in Cassandra -Put `es-index-9.1.000.xx-jar-with-dependencies.jar` in the lib folder of Cassandra along with other Cassandra jars, +Put `es-index-9.2.000.xx-jar-with-dependencies.jar` in the lib folder of Cassandra along with other Cassandra jars, for example '/usr/share/cassandra/lib' on all Cassandra nodes. Start or restart your Cassandra node(s). ## Upgrade of an existing version 1. Stop Cassandra node. -2. Remove old es-index-9.1.001.\-jar-with-dependencies.jar -3. Add new es-index-9.1.001.\-jar-with-dependencies.jar +2. Remove old es-index-9.2.000.\-jar-with-dependencies.jar +3. Add new es-index-9.2.000.\-jar-with-dependencies.jar 4. Start Cassandra node. 5. Proceed to next node. @@ -886,6 +886,10 @@ This is an example of asynchronous write, Cassandra operation will **not** fail ![Write Path async](doc/write-path-async-fail.png) # Changes +## Version 9.2.000 +* Support for Cassandra 4.0.x +* Drop support for Cassandra 3.x + ## Version 9.1.003 * Support for ES 7.x (plugin needs to be upraded before ES, using 7.x with older plugin will not work) * Support for Cassandra 3.11.5 (just testing, older versions will work as well) diff --git a/pom.xml b/pom.xml index c214ebd..c3b88fa 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ com.genesys es-index - 9.1.003.00 + 9.2.000.00 jar es-index @@ -23,7 +23,7 @@ Genesys Telecommunications Laboratories, Inc. - http://www.genesys.com + https://www.genesys.com @@ -44,7 +44,7 @@ Jacques-Henri Berthemet support@genesys.com Genesys - http://genesys.com + https://genesys.com @@ -65,7 +65,7 @@ UTF-8 - 4.0.10 + 4.0.13 5.3.2 2.8.9 3.0.1u2 @@ -87,7 +87,7 @@ - + org.apache.cassandra cassandra-all @@ -95,7 +95,6 @@ ${version.cassandra.esindex} - org.apache.httpcomponents httpcore @@ -109,7 +108,7 @@ ${version.jest} - org.slf4j + org.slf4j slf4j-api @@ -117,7 +116,7 @@ commons-codec - com.google.guava + com.google.guava guava @@ -161,8 +160,8 @@ ${version.mockito} test - + @@ -180,19 +179,6 @@ ${java-source-version} - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - attach-sources - - jar-no-fork - - - - org.apache.maven.plugins maven-assembly-plugin @@ -207,51 +193,13 @@ assemble-all - package + prepare-package single - - org.apache.maven.plugins - maven-javadoc-plugin - 3.1.1 - - - attach-javadocs - - jar - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - - sign-artifacts - verify - - sign - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true - - ossrh - https://oss.sonatype.org/ - true - - org.apache.maven.plugins maven-site-plugin @@ -262,6 +210,20 @@ maven-project-info-reports-plugin ${version.maven.project.info.reports.plugin} - + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + verify + + jar-no-fork + + + + + diff --git a/src/main/java/com/genesyslab/webme/commons/index/CStarUtils.java b/src/main/java/com/genesyslab/webme/commons/index/CStarUtils.java index a633601..f0bd294 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/CStarUtils.java +++ b/src/main/java/com/genesyslab/webme/commons/index/CStarUtils.java @@ -1,25 +1,35 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index; -import com.genesyslab.webme.commons.index.CellElement.CollectionValue; -import com.genesyslab.webme.commons.index.config.IndexConfig; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQL3Type; import org.apache.cassandra.db.ColumnFamilyStore; @@ -55,29 +65,18 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.TimestampSerializer; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import java.io.IOException; -import java.net.InetAddress; -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.nio.charset.StandardCharsets.UTF_8; +import com.genesyslab.webme.commons.index.CellElement.CollectionValue; +import com.genesyslab.webme.commons.index.config.IndexConfig; /** * Created by Jacques-Henri Berthemet on 10/24/2014. Some utils to process Cassandra CFs @@ -88,26 +87,25 @@ public class CStarUtils { * Convert a rowKey to a map of column names and put corresponding values in the map. It includes * partition keys and clustering columns. * - * @param rowKey can't be null + * @param rowKey can't be null * @param tableMetadata can't be null, table metadata, not index metadata * @return never null, left is pkName, right is pkValue */ @Nonnull - static List> getPartitionKeys(@Nonnull ByteBuffer rowKey, @Nonnull CFMetaData tableMetadata) - throws CharacterCodingException { + static List> getPartitionKeys(@Nonnull ByteBuffer rowKey, @Nonnull TableMetadata tableMetadata) + throws CharacterCodingException { List> partitionKeys = new ArrayList<>(1); - List columns = tableMetadata.partitionKeyColumns(); - ColumnDefinition[] pkColDefinitions = columns.toArray(new ColumnDefinition[columns.size()]); + List columns = tableMetadata.partitionKeyColumns(); + ColumnMetadata[] pkColDefinitions = columns.toArray(new ColumnMetadata[columns.size()]); - AbstractType pkValidator = tableMetadata.getKeyValidator(); + AbstractType pkValidator = tableMetadata.partitionKeyType; // PK is composite we need to extract sub-keys if (pkValidator instanceof CompositeType) { - int pos = 0; - for (ByteBuffer key : CompositeType.splitName(rowKey)) { - CompositeType type = (CompositeType) pkValidator; + CompositeType type = (CompositeType) pkValidator; + for (ByteBuffer key : type.split(rowKey)) { String pkName = ByteBufferUtil.string(pkColDefinitions[pos].name.bytes); String pkValue = type.types.get(pos).getString(key); @@ -116,7 +114,7 @@ static List> getPartitionKeys(@Nonnull ByteBuffer rowKey, @ } } else { // PK is a single column - ColumnDefinition pkDefinition = pkColDefinitions[0]; + ColumnMetadata pkDefinition = pkColDefinitions[0]; String pkName = ByteBufferUtil.string(pkDefinition.name.bytes); String pkValue = pkValidator.getString(rowKey); @@ -133,11 +131,11 @@ static List> getPartitionKeys(@Nonnull ByteBuffer rowKey, @ * @param keys not null, not empty */ @Nonnull - static ByteBuffer getPartitionKeys(@Nonnull String[] keys, @Nonnull CFMetaData tableMetadata) { - List columns = tableMetadata.partitionKeyColumns(); - ColumnDefinition[] pkColDefinitions = columns.toArray(new ColumnDefinition[columns.size()]); + static ByteBuffer getPartitionKeys(@Nonnull String[] keys, @Nonnull TableMetadata tableMetadata) { + List columns = tableMetadata.partitionKeyColumns(); + ColumnMetadata[] pkColDefinitions = columns.toArray(new ColumnMetadata[columns.size()]); - AbstractType pkValidator = tableMetadata.getKeyValidator(); + AbstractType pkValidator = tableMetadata.partitionKeyType; // PK is composite we need to extract sub-keys if (pkValidator instanceof CompositeType) { @@ -146,7 +144,7 @@ static ByteBuffer getPartitionKeys(@Nonnull String[] keys, @Nonnull CFMetaData t Object[] objects = new Object[pkColDefinitions.length]; int pos = 0; - for (ColumnDefinition column : columns) { + for (ColumnMetadata column : columns) { if (column.type.asCQL3Type().equals(CQL3Type.Native.INT)) { objects[pos] = Integer.valueOf(keys[pos]); } else { @@ -170,7 +168,7 @@ static ByteBuffer getPartitionKeys(@Nonnull String[] keys, @Nonnull CFMetaData t @Nullable static String cellValueToString(@Nonnull Cell cell) throws IOException { if (cell.isLive(FBUtilities.nowInSeconds())) { - return byteBufferToString(cell.column().type, cell.value()).left; + return byteBufferToString(cell.column().type, cell.buffer()).left; } else { return null; } @@ -180,13 +178,13 @@ static String cellValueToString(@Nonnull Cell cell) throws IOException { * Convert a cell's (single) value to a String according to AbstractType
* * @param abstractType not null - * @param value not null + * @param value not null * @return may be null * @throws IOException if type is unknown */ @Nonnull private static Pair byteBufferToString(@Nonnull AbstractType abstractType, @Nullable ByteBuffer value) - throws IOException { + throws IOException { if (value == null) { return Pair.create(null, Boolean.FALSE); @@ -345,7 +343,7 @@ static CollectionValue getCollectionElement(@Nonnull Cell cell) throws IOExcepti } if (cell.isLive(FBUtilities.nowInSeconds())) { // isLive() is better than isTombstone in case of commitlog replay or hints - Pair pair = byteBufferToString(abstractType, cell.value()); + Pair pair = byteBufferToString(abstractType, cell.buffer()); if (pair.right) { return CollectionValue.create(key, pair.left, CollectionValue.CollectionType.JSON); } else { @@ -370,7 +368,7 @@ static boolean isCollection(@Nullable Cell cell) { * Convert a list of PK + CK to a single line id
* PK-PK-CK-CK-CK * - * @param partitionKeys not null + * @param partitionKeys not null * @param clusteringKeys can be null * @return null if map is empty */ @@ -414,11 +412,11 @@ private static void addKeys(Iterator> keyIterator, StringBu * @return Partition keys */ @Nonnull - static List getPartitionKeyNames(@Nonnull CFMetaData metadata) throws CharacterCodingException { - List partitionKeys = metadata.partitionKeyColumns(); + static List getPartitionKeyNames(@Nonnull TableMetadata metadata) throws CharacterCodingException { + List partitionKeys = metadata.partitionKeyColumns(); List primaryKeys = new ArrayList<>(partitionKeys.size()); - for (ColumnDefinition colDef : partitionKeys) { + for (ColumnMetadata colDef : partitionKeys) { String keyName = ByteBufferUtil.string(colDef.name.bytes); primaryKeys.add(keyName); } @@ -433,11 +431,11 @@ static List getPartitionKeyNames(@Nonnull CFMetaData metadata) throws Ch * @return Clustering keys, can be empty */ @Nonnull - static List getClusteringColumnsNames(@Nonnull CFMetaData metadata) throws CharacterCodingException { - List clusteringColumns = metadata.clusteringColumns(); + static List getClusteringColumnsNames(@Nonnull TableMetadata metadata) throws CharacterCodingException { + List clusteringColumns = metadata.clusteringColumns(); List clusteringColumnsNames = new ArrayList<>(clusteringColumns.size()); - for (ColumnDefinition colDef : clusteringColumns) { + for (ColumnMetadata colDef : clusteringColumns) { String keyName = ByteBufferUtil.string(colDef.name.bytes); clusteringColumnsNames.add(keyName); } @@ -448,14 +446,14 @@ static List getClusteringColumnsNames(@Nonnull CFMetaData metadata) thro /** * Retrieve ClusteringKeys value from cell name * - * @param row not null - * @param tableMetadata not null + * @param row not null + * @param tableMetadata not null * @param clusteringColumnsNames can be null * @return null if ColumnFamily has no collections, a list else */ @Nullable - static List> getClusteringKeys(@Nonnull Row row, @Nonnull CFMetaData tableMetadata, - @Nonnull List clusteringColumnsNames) { + static List> getClusteringKeys(@Nonnull Row row, @Nonnull TableMetadata tableMetadata, + @Nonnull List clusteringColumnsNames) { int clusteringPrefixSize = row.clustering().size(); if (clusteringPrefixSize > 0) { List> keys = new ArrayList<>(clusteringPrefixSize); @@ -463,7 +461,7 @@ static List> getClusteringKeys(@Nonnull Row row, @Nonnull C for (int prefixNb = 0; prefixNb < clusteringPrefixSize; prefixNb++) { String name = clusteringColumnsNames.get(prefixNb); AbstractType subtype = tableMetadata.comparator.subtype(prefixNb); - ByteBuffer clusteringKeyBytes = row.clustering().get(prefixNb); + ByteBuffer clusteringKeyBytes = row.clustering().bufferAt(prefixNb); String value = subtype.getString(clusteringKeyBytes); keys.add(Pair.create(name, value)); } @@ -491,26 +489,29 @@ static String queryString(@Nonnull ReadCommand command) { static boolean isOwner(@Nonnull ColumnFamilyStore cfs, @Nonnull Token token) { // Get all live endpoints which was selected to replicate this data - List addresses = StorageService.instance.getLiveNaturalEndpoints(cfs.keyspace, token); - Map indexers = new HashMap<>(); - - // Build DC-based map - select only single (first) node to index, because getLiveNaturalEndpoints returns same values for all nodes - for (InetAddress address : addresses) { - String datacenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(address); - if (!indexers.containsKey(datacenter)) { - indexers.put(datacenter, address); + List addresses = cfs.keyspace.getReplicationStrategy().getNaturalReplicasForToken(token).endpointList(); + Map indexers = new HashMap<>(); + + // Build DC-based map - select only single (first) node to index, because getLiveNaturalEndpoints + // returns same values for all nodes + for (InetAddressAndPort address : addresses) { + if (FailureDetector.instance.isAlive(address)) { + String datacenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(address); + if (!indexers.containsKey(datacenter)) { + indexers.put(datacenter, address); + } } } - return indexers.containsValue(FBUtilities.getBroadcastAddress()); // Current node is not indexer (not first) + return indexers.containsValue(FBUtilities.getBroadcastAddressAndPort()); // Current node is not indexer (not first) } public static String getLocalDC() { - return DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress()); + return DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); } public static List getDCs() { - Set addresses = StorageService.instance.getTokenMetadata().getAllEndpoints(); + Set addresses = StorageService.instance.getTokenMetadata().getAllEndpoints(); return addresses.stream().map(address -> DatabaseDescriptor.getEndpointSnitch().getDatacenter(address)).distinct() - .collect(Collectors.toList()); + .collect(Collectors.toList()); } } diff --git a/src/main/java/com/genesyslab/webme/commons/index/ElasticIndex.java b/src/main/java/com/genesyslab/webme/commons/index/ElasticIndex.java index 08c310b..b8a8e3f 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/ElasticIndex.java +++ b/src/main/java/com/genesyslab/webme/commons/index/ElasticIndex.java @@ -1,897 +1,897 @@ -/* - * Copyright 2019 Genesys Telecommunications Laboratories, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.genesyslab.webme.commons.index; - -import com.genesyslab.webme.commons.index.config.IndexConfig; -import com.genesyslab.webme.commons.index.monitor.EsJmxBridge; -import com.genesyslab.webme.commons.index.requests.ElasticClientFactory; -import com.genesyslab.webme.commons.index.requests.GenericRequest; -import com.genesyslab.webme.commons.index.requests.ResponseHandler; -import com.genesyslab.webme.commons.index.requests.UpdatePipeline; - -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.exceptions.CassandraException; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.nio.conn.SchemeIOSessionStrategy; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.TrustStrategy; -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.map.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.searchbox.action.Action; -import io.searchbox.client.JestClient; -import io.searchbox.client.JestClientFactory; -import io.searchbox.client.JestResult; -import io.searchbox.client.config.HttpClientConfig; -import io.searchbox.cluster.Health; -import io.searchbox.core.Count; -import io.searchbox.core.CountResult; -import io.searchbox.core.Delete; -import io.searchbox.core.DeleteByQuery; -import io.searchbox.core.DocumentResult; -import io.searchbox.core.Index; -import io.searchbox.core.Search; -import io.searchbox.core.Update; -import io.searchbox.core.Validate; -import io.searchbox.indices.CreateIndex; -import io.searchbox.indices.DeleteIndex; -import io.searchbox.indices.Flush; -import io.searchbox.indices.IndicesExists; -import io.searchbox.indices.aliases.AddAliasMapping; -import io.searchbox.indices.aliases.AliasMapping; -import io.searchbox.indices.aliases.GetAliases; -import io.searchbox.indices.aliases.ModifyAliases; -import io.searchbox.indices.mapping.GetMapping; -import io.searchbox.indices.mapping.PutMapping; -import io.searchbox.indices.settings.UpdateSettings; - -import javax.annotation.Nonnull; -import javax.net.ssl.SSLContext; - -import java.io.IOException; -import java.io.StringWriter; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static com.genesyslab.webme.commons.index.JsonUtils.dotedToStructured; -import static io.searchbox.params.Parameters.EXPLAIN; -import static io.searchbox.params.Parameters.RETRY_ON_CONFLICT; -import static org.json.simple.JSONValue.escape; - -/** - * ES client based on Jest - *

- * Created by Jacques-Henri Berthemet on 05/07/2017. - */ -public class ElasticIndex implements IndexInterface { - - private static final Logger LOGGER = LoggerFactory.getLogger(ElasticIndex.class); - - // ES constants - private static final String ES_HITS = "hits"; - private static final String ES_SOURCE = "_source"; - private static final String ES_ID = "_id"; - private static final String ES_PIPELINE = "pipeline"; - private static final String ES_LOCALHOST = "http://localhost:"; - private static final String ES_CREDENTIALS = "ESCREDENTIALS"; - - // Can be useful to restart a Cassandra node with bad JSON - private static final boolean SKIP_BAD_JSON = Boolean.getBoolean(IndexConfig.ES_CONFIG_PREFIX + "skip-bad-json"); - private static final boolean ENABLE_INDEXATION_DATE = !Boolean.getBoolean(IndexConfig.ES_CONFIG_PREFIX + "disable-index-date"); - private static final long DISCOVERY_FREQ = Long.getLong(IndexConfig.ES_CONFIG_PREFIX + "discovery-frequency", 5); - - // Special fields - private static final String TTL_FIELD = "_cassandraTtl"; - private static final String INDEXATION_DATE = "IndexationDate"; - - // Wrapped queries - private static final String QUERY_WRAPPER_WITH_SIZE = "{\"size\":%d,\"query\":{\"query_string\":{\"query\":\"%s\"}}}"; - private static final String QUERY_WRAPPER = "{\"query\":%s}"; - private static final String QUERY_WRAPPER_WITH_QUOTES = "{\"query\":{\"query_string\":{\"query\":\"%s\"}}}"; - static final String DOC_AS_UPSERT = "{\"doc\":%s,\"doc_as_upsert\":true}"; - private static final String MATCH_ALL = "*"; - private static final String MATCH_LTE = "{\"conflicts\":\"proceed\",\"query\":{\"range\":{\"" + TTL_FIELD + "\":{\"lte\":%d}}}}"; - private static final String JSON_PREFIX = "{"; - - private static EsJmxBridge jmxMon; - - private static String esUserName; - private static String esPassword; - - static { - readEsCredentials(); - } - - - static void readEsCredentials() { - esUserName = null; - esPassword = null; - - String credentialsOrigin = ""; - - String credentials = System.getenv(ES_CREDENTIALS); - if (credentials == null) { - credentials = System.getProperty(ES_CREDENTIALS); - if (credentials != null) { - credentialsOrigin = " from system properties"; - } - } - - if (credentials != null) { - int colon = credentials.indexOf(':'); - if (colon > 0) { - esUserName = credentials.substring(0, colon); - esPassword = credentials.substring(colon + 1); - LOGGER.info("Elasticsearch credentials provided{}", credentialsOrigin); - } else { - LOGGER.info("Elasticsearch credentials{} are incorrect, missing colon", credentialsOrigin); - } - } - } - - final String typeName; - final IndexManager indexManager; - - private final JestClient client; - private final IndexConfig indexConfig; - private final JsonFactory jsonFactory = new JsonFactory(); - private final ObjectMapper mapper = new ObjectMapper(); - private final AtomicBoolean newIndex = new AtomicBoolean(); - private final String pkIncludePattern; - private final List partitionKeysNames; - private final List clusteringColumnsNames; - private final boolean hasClusteringColumns; - - private boolean usePipeline; - private int ttlShift; - private boolean isConcurrentLock; - private Set jsonFlatSerializedFields; - private Set jsonSerializedFields; - private int maxResults; - private boolean isValidateQuery; - private boolean isAsyncWrite; - private boolean insertOnly; - private int httpPort; - private boolean isV6 = true; //v6 or less - - ElasticIndex(@Nonnull IndexConfig indexConfig, @Nonnull String indexName, @Nonnull String tableName, - @Nonnull List partitionKeysNames, @Nonnull List clusteringColumnsNames) throws ConfigurationException { - this.indexConfig = indexConfig; - this.partitionKeysNames = partitionKeysNames; - this.clusteringColumnsNames = clusteringColumnsNames; - this.typeName = tableName; - - this.indexManager = getIndexManager(indexConfig, indexName); - updateIndexConfigOptions(); - - String unicastHosts = indexConfig.getUnicastHosts(); - List esUrls = new ArrayList<>(); - - String defaultSchemeForDiscoveredNodes = "http"; - for (String host : (unicastHosts == null ? ES_LOCALHOST + httpPort : unicastHosts).split(",")) { - if (host.startsWith("https")) { - defaultSchemeForDiscoveredNodes = "https"; - } - host = host.startsWith("http") ? host : "http://" + host; - host = host.substring("http://".length()).contains(":") ? host : host + ":" + httpPort; // also works for https:// - esUrls.add(host); - } - - int timeout = (int) Math.max(DatabaseDescriptor.getWriteRpcTimeout(), DatabaseDescriptor.getReadRpcTimeout()); - int maxCon = DatabaseDescriptor.getConcurrentWriters() + DatabaseDescriptor.getConcurrentReaders(); - - LOGGER.info("Request timeout: {}ms, max connections: {}, discovery: {}m", timeout, maxCon, DISCOVERY_FREQ); - - HttpClientConfig.Builder httpConfigBuilder = new HttpClientConfig.Builder(esUrls) - .multiThreaded(true) - .discoveryEnabled(true) - .discoveryFrequency(DISCOVERY_FREQ, TimeUnit.MINUTES) - .defaultMaxTotalConnectionPerRoute(indexConfig.getMaxTotalConnectionPerRoute()) - .maxTotalConnection(maxCon) - .readTimeout(timeout); // ms - - if (esUserName != null) { - httpConfigBuilder.defaultCredentials(esUserName, esPassword); - } - - if (Boolean.parseBoolean(System.getProperty("genesys-es-trustall", "true"))) { - try { - TrustStrategy trustAll = (x509Certificates, authenticationType) -> true; - SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, trustAll).build(); - SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); - SchemeIOSessionStrategy httpsIOSessionStrategy = new SSLIOSessionStrategy(sslContext, NoopHostnameVerifier.INSTANCE); - - httpConfigBuilder - .defaultSchemeForDiscoveredNodes(defaultSchemeForDiscoveredNodes) - .sslSocketFactory(sslSocketFactory) - .httpsIOSessionStrategy(httpsIOSessionStrategy); - - } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { - LOGGER.warn("While configuring TLS, ", e); - } - } - - JestClientFactory factory = ElasticClientFactory.getJestClientFactory(); - factory.setHttpClientConfig(httpConfigBuilder.build()); - - // Jest should be as good as ES REST client: - // https://www.elastic.co/blog/benchmarking-rest-client-transport-client - this.client = factory.getObject(); - - StringBuilder include = new StringBuilder(); - Iterator pkIterator = partitionKeysNames.iterator(); - while (pkIterator.hasNext()) { - include.append(pkIterator.next()); - if (pkIterator.hasNext()) { - include.append("|"); - } - } - Iterator clkIterator = clusteringColumnsNames.iterator(); - while (clkIterator.hasNext()) { - include.append(clkIterator.next()); - if (clkIterator.hasNext()) { - include.append("|"); - } - } - - this.pkIncludePattern = include.toString(); - this.hasClusteringColumns = !clusteringColumnsNames.isEmpty(); - } - - private IndexManager getIndexManager(@Nonnull IndexConfig indexConfig, String indexName) { - IndexManager result; - String className = indexConfig.getIndexManagerName(); - try { - Class clazz = Class.forName(className); - Constructor ctor = clazz.getConstructor(getClass(), IndexConfig.class, String.class); - result = (IndexManager) ctor.newInstance(this, indexConfig, indexName); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - String msg = "Index management " + className + " initialization failed"; - LOGGER.error(msg, e); - throw new ConfigurationException(msg, e); - } - return result; - } - - @Override - public void init() throws ConfigurationException { - try { - if (jmxMon == null) { - jmxMon = new EsJmxBridge(client); - } else { - LOGGER.debug("ES JMX bridge already registered"); - } - } catch (Exception e) { - LOGGER.error("Can't initialize the ES JMX bridge", e); - if (!EsSecondaryIndex.START_WITH_FAILED_INDEX) { - throw new ConfigurationException("Can't initialize the ES JMX bridge", e); - } - } - - LOGGER.info("ElasticIndex '{}' type '{}' initialization", indexManager.getAliasName(), typeName); - - // We now wait for the yellow (or green) status - JestResult res = execute(new Health.Builder().waitForStatus(Health.Status.YELLOW).build()).waitForSuccess(); - LOGGER.debug("Got cluster status: {}", res.getJsonString()); - - JsonObject version = execute(new GenericRequest("GET", "/", null)).waitForSuccess().getJsonObject().getAsJsonObject("version"); - String number = version.get("number").getAsString(); - LOGGER.info("Connected to Elasticsearch version {}", number); - isV6 = Integer.parseInt(number.substring(0, 1)) < 7; - - setupIndex(indexManager.getCurrentName()); // Will create the ES index if needed - - LOGGER.debug("ElasticIndex '{}/{}' initialized, pipeline:{}", indexManager.getAliasName(), indexManager.getCurrentName(), usePipeline); - } - - /** - * Setup the index, either create new one or update existing - * - * @param indexName indexName to create - * @throws ConfigurationException if create index failed - */ - void setupIndex(String indexName) throws ConfigurationException { - JsonObject indexProperties = indexConfig.getProperties(); - LOGGER.debug("Configuring {}/{} with settings:{}", indexManager.getAliasName(), indexName, indexProperties); - - indexProperties = JsonUtils.filterKeys(indexProperties, IndexConfig.KNOWN_LEGACY_SETTINGS); - indexProperties = JsonUtils.filterKeys(indexProperties, IndexConfig.SETTINGS_TO_SKIP); - indexProperties.entrySet().removeIf(elem -> elem.getKey().endsWith(IndexConfig.ES_UNICAST_HOSTS)); - - if ( - indexProperties.get(IndexConfig.ES_TRANSLOG) == null) { // https://intranet.genesys.com/pages/viewpage.action?pageId=63998861 - indexProperties.addProperty(IndexConfig.ES_TRANSLOG, IndexConfig.ES_TRANSLOG_ASYNC); - } - - //ES7 don't support dotted props anymore and requires them in the settings object - if (!isV6 && !indexProperties.has("settings")) { - LOGGER.warn("Received dotted index properties, converting to structured json compatible with ES v7"); - JsonObject settings = dotedToStructured(indexProperties); - indexProperties = new JsonObject(); - indexProperties.add("settings", settings); - } - - boolean indexExists = execute(new IndicesExists.Builder(indexName).build()).waitForResult().isSucceeded(); - if (indexExists) { - LOGGER.warn("Index '{}' already exists, updating.", indexName); - setupTypeMapping(indexName); - setupPipelines(); - - if (indexProperties.size() == 0) { - LOGGER.debug("Index '{}' has no custom setting to apply", indexName); - return; - } - - JsonObject updatableProperties = JsonUtils.filter(indexProperties, IndexConfig.UPDATABLE_SETTINGS::contains); - - if (updatableProperties.size() == 0) { - LOGGER.debug("No settings to update"); - } else { - LOGGER.info("Applying updatable settings from cfg {}", updatableProperties); - JestResult res = execute(new UpdateSettings.Builder(updatableProperties).addIndex(indexName).build()).waitForSuccess(); - LOGGER.info("Index settings update result is: {}", res.isSucceeded()); - } - } else { - LOGGER.warn("Index '{}' does not exist, creating...", indexName); - - CreateIndex.Builder createIndex = new CreateIndex.Builder(indexName); - createIndex.settings(indexProperties.toString()); - - JestResult createIndexResult = execute(createIndex.build()).waitForResult(); - boolean success = createIndexResult.isSucceeded(); - LOGGER.warn("Index creation result is: {}", success); - - if (success) { - putAlias(indexName, indexManager.getAliasName()); - setupTypeMapping(indexName); - setupPipelines(); - - newIndex.set(true); // automatic rebuild support - } else { - if (execute(new IndicesExists.Builder(indexName).build()).waitForResult().isSucceeded()) { - LOGGER.warn("Creation of index '{}' failed, but it exists now, it was created and configured by another node, proceeding...", - indexName); - } else { - LOGGER.error("Failed to create the index '{}' {}", indexName, createIndexResult.getJsonString()); - throw new ConfigurationException(createIndexResult.getErrorMessage()); - } - } - } - } - - /** - * Create pipelines
- * Instead mapping transform we can use pipeline: - * https://www.elastic.co/guide/en/elasticsearch/reference/5.0/breaking_50_mapping_changes.html#_source_transform_removed - * We can define a pipeline for every type, and when we make insert we will the pipeline if the - * pipeline is defined for this type - */ - private void setupPipelines() { - indexConfig.getPipelines().stream().filter(StringUtils::isNotBlank) // Check null or empty - .filter(type -> StringUtils.isNotBlank(indexConfig.getPipeline(type))) // Check pipeline definition exists - .forEach(type -> { - execute(new UpdatePipeline.Builder(type, indexConfig.getPipeline(type)).build()).waitForSuccess(); - LOGGER.debug("Pipeline created for '{}'", type); - }); - } - - /** - * Update the type mapping of an existing index - */ - private void setupTypeMapping(String indexName) throws ConfigurationException { - String mapping = indexConfig.getTypeMapping(typeName); - - if (StringUtils.isNotBlank(mapping)) { - LOGGER.debug("Updating type mapping for '{}' to:\n\t{}", typeName, mapping); - putMapping(indexName, mapping); - } - } - - @Override - public SearchResult putMapping(String index, String source) { - // We put the new getMapping on current index, not the alias - PutMapping.Builder put = new PutMapping.Builder(index, typeName, source); - if (!isV6) { - put.setParameter("include_type_name", "true"); - } - JestResult result = execute(put.build()).waitForSuccess(); - return new SearchResult(new ArrayList<>(), result.getJsonObject()); - } - - public SearchResult getMapping(String index) { - JestResult result = execute(new GetMapping.Builder().addIndex(index).build()).waitForResult(); - return new SearchResult(new ArrayList<>(), result.getJsonObject()); - } - - @Override - public void index(@Nonnull List> partitionKeys, @Nonnull List elements, long expirationTime, - boolean isInsert) throws IOException { - if (isConcurrentLock) { - // lock on intern representation of type+PK, for example: "Interaction[(Id,0001HZO1Qq0haiGs)]" - // This prevents concurrent upserts on the same doc from the same node - synchronized ((typeName + partitionKeys).intern()) { - indexInternal(partitionKeys, elements, expirationTime); - } - } else { - indexInternal(partitionKeys, elements, expirationTime); - } - } - - private void indexInternal(List> partitionKeys, List elements, long expirationTime) - throws IOException { - - Map> groupedMap = group(partitionKeys, elements); - - for (Map.Entry> entry : groupedMap.entrySet()) { - update(partitionKeys, entry.getKey(), entry.getValue(), expirationTime); - } - } - - private void update(List> partitionKeys, String docId, List elements, long expirationTime) - throws IOException { - - StringWriter stringWriter = new StringWriter(); - - try (JsonGenerator builder = jsonFactory.createJsonGenerator(stringWriter)) { - builder.writeStartObject(); - - for (Pair pk : partitionKeys) { - builder.writeStringField(pk.left, pk.right); - } - - boolean clusteringKeysSet = false; - - Map> collections = null; - - // Fill simple fields and map complex types - for (CellElement element : elements) { - - if (element.clusteringKeys != null) { - if (!clusteringKeysSet) { // Insert clustering keys if not already done - for (Pair key : element.clusteringKeys) { - builder.writeStringField(key.left, key.right); - } - - clusteringKeysSet = true; - } - } - - if (element.isCollection() && element.collectionValue != null) { - if (collections == null) { - collections = new HashMap<>(); - } - - collections.computeIfAbsent(element, k -> new HashMap<>()).put(element.collectionValue.name, element.collectionValue.value); - - } else if (element.value != null) { - try { - if (jsonFlatSerializedFields.contains(element.name)) { - String flattenedJson = JsonUtils.flatten(element.value); - builder.writeFieldName(element.name); - builder.writeRawValue(flattenedJson); - } else if (jsonSerializedFields.contains(element.name)) { - builder.writeFieldName(element.name); - builder.writeRawValue(element.value); - } else { // Simple field - builder.writeStringField(element.name, element.value); - } - } catch (IOException ex) { - if (SKIP_BAD_JSON) { - LOGGER.warn("Skipped bad json for field {} of document {}", element.name, docId, ex); - } else { - throw ex; - } - } - } else { // null value - builder.writeNullField(element.name); - } - } - - if (collections != null) { - // Fill the collections now that they are sorted - for (Map.Entry> collection : collections.entrySet()) { - CellElement element = collection.getKey(); - - if (element.collectionValue != null) { - switch (element.collectionValue.type) { - case JSON: - builder.writeObjectFieldStart(element.name); - - for (Map.Entry en : collection.getValue().entrySet()) { - String value = en.getValue(); - if (value == null) { - builder.writeNullField(en.getKey()); - } else { - builder.writeFieldName(en.getKey()); - builder.writeRawValue(value); - } - } - builder.writeEndObject(); - break; - - case MAP: - builder.writeObjectFieldStart(element.name); - for (Map.Entry entry : collection.getValue().entrySet()) { - builder.writeStringField(entry.getKey(), entry.getValue()); - } - builder.writeEndObject(); - break; - - case LIST: - case SET: - builder.writeArrayFieldStart(element.name); - for (String value : collection.getValue().keySet()) { - builder.writeString(value); - } - builder.writeEndArray(); - break; - - default: - // There is no other CollectionType - } - } - } - } - - if (ENABLE_INDEXATION_DATE) { - builder.writeStringField(INDEXATION_DATE, JsonUtils.getIso8601Date(new Date())); - } - - if (indexManager.isTTLFieldRequired()) { - builder.writeNumberField(TTL_FIELD, expirationTime); - } - - builder.writeEndObject(); - builder.close(); // calling close() early because we want the output now - String jsonDoc = stringWriter.toString(); - - if (EsSecondaryIndex.DEBUG_SHOW_VALUES) { - String operation = (insertOnly || usePipeline) ? "insert" : "upsert"; - LOGGER.debug("Document {} index {} {} with content {}", typeName, operation, docId, jsonDoc); - } - - String currentName = indexManager.getCurrentName(); - ResponseHandler handler; - if (insertOnly || usePipeline) { - Index.Builder indexRequest = new Index.Builder(jsonDoc).index(currentName).type(typeName).id(docId); - - if (usePipeline) { // https://www.elastic.co/guide/en/elasticsearch/reference/5.5/ingest.html - indexRequest.setParameter(ES_PIPELINE, typeName); - } - handler = execute(indexRequest.build()); - - } else { - // Pipelines can only be used with index or bulk - Update.Builder update = new Update.Builder(String.format(DOC_AS_UPSERT, jsonDoc)) - .index(currentName) - .type(typeName) - .id(docId); - - if (indexConfig.getRetryOnConflict() > -1) { - update.setParameter(RETRY_ON_CONFLICT, indexConfig.getRetryOnConflict()); - } - - handler = execute(update.build()); - } - - if (!isAsyncWrite) { - handler.waitForSuccess(); // Will block until response anc ensure result is a success - } - } - } - - /** - * Group all CellElement according to their clustering keys - * - * @param partitionKeys not null - * @param elements not null - * @return a grouped map - */ - private Map> group(List> partitionKeys, List elements) { - Map> sortedCells = new HashMap<>(); - - for (CellElement element : elements) { - String docId = CStarUtils.toEsId(partitionKeys, element.clusteringKeys); - sortedCells.computeIfAbsent(docId, k -> new ArrayList<>()).add(element); - } - - return sortedCells; - } - - @Override - public void delete(@Nonnull List> partitionKeys) { - String docId = CStarUtils.toEsId(partitionKeys, null); - String currentName = indexManager.getCurrentName(); - ResponseHandler handler = execute(new Delete.Builder(docId).index(currentName).type(typeName).build()); - if (!isAsyncWrite) { - handler.waitForStatus(200, 404, 204); // Blocks until response. Does not ensure result is a success. - } - } - - @Override - public Object flush() { - return execute(new Flush.Builder().addIndex(indexManager.getCurrentName()).force(true).build()).waitForSuccess(); - } - - @Override - @Nonnull - public SearchResult search(@Nonnull QueryMetaData queryMetaData) { - String queryString = queryMetaData.query; - - LOGGER.trace("Index {} search with query {}", typeName, queryString); - - if (!queryString.startsWith(JSON_PREFIX)) { - queryString = String.format(QUERY_WRAPPER_WITH_SIZE, maxResults, escape(queryString)); - } - - Search.Builder builder = new Search.Builder(queryString).addIndex(indexManager.getAliasName()).addType(typeName); - - io.searchbox.core.SearchResult searchResponse; - try { - Search searchRequest = queryMetaData.loadSource() ? builder.build() : builder.addSourceIncludePattern(pkIncludePattern).build(); - searchResponse = execute(searchRequest).waitForSuccess(); - } catch (CassandraException e) { - throw new InvalidRequestException(e.getMessage()); - } - - LOGGER.trace("Index {} search result: {}", typeName, searchResponse); - - final List idList = new ArrayList<>(); - - JsonElement hits = JsonUtils.getJsonObject(searchResponse, ES_HITS).get(ES_HITS); - if (hits != null) { - List primaryKeys; - - if (hasClusteringColumns) { - primaryKeys = new ArrayList<>(partitionKeysNames.size() + clusteringColumnsNames.size()); - primaryKeys.addAll(partitionKeysNames); - primaryKeys.addAll(clusteringColumnsNames); - } else { - primaryKeys = partitionKeysNames; - } - - int pkSize = primaryKeys.size(); - - hits.getAsJsonArray().forEach(result -> { - String[] primaryKey = new String[pkSize]; - - int keyNb = 0; - - for (String keyName : primaryKeys) { - String value = JsonUtils.getString(result, ES_SOURCE, keyName); - - if (value == null) { - LOGGER.warn("Missing pk {} from ES results, skipping hit:{}", keyName, JsonUtils.getString(result, ES_ID)); - continue; - } else { - primaryKey[keyNb] = value; - } - keyNb++; - } - - if (keyNb == pkSize) { // Will only be false if we skipped a hit, see above warning - idList.add(new SearchResultRow(primaryKey, result.getAsJsonObject())); - } - - }); - } - - // Remove the content of {"hits":{"hits": (big values) } } - JsonObject metadata = JsonUtils.filterPath(searchResponse.getJsonObject(), ES_HITS, ES_HITS); - return new SearchResult(idList, metadata); - } - - private String extractQuery(String query) { - try { - return mapper.readTree(query).get("query").toString(); - } catch (Exception e) { - LOGGER.trace("Could not extract query node from '{}' for Index {}", query, indexManager.getAliasName()); - } - return query; - } - - @Override - public void validate(@Nonnull String query) throws InvalidRequestException { - if (!isValidateQuery) { - return; - } - LOGGER.trace("Validating query {}", query); - - String esQuery = query; - - // Ignore index management queries like #update# .... # - if (query.startsWith("#")) { - if (query.endsWith("#")) { //WCC-886 - return; - } else { - int optionEnd = query.indexOf("#", 1); - if (optionEnd < 0) { - throw new InvalidRequestException("Query starts with '#', but second '#' is missing"); - } - esQuery = query.substring(optionEnd + 1); - } - } - - String formattedQuery; - if (esQuery.startsWith(JSON_PREFIX)) { - formattedQuery = String.format(QUERY_WRAPPER, extractQuery(esQuery)); // WCC-876 - } else { - formattedQuery = String.format(QUERY_WRAPPER_WITH_QUOTES, esQuery); - } - - LOGGER.trace("Validating query {}", formattedQuery); - try { - Validate.Builder validateBuilder = new Validate.Builder(formattedQuery); - validateBuilder.setParameter(EXPLAIN, String.valueOf(true)); - JestResult res = execute(validateBuilder.build()).waitForResult(); - if (!res.isSucceeded()) { - LOGGER.info("Query {} is invalid", formattedQuery); - throw new InvalidRequestException(res.getErrorMessage()); - } else { - String valid = res.getJsonObject().get("valid").getAsString(); - if (Boolean.parseBoolean(valid)) { - LOGGER.trace("Query {} is valid", formattedQuery); - } else { - throw new InvalidRequestException(res.getJsonObject().toString()); - } - } - - } catch (Exception e) { - throw new InvalidRequestException(e.getMessage()); - } - } - - @Override - public void settingsUpdated() { - indexManager.checkForUpdate(); - setupIndex(indexManager.getCurrentName()); - } - - @Override - public boolean isNewIndex() { - return newIndex.getAndSet(false); - } - - @Nonnull - private ResponseHandler execute(Action request) { - ResponseHandler handler = new ResponseHandler<>(typeName, request); - client.executeAsync(request, handler); - return handler; - } - - @Override - public Object drop() { - if (indexConfig.isPerIndexType()) { - String indexName = indexManager.getCurrentName(); - LOGGER.warn("Index {}/{} is being dropped, stopping purge task, deleting ES index", indexName, typeName); - indexManager.stop(); - - JestResult res = execute(new Delete.Builder("").index(indexName).build()).waitForResult(); - return res.isSucceeded(); - } else { - return truncate(); - } - } - - @Override - public Object truncate() { - String aliasName = indexManager.getAliasName(); - LOGGER.warn("Index {}/{} is being truncated, deleting documents", aliasName, typeName); - JestResult res = execute(new Delete.Builder(MATCH_ALL).index(aliasName).type(typeName).build()).waitForResult(); - return res.isSucceeded(); - } - - @Override - public void deleteExpired() { - String aliasName = indexManager.getAliasName(); - long ttl = FBUtilities.nowInSeconds() + ttlShift; - - DeleteByQuery deleteQuery = new DeleteByQuery.Builder(String.format(MATCH_LTE, ttl)).addIndex(aliasName).addType(typeName).build(); - JestResult res = execute(deleteQuery).waitForSuccess(); - - Long deleted = JsonUtils.getLong(res.getJsonObject(), "deleted"); - if (deleted != null && deleted > 0) { - LOGGER.debug("Index {} deleted {} documents where _cassandraTtl < {}", indexManager.getAliasName(), deleted, ttl); - } - } - - @Override - public void purgeEmptyIndexes() { - String aliasName = indexManager.getAliasName(); - LOGGER.debug("Start segmented index cleanup for {}", aliasName); - JestResult aliasesResponses = execute(new GetAliases.Builder().addIndex(aliasName).build()).waitForResult(); - - if (aliasesResponses.isSucceeded()) { - JsonUtils.getJsonObject(aliasesResponses, aliasName, "aliases").entrySet().forEach(alias -> { - String indexToDelete = alias.getKey(); - CountResult count = execute(new Count.Builder().addIndex(indexToDelete).build()).waitForResult(); - - if (count.isSucceeded() && count.getCount().intValue() == 0) { - dropIndex(indexToDelete); - } else { - LOGGER.debug("Index {} is not empty", indexToDelete); - } - }); - } - } - - @Override - public void updateIndexConfigOptions() { - ttlShift = indexConfig.getTtlShift(); - isConcurrentLock = indexConfig.isConcurrentLock(); - jsonFlatSerializedFields = indexConfig.getJsonFlatSerializedFields(); - jsonSerializedFields = indexConfig.getJsonSerializedFields(); - maxResults = indexConfig.getMaxResults(); - isValidateQuery = indexConfig.isValidateQuery(); - isAsyncWrite = indexConfig.isAsyncWrite(); - insertOnly = indexConfig.isInsertOnly(); - usePipeline = StringUtils.isNotBlank(indexConfig.getPipeline(typeName)); - httpPort = indexConfig.getHttpPort(); - indexManager.updateOptions(); - } - - @Override - public List getIndexNames() { - List result = new LinkedList<>(); - JestResult res = execute(new GetAliases.Builder().addIndex(indexManager.getAliasName()).build()).waitForResult(); - Set> set = res.getJsonObject().entrySet(); - for (Map.Entry entry : set) { - result.add(entry.getKey()); - } - return result; - } - - @Override - public void dropIndex(String indexName) { - LOGGER.info("Deleting index {}", indexName); - boolean success = execute(new DeleteIndex.Builder(indexName).build()).waitForResult().isSucceeded(); - LOGGER.info("Index {} deletion {}", indexName, success ? "successful" : "failed"); - } - - private void putAlias(String indexName, String alias) { - LOGGER.warn("Creating index alias '{}'", indexManager.getAliasName()); - AliasMapping aliases = new AddAliasMapping.Builder(indexName, alias).build(); - JestResult addAliasResult = execute(new ModifyAliases.Builder(aliases).build()).waitForResult(); - LOGGER.warn("Index alias creation result is: {}", addAliasResult.isSucceeded()); - } -} +/* + * Copyright 2019 Genesys Telecommunications Laboratories, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.genesyslab.webme.commons.index; + +import com.genesyslab.webme.commons.index.config.IndexConfig; +import com.genesyslab.webme.commons.index.monitor.EsJmxBridge; +import com.genesyslab.webme.commons.index.requests.ElasticClientFactory; +import com.genesyslab.webme.commons.index.requests.GenericRequest; +import com.genesyslab.webme.commons.index.requests.ResponseHandler; +import com.genesyslab.webme.commons.index.requests.UpdatePipeline; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.exceptions.CassandraException; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.nio.conn.SchemeIOSessionStrategy; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.TrustStrategy; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.searchbox.action.Action; +import io.searchbox.client.JestClient; +import io.searchbox.client.JestClientFactory; +import io.searchbox.client.JestResult; +import io.searchbox.client.config.HttpClientConfig; +import io.searchbox.cluster.Health; +import io.searchbox.core.Count; +import io.searchbox.core.CountResult; +import io.searchbox.core.Delete; +import io.searchbox.core.DeleteByQuery; +import io.searchbox.core.DocumentResult; +import io.searchbox.core.Index; +import io.searchbox.core.Search; +import io.searchbox.core.Update; +import io.searchbox.core.Validate; +import io.searchbox.indices.CreateIndex; +import io.searchbox.indices.DeleteIndex; +import io.searchbox.indices.Flush; +import io.searchbox.indices.IndicesExists; +import io.searchbox.indices.aliases.AddAliasMapping; +import io.searchbox.indices.aliases.AliasMapping; +import io.searchbox.indices.aliases.GetAliases; +import io.searchbox.indices.aliases.ModifyAliases; +import io.searchbox.indices.mapping.GetMapping; +import io.searchbox.indices.mapping.PutMapping; +import io.searchbox.indices.settings.UpdateSettings; + +import javax.annotation.Nonnull; +import javax.net.ssl.SSLContext; + +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.genesyslab.webme.commons.index.JsonUtils.dotedToStructured; +import static io.searchbox.params.Parameters.EXPLAIN; +import static io.searchbox.params.Parameters.RETRY_ON_CONFLICT; +import static org.json.simple.JSONValue.escape; + +/** + * ES client based on Jest + *

+ * Created by Jacques-Henri Berthemet on 05/07/2017. + */ +public class ElasticIndex implements IndexInterface { + + private static final Logger LOGGER = LoggerFactory.getLogger(ElasticIndex.class); + + // ES constants + private static final String ES_HITS = "hits"; + private static final String ES_SOURCE = "_source"; + private static final String ES_ID = "_id"; + private static final String ES_PIPELINE = "pipeline"; + private static final String ES_LOCALHOST = "http://localhost:"; + private static final String ES_CREDENTIALS = "ESCREDENTIALS"; + + // Can be useful to restart a Cassandra node with bad JSON + private static final boolean SKIP_BAD_JSON = Boolean.getBoolean(IndexConfig.ES_CONFIG_PREFIX + "skip-bad-json"); + private static final boolean ENABLE_INDEXATION_DATE = !Boolean.getBoolean(IndexConfig.ES_CONFIG_PREFIX + "disable-index-date"); + private static final long DISCOVERY_FREQ = Long.getLong(IndexConfig.ES_CONFIG_PREFIX + "discovery-frequency", 5); + + // Special fields + private static final String TTL_FIELD = "_cassandraTtl"; + private static final String INDEXATION_DATE = "IndexationDate"; + + // Wrapped queries + private static final String QUERY_WRAPPER_WITH_SIZE = "{\"size\":%d,\"query\":{\"query_string\":{\"query\":\"%s\"}}}"; + private static final String QUERY_WRAPPER = "{\"query\":%s}"; + private static final String QUERY_WRAPPER_WITH_QUOTES = "{\"query\":{\"query_string\":{\"query\":\"%s\"}}}"; + static final String DOC_AS_UPSERT = "{\"doc\":%s,\"doc_as_upsert\":true}"; + private static final String MATCH_ALL = "*"; + private static final String MATCH_LTE = "{\"conflicts\":\"proceed\",\"query\":{\"range\":{\"" + TTL_FIELD + "\":{\"lte\":%d}}}}"; + private static final String JSON_PREFIX = "{"; + + private static EsJmxBridge jmxMon; + + private static String esUserName; + private static String esPassword; + + static { + readEsCredentials(); + } + + + static void readEsCredentials() { + esUserName = null; + esPassword = null; + + String credentialsOrigin = ""; + + String credentials = System.getenv(ES_CREDENTIALS); + if (credentials == null) { + credentials = System.getProperty(ES_CREDENTIALS); + if (credentials != null) { + credentialsOrigin = " from system properties"; + } + } + + if (credentials != null) { + int colon = credentials.indexOf(':'); + if (colon > 0) { + esUserName = credentials.substring(0, colon); + esPassword = credentials.substring(colon + 1); + LOGGER.info("Elasticsearch credentials provided{}", credentialsOrigin); + } else { + LOGGER.info("Elasticsearch credentials{} are incorrect, missing colon", credentialsOrigin); + } + } + } + + final String typeName; + final IndexManager indexManager; + + private final JestClient client; + private final IndexConfig indexConfig; + private final JsonFactory jsonFactory = new JsonFactory(); + private final ObjectMapper mapper = new ObjectMapper(); + private final AtomicBoolean newIndex = new AtomicBoolean(); + private final String pkIncludePattern; + private final List partitionKeysNames; + private final List clusteringColumnsNames; + private final boolean hasClusteringColumns; + + private boolean usePipeline; + private int ttlShift; + private boolean isConcurrentLock; + private Set jsonFlatSerializedFields; + private Set jsonSerializedFields; + private int maxResults; + private boolean isValidateQuery; + private boolean isAsyncWrite; + private boolean insertOnly; + private int httpPort; + private boolean isV6 = true; //v6 or less + + ElasticIndex(@Nonnull IndexConfig indexConfig, @Nonnull String indexName, @Nonnull String tableName, + @Nonnull List partitionKeysNames, @Nonnull List clusteringColumnsNames) throws ConfigurationException { + this.indexConfig = indexConfig; + this.partitionKeysNames = partitionKeysNames; + this.clusteringColumnsNames = clusteringColumnsNames; + this.typeName = tableName; + + this.indexManager = getIndexManager(indexConfig, indexName); + updateIndexConfigOptions(); + + String unicastHosts = indexConfig.getUnicastHosts(); + List esUrls = new ArrayList<>(); + + String defaultSchemeForDiscoveredNodes = "http"; + for (String host : (unicastHosts == null ? ES_LOCALHOST + httpPort : unicastHosts).split(",")) { + if (host.startsWith("https")) { + defaultSchemeForDiscoveredNodes = "https"; + } + host = host.startsWith("http") ? host : "http://" + host; + host = host.substring("http://".length()).contains(":") ? host : host + ":" + httpPort; // also works for https:// + esUrls.add(host); + } + + int timeout = (int) Math.max(DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MILLISECONDS), DatabaseDescriptor.getReadRpcTimeout(TimeUnit.MILLISECONDS)); + int maxCon = DatabaseDescriptor.getConcurrentWriters() + DatabaseDescriptor.getConcurrentReaders(); + + LOGGER.info("Request timeout: {}ms, max connections: {}, discovery: {}m", timeout, maxCon, DISCOVERY_FREQ); + + HttpClientConfig.Builder httpConfigBuilder = new HttpClientConfig.Builder(esUrls) + .multiThreaded(true) + .discoveryEnabled(true) + .discoveryFrequency(DISCOVERY_FREQ, TimeUnit.MINUTES) + .defaultMaxTotalConnectionPerRoute(indexConfig.getMaxTotalConnectionPerRoute()) + .maxTotalConnection(maxCon) + .readTimeout(timeout); // ms + + if (esUserName != null) { + httpConfigBuilder.defaultCredentials(esUserName, esPassword); + } + + if (Boolean.parseBoolean(System.getProperty("genesys-es-trustall", "true"))) { + try { + TrustStrategy trustAll = (x509Certificates, authenticationType) -> true; + SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, trustAll).build(); + SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + SchemeIOSessionStrategy httpsIOSessionStrategy = new SSLIOSessionStrategy(sslContext, NoopHostnameVerifier.INSTANCE); + + httpConfigBuilder + .defaultSchemeForDiscoveredNodes(defaultSchemeForDiscoveredNodes) + .sslSocketFactory(sslSocketFactory) + .httpsIOSessionStrategy(httpsIOSessionStrategy); + + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + LOGGER.warn("While configuring TLS, ", e); + } + } + + JestClientFactory factory = ElasticClientFactory.getJestClientFactory(); + factory.setHttpClientConfig(httpConfigBuilder.build()); + + // Jest should be as good as ES REST client: + // https://www.elastic.co/blog/benchmarking-rest-client-transport-client + this.client = factory.getObject(); + + StringBuilder include = new StringBuilder(); + Iterator pkIterator = partitionKeysNames.iterator(); + while (pkIterator.hasNext()) { + include.append(pkIterator.next()); + if (pkIterator.hasNext()) { + include.append("|"); + } + } + Iterator clkIterator = clusteringColumnsNames.iterator(); + while (clkIterator.hasNext()) { + include.append(clkIterator.next()); + if (clkIterator.hasNext()) { + include.append("|"); + } + } + + this.pkIncludePattern = include.toString(); + this.hasClusteringColumns = !clusteringColumnsNames.isEmpty(); + } + + private IndexManager getIndexManager(@Nonnull IndexConfig indexConfig, String indexName) { + IndexManager result; + String className = indexConfig.getIndexManagerName(); + try { + Class clazz = Class.forName(className); + Constructor ctor = clazz.getConstructor(getClass(), IndexConfig.class, String.class); + result = (IndexManager) ctor.newInstance(this, indexConfig, indexName); + } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + String msg = "Index management " + className + " initialization failed"; + LOGGER.error(msg, e); + throw new ConfigurationException(msg, e); + } + return result; + } + + @Override + public void init() throws ConfigurationException { + try { + if (jmxMon == null) { + jmxMon = new EsJmxBridge(client); + } else { + LOGGER.debug("ES JMX bridge already registered"); + } + } catch (Exception e) { + LOGGER.error("Can't initialize the ES JMX bridge", e); + if (!EsSecondaryIndex.START_WITH_FAILED_INDEX) { + throw new ConfigurationException("Can't initialize the ES JMX bridge", e); + } + } + + LOGGER.info("ElasticIndex '{}' type '{}' initialization", indexManager.getAliasName(), typeName); + + // We now wait for the yellow (or green) status + JestResult res = execute(new Health.Builder().waitForStatus(Health.Status.YELLOW).build()).waitForSuccess(); + LOGGER.debug("Got cluster status: {}", res.getJsonString()); + + JsonObject version = execute(new GenericRequest("GET", "/", null)).waitForSuccess().getJsonObject().getAsJsonObject("version"); + String number = version.get("number").getAsString(); + LOGGER.info("Connected to Elasticsearch version {}", number); + isV6 = Integer.parseInt(number.substring(0, 1)) < 7; + + setupIndex(indexManager.getCurrentName()); // Will create the ES index if needed + + LOGGER.debug("ElasticIndex '{}/{}' initialized, pipeline:{}", indexManager.getAliasName(), indexManager.getCurrentName(), usePipeline); + } + + /** + * Setup the index, either create new one or update existing + * + * @param indexName indexName to create + * @throws ConfigurationException if create index failed + */ + void setupIndex(String indexName) throws ConfigurationException { + JsonObject indexProperties = indexConfig.getProperties(); + LOGGER.debug("Configuring {}/{} with settings:{}", indexManager.getAliasName(), indexName, indexProperties); + + indexProperties = JsonUtils.filterKeys(indexProperties, IndexConfig.KNOWN_LEGACY_SETTINGS); + indexProperties = JsonUtils.filterKeys(indexProperties, IndexConfig.SETTINGS_TO_SKIP); + indexProperties.entrySet().removeIf(elem -> elem.getKey().endsWith(IndexConfig.ES_UNICAST_HOSTS)); + + if ( + indexProperties.get(IndexConfig.ES_TRANSLOG) == null) { // https://intranet.genesys.com/pages/viewpage.action?pageId=63998861 + indexProperties.addProperty(IndexConfig.ES_TRANSLOG, IndexConfig.ES_TRANSLOG_ASYNC); + } + + //ES7 don't support dotted props anymore and requires them in the settings object + if (!isV6 && !indexProperties.has("settings")) { + LOGGER.warn("Received dotted index properties, converting to structured json compatible with ES v7"); + JsonObject settings = dotedToStructured(indexProperties); + indexProperties = new JsonObject(); + indexProperties.add("settings", settings); + } + + boolean indexExists = execute(new IndicesExists.Builder(indexName).build()).waitForResult().isSucceeded(); + if (indexExists) { + LOGGER.warn("Index '{}' already exists, updating.", indexName); + setupTypeMapping(indexName); + setupPipelines(); + + if (indexProperties.size() == 0) { + LOGGER.debug("Index '{}' has no custom setting to apply", indexName); + return; + } + + JsonObject updatableProperties = JsonUtils.filter(indexProperties, IndexConfig.UPDATABLE_SETTINGS::contains); + + if (updatableProperties.size() == 0) { + LOGGER.debug("No settings to update"); + } else { + LOGGER.info("Applying updatable settings from cfg {}", updatableProperties); + JestResult res = execute(new UpdateSettings.Builder(updatableProperties).addIndex(indexName).build()).waitForSuccess(); + LOGGER.info("Index settings update result is: {}", res.isSucceeded()); + } + } else { + LOGGER.warn("Index '{}' does not exist, creating...", indexName); + + CreateIndex.Builder createIndex = new CreateIndex.Builder(indexName); + createIndex.settings(indexProperties.toString()); + + JestResult createIndexResult = execute(createIndex.build()).waitForResult(); + boolean success = createIndexResult.isSucceeded(); + LOGGER.warn("Index creation result is: {}", success); + + if (success) { + putAlias(indexName, indexManager.getAliasName()); + setupTypeMapping(indexName); + setupPipelines(); + + newIndex.set(true); // automatic rebuild support + } else { + if (execute(new IndicesExists.Builder(indexName).build()).waitForResult().isSucceeded()) { + LOGGER.warn("Creation of index '{}' failed, but it exists now, it was created and configured by another node, proceeding...", + indexName); + } else { + LOGGER.error("Failed to create the index '{}' {}", indexName, createIndexResult.getJsonString()); + throw new ConfigurationException(createIndexResult.getErrorMessage()); + } + } + } + } + + /** + * Create pipelines
+ * Instead mapping transform we can use pipeline: + * https://www.elastic.co/guide/en/elasticsearch/reference/5.0/breaking_50_mapping_changes.html#_source_transform_removed + * We can define a pipeline for every type, and when we make insert we will the pipeline if the + * pipeline is defined for this type + */ + private void setupPipelines() { + indexConfig.getPipelines().stream().filter(StringUtils::isNotBlank) // Check null or empty + .filter(type -> StringUtils.isNotBlank(indexConfig.getPipeline(type))) // Check pipeline definition exists + .forEach(type -> { + execute(new UpdatePipeline.Builder(type, indexConfig.getPipeline(type)).build()).waitForSuccess(); + LOGGER.debug("Pipeline created for '{}'", type); + }); + } + + /** + * Update the type mapping of an existing index + */ + private void setupTypeMapping(String indexName) throws ConfigurationException { + String mapping = indexConfig.getTypeMapping(typeName); + + if (StringUtils.isNotBlank(mapping)) { + LOGGER.debug("Updating type mapping for '{}' to:\n\t{}", typeName, mapping); + putMapping(indexName, mapping); + } + } + + @Override + public SearchResult putMapping(String index, String source) { + // We put the new getMapping on current index, not the alias + PutMapping.Builder put = new PutMapping.Builder(index, typeName, source); + if (!isV6) { + put.setParameter("include_type_name", "true"); + } + JestResult result = execute(put.build()).waitForSuccess(); + return new SearchResult(new ArrayList<>(), result.getJsonObject()); + } + + public SearchResult getMapping(String index) { + JestResult result = execute(new GetMapping.Builder().addIndex(index).build()).waitForResult(); + return new SearchResult(new ArrayList<>(), result.getJsonObject()); + } + + @Override + public void index(@Nonnull List> partitionKeys, @Nonnull List elements, long expirationTime, + boolean isInsert) throws IOException { + if (isConcurrentLock) { + // lock on intern representation of type+PK, for example: "Interaction[(Id,0001HZO1Qq0haiGs)]" + // This prevents concurrent upserts on the same doc from the same node + synchronized ((typeName + partitionKeys).intern()) { + indexInternal(partitionKeys, elements, expirationTime); + } + } else { + indexInternal(partitionKeys, elements, expirationTime); + } + } + + private void indexInternal(List> partitionKeys, List elements, long expirationTime) + throws IOException { + + Map> groupedMap = group(partitionKeys, elements); + + for (Map.Entry> entry : groupedMap.entrySet()) { + update(partitionKeys, entry.getKey(), entry.getValue(), expirationTime); + } + } + + private void update(List> partitionKeys, String docId, List elements, long expirationTime) + throws IOException { + + StringWriter stringWriter = new StringWriter(); + + try (JsonGenerator builder = jsonFactory.createJsonGenerator(stringWriter)) { + builder.writeStartObject(); + + for (Pair pk : partitionKeys) { + builder.writeStringField(pk.left, pk.right); + } + + boolean clusteringKeysSet = false; + + Map> collections = null; + + // Fill simple fields and map complex types + for (CellElement element : elements) { + + if (element.clusteringKeys != null) { + if (!clusteringKeysSet) { // Insert clustering keys if not already done + for (Pair key : element.clusteringKeys) { + builder.writeStringField(key.left, key.right); + } + + clusteringKeysSet = true; + } + } + + if (element.isCollection() && element.collectionValue != null) { + if (collections == null) { + collections = new HashMap<>(); + } + + collections.computeIfAbsent(element, k -> new HashMap<>()).put(element.collectionValue.name, element.collectionValue.value); + + } else if (element.value != null) { + try { + if (jsonFlatSerializedFields.contains(element.name)) { + String flattenedJson = JsonUtils.flatten(element.value); + builder.writeFieldName(element.name); + builder.writeRawValue(flattenedJson); + } else if (jsonSerializedFields.contains(element.name)) { + builder.writeFieldName(element.name); + builder.writeRawValue(element.value); + } else { // Simple field + builder.writeStringField(element.name, element.value); + } + } catch (IOException ex) { + if (SKIP_BAD_JSON) { + LOGGER.warn("Skipped bad json for field {} of document {}", element.name, docId, ex); + } else { + throw ex; + } + } + } else { // null value + builder.writeNullField(element.name); + } + } + + if (collections != null) { + // Fill the collections now that they are sorted + for (Map.Entry> collection : collections.entrySet()) { + CellElement element = collection.getKey(); + + if (element.collectionValue != null) { + switch (element.collectionValue.type) { + case JSON: + builder.writeObjectFieldStart(element.name); + + for (Map.Entry en : collection.getValue().entrySet()) { + String value = en.getValue(); + if (value == null) { + builder.writeNullField(en.getKey()); + } else { + builder.writeFieldName(en.getKey()); + builder.writeRawValue(value); + } + } + builder.writeEndObject(); + break; + + case MAP: + builder.writeObjectFieldStart(element.name); + for (Map.Entry entry : collection.getValue().entrySet()) { + builder.writeStringField(entry.getKey(), entry.getValue()); + } + builder.writeEndObject(); + break; + + case LIST: + case SET: + builder.writeArrayFieldStart(element.name); + for (String value : collection.getValue().keySet()) { + builder.writeString(value); + } + builder.writeEndArray(); + break; + + default: + // There is no other CollectionType + } + } + } + } + + if (ENABLE_INDEXATION_DATE) { + builder.writeStringField(INDEXATION_DATE, JsonUtils.getIso8601Date(new Date())); + } + + if (indexManager.isTTLFieldRequired()) { + builder.writeNumberField(TTL_FIELD, expirationTime); + } + + builder.writeEndObject(); + builder.close(); // calling close() early because we want the output now + String jsonDoc = stringWriter.toString(); + + if (EsSecondaryIndex.DEBUG_SHOW_VALUES) { + String operation = (insertOnly || usePipeline) ? "insert" : "upsert"; + LOGGER.debug("Document {} index {} {} with content {}", typeName, operation, docId, jsonDoc); + } + + String currentName = indexManager.getCurrentName(); + ResponseHandler handler; + if (insertOnly || usePipeline) { + Index.Builder indexRequest = new Index.Builder(jsonDoc).index(currentName).type(typeName).id(docId); + + if (usePipeline) { // https://www.elastic.co/guide/en/elasticsearch/reference/5.5/ingest.html + indexRequest.setParameter(ES_PIPELINE, typeName); + } + handler = execute(indexRequest.build()); + + } else { + // Pipelines can only be used with index or bulk + Update.Builder update = new Update.Builder(String.format(DOC_AS_UPSERT, jsonDoc)) + .index(currentName) + .type(typeName) + .id(docId); + + if (indexConfig.getRetryOnConflict() > -1) { + update.setParameter(RETRY_ON_CONFLICT, indexConfig.getRetryOnConflict()); + } + + handler = execute(update.build()); + } + + if (!isAsyncWrite) { + handler.waitForSuccess(); // Will block until response anc ensure result is a success + } + } + } + + /** + * Group all CellElement according to their clustering keys + * + * @param partitionKeys not null + * @param elements not null + * @return a grouped map + */ + private Map> group(List> partitionKeys, List elements) { + Map> sortedCells = new HashMap<>(); + + for (CellElement element : elements) { + String docId = CStarUtils.toEsId(partitionKeys, element.clusteringKeys); + sortedCells.computeIfAbsent(docId, k -> new ArrayList<>()).add(element); + } + + return sortedCells; + } + + @Override + public void delete(@Nonnull List> partitionKeys) { + String docId = CStarUtils.toEsId(partitionKeys, null); + String currentName = indexManager.getCurrentName(); + ResponseHandler handler = execute(new Delete.Builder(docId).index(currentName).type(typeName).build()); + if (!isAsyncWrite) { + handler.waitForStatus(200, 404, 204); // Blocks until response. Does not ensure result is a success. + } + } + + @Override + public Object flush() { + return execute(new Flush.Builder().addIndex(indexManager.getCurrentName()).force(true).build()).waitForSuccess(); + } + + @Override + @Nonnull + public SearchResult search(@Nonnull QueryMetaData queryMetaData) { + String queryString = queryMetaData.query; + + LOGGER.trace("Index {} search with query {}", typeName, queryString); + + if (!queryString.startsWith(JSON_PREFIX)) { + queryString = String.format(QUERY_WRAPPER_WITH_SIZE, maxResults, escape(queryString)); + } + + Search.Builder builder = new Search.Builder(queryString).addIndex(indexManager.getAliasName()).addType(typeName); + + io.searchbox.core.SearchResult searchResponse; + try { + Search searchRequest = queryMetaData.loadSource() ? builder.build() : builder.addSourceIncludePattern(pkIncludePattern).build(); + searchResponse = execute(searchRequest).waitForSuccess(); + } catch (CassandraException e) { + throw new InvalidRequestException(e.getMessage()); + } + + LOGGER.trace("Index {} search result: {}", typeName, searchResponse); + + final List idList = new ArrayList<>(); + + JsonElement hits = JsonUtils.getJsonObject(searchResponse, ES_HITS).get(ES_HITS); + if (hits != null) { + List primaryKeys; + + if (hasClusteringColumns) { + primaryKeys = new ArrayList<>(partitionKeysNames.size() + clusteringColumnsNames.size()); + primaryKeys.addAll(partitionKeysNames); + primaryKeys.addAll(clusteringColumnsNames); + } else { + primaryKeys = partitionKeysNames; + } + + int pkSize = primaryKeys.size(); + + hits.getAsJsonArray().forEach(result -> { + String[] primaryKey = new String[pkSize]; + + int keyNb = 0; + + for (String keyName : primaryKeys) { + String value = JsonUtils.getString(result, ES_SOURCE, keyName); + + if (value == null) { + LOGGER.warn("Missing pk {} from ES results, skipping hit:{}", keyName, JsonUtils.getString(result, ES_ID)); + continue; + } else { + primaryKey[keyNb] = value; + } + keyNb++; + } + + if (keyNb == pkSize) { // Will only be false if we skipped a hit, see above warning + idList.add(new SearchResultRow(primaryKey, result.getAsJsonObject())); + } + + }); + } + + // Remove the content of {"hits":{"hits": (big values) } } + JsonObject metadata = JsonUtils.filterPath(searchResponse.getJsonObject(), ES_HITS, ES_HITS); + return new SearchResult(idList, metadata); + } + + private String extractQuery(String query) { + try { + return mapper.readTree(query).get("query").toString(); + } catch (Exception e) { + LOGGER.trace("Could not extract query node from '{}' for Index {}", query, indexManager.getAliasName()); + } + return query; + } + + @Override + public void validate(@Nonnull String query) throws InvalidRequestException { + if (!isValidateQuery) { + return; + } + LOGGER.trace("Validating query {}", query); + + String esQuery = query; + + // Ignore index management queries like #update# .... # + if (query.startsWith("#")) { + if (query.endsWith("#")) { //WCC-886 + return; + } else { + int optionEnd = query.indexOf("#", 1); + if (optionEnd < 0) { + throw new InvalidRequestException("Query starts with '#', but second '#' is missing"); + } + esQuery = query.substring(optionEnd + 1); + } + } + + String formattedQuery; + if (esQuery.startsWith(JSON_PREFIX)) { + formattedQuery = String.format(QUERY_WRAPPER, extractQuery(esQuery)); // WCC-876 + } else { + formattedQuery = String.format(QUERY_WRAPPER_WITH_QUOTES, esQuery); + } + + LOGGER.trace("Validating query {}", formattedQuery); + try { + Validate.Builder validateBuilder = new Validate.Builder(formattedQuery); + validateBuilder.setParameter(EXPLAIN, String.valueOf(true)); + JestResult res = execute(validateBuilder.build()).waitForResult(); + if (!res.isSucceeded()) { + LOGGER.info("Query {} is invalid", formattedQuery); + throw new InvalidRequestException(res.getErrorMessage()); + } else { + String valid = res.getJsonObject().get("valid").getAsString(); + if (Boolean.parseBoolean(valid)) { + LOGGER.trace("Query {} is valid", formattedQuery); + } else { + throw new InvalidRequestException(res.getJsonObject().toString()); + } + } + + } catch (Exception e) { + throw new InvalidRequestException(e.getMessage()); + } + } + + @Override + public void settingsUpdated() { + indexManager.checkForUpdate(); + setupIndex(indexManager.getCurrentName()); + } + + @Override + public boolean isNewIndex() { + return newIndex.getAndSet(false); + } + + @Nonnull + private ResponseHandler execute(Action request) { + ResponseHandler handler = new ResponseHandler<>(typeName, request); + client.executeAsync(request, handler); + return handler; + } + + @Override + public Object drop() { + if (indexConfig.isPerIndexType()) { + String indexName = indexManager.getCurrentName(); + LOGGER.warn("Index {}/{} is being dropped, stopping purge task, deleting ES index", indexName, typeName); + indexManager.stop(); + + JestResult res = execute(new Delete.Builder("").index(indexName).build()).waitForResult(); + return res.isSucceeded(); + } else { + return truncate(); + } + } + + @Override + public Object truncate() { + String aliasName = indexManager.getAliasName(); + LOGGER.warn("Index {}/{} is being truncated, deleting documents", aliasName, typeName); + JestResult res = execute(new Delete.Builder(MATCH_ALL).index(aliasName).type(typeName).build()).waitForResult(); + return res.isSucceeded(); + } + + @Override + public void deleteExpired() { + String aliasName = indexManager.getAliasName(); + long ttl = FBUtilities.nowInSeconds() + ttlShift; + + DeleteByQuery deleteQuery = new DeleteByQuery.Builder(String.format(MATCH_LTE, ttl)).addIndex(aliasName).addType(typeName).build(); + JestResult res = execute(deleteQuery).waitForSuccess(); + + Long deleted = JsonUtils.getLong(res.getJsonObject(), "deleted"); + if (deleted != null && deleted > 0) { + LOGGER.debug("Index {} deleted {} documents where _cassandraTtl < {}", indexManager.getAliasName(), deleted, ttl); + } + } + + @Override + public void purgeEmptyIndexes() { + String aliasName = indexManager.getAliasName(); + LOGGER.debug("Start segmented index cleanup for {}", aliasName); + JestResult aliasesResponses = execute(new GetAliases.Builder().addIndex(aliasName).build()).waitForResult(); + + if (aliasesResponses.isSucceeded()) { + JsonUtils.getJsonObject(aliasesResponses, aliasName, "aliases").entrySet().forEach(alias -> { + String indexToDelete = alias.getKey(); + CountResult count = execute(new Count.Builder().addIndex(indexToDelete).build()).waitForResult(); + + if (count.isSucceeded() && count.getCount().intValue() == 0) { + dropIndex(indexToDelete); + } else { + LOGGER.debug("Index {} is not empty", indexToDelete); + } + }); + } + } + + @Override + public void updateIndexConfigOptions() { + ttlShift = indexConfig.getTtlShift(); + isConcurrentLock = indexConfig.isConcurrentLock(); + jsonFlatSerializedFields = indexConfig.getJsonFlatSerializedFields(); + jsonSerializedFields = indexConfig.getJsonSerializedFields(); + maxResults = indexConfig.getMaxResults(); + isValidateQuery = indexConfig.isValidateQuery(); + isAsyncWrite = indexConfig.isAsyncWrite(); + insertOnly = indexConfig.isInsertOnly(); + usePipeline = StringUtils.isNotBlank(indexConfig.getPipeline(typeName)); + httpPort = indexConfig.getHttpPort(); + indexManager.updateOptions(); + } + + @Override + public List getIndexNames() { + List result = new LinkedList<>(); + JestResult res = execute(new GetAliases.Builder().addIndex(indexManager.getAliasName()).build()).waitForResult(); + Set> set = res.getJsonObject().entrySet(); + for (Map.Entry entry : set) { + result.add(entry.getKey()); + } + return result; + } + + @Override + public void dropIndex(String indexName) { + LOGGER.info("Deleting index {}", indexName); + boolean success = execute(new DeleteIndex.Builder(indexName).build()).waitForResult().isSucceeded(); + LOGGER.info("Index {} deletion {}", indexName, success ? "successful" : "failed"); + } + + private void putAlias(String indexName, String alias) { + LOGGER.warn("Creating index alias '{}'", indexManager.getAliasName()); + AliasMapping aliases = new AddAliasMapping.Builder(indexName, alias).build(); + JestResult addAliasResult = execute(new ModifyAliases.Builder(aliases).build()).waitForResult(); + LOGGER.warn("Index alias creation result is: {}", addAliasResult.isSucceeded()); + } +} diff --git a/src/main/java/com/genesyslab/webme/commons/index/EsIndexBuilder.java b/src/main/java/com/genesyslab/webme/commons/index/EsIndexBuilder.java index a8a23a6..7c97432 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/EsIndexBuilder.java +++ b/src/main/java/com/genesyslab/webme/commons/index/EsIndexBuilder.java @@ -1,21 +1,21 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index; -import com.google.common.base.Stopwatch; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.compaction.CompactionInfo; @@ -29,9 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; -import java.util.UUID; -import java.util.concurrent.TimeUnit; +import com.google.common.base.Stopwatch; /** * Index building task that reads all live SSTables and index the content @@ -67,31 +65,30 @@ public void build() { index.esIndex.truncate(); } - //For each of the SSTables, we get a partition scanner, for each partition we get its row and we index it - ssTables.forEach(ssTableReader -> ssTableReader.getScanner() - .forEachRemaining(partition -> { - partition.forEachRemaining(row -> { - if (isStopRequested()) { - LOGGER.warn("{} build {} stop requested {}/{} rows", index.name, compactionId, processed, total); - throw new CompactionInterruptedException(getCompactionInfo()); - } + // For each of the SSTables, we get a partition scanner, for each partition we get its row and we + // index it + ssTables.forEach(ssTableReader -> ssTableReader.getScanner().forEachRemaining(partition -> { + partition.forEachRemaining(row -> { + if (isStopRequested()) { + LOGGER.warn("{} build {} stop requested {}/{} rows", index.name, compactionId, processed, total); + throw new CompactionInterruptedException(getCompactionInfo()); + } - DecoratedKey key = partition.partitionKey(); - if (row instanceof Row) { //not sure what else it could be - index.index(key, (Row) row, null, FBUtilities.nowInSeconds()); - } else { - LOGGER.warn("{} build {} skipping unsupported {} {}", index.name, compactionId, row.getClass().getName(), key); - } - processed++; - }); + DecoratedKey key = partition.partitionKey(); + if (row instanceof Row) { // not sure what else it could be + index.index(key, (Row) row, null, FBUtilities.nowInSeconds()); + } else { + LOGGER.warn("{} build {} skipping unsupported {} {}", index.name, compactionId, row.getClass().getName(), key); } - ) - ); + processed++; + }); + })); LOGGER.info("{} build {} completed in {} minutes for {} rows", index.name, compactionId, stopwatch.elapsed(TimeUnit.MINUTES), total); } + @Override public CompactionInfo getCompactionInfo() { - return new CompactionInfo(null, OperationType.INDEX_BUILD, processed, total, null, compactionId); + return CompactionInfo.withoutSSTables(null, OperationType.INDEX_BUILD, processed, total, null, compactionId); } } diff --git a/src/main/java/com/genesyslab/webme/commons/index/EsSecondaryIndex.java b/src/main/java/com/genesyslab/webme/commons/index/EsSecondaryIndex.java index 1e9fe1e..4a2f08e 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/EsSecondaryIndex.java +++ b/src/main/java/com/genesyslab/webme/commons/index/EsSecondaryIndex.java @@ -1,47 +1,52 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index; -import com.genesyslab.webme.commons.index.config.IndexConfig; -import com.genesyslab.webme.commons.index.config.IndexConfiguration; -import com.genesyslab.webme.commons.index.config.LogConfigurator; -import com.genesyslab.webme.commons.index.indexers.EsIndexer; -import com.genesyslab.webme.commons.index.indexers.FakePartitionIterator; -import com.genesyslab.webme.commons.index.indexers.NoOpIndexer; -import com.genesyslab.webme.commons.index.indexers.NoOpPartitionIterator; -import com.genesyslab.webme.commons.index.indexers.StreamingPartitionIterator; +import static com.genesyslab.webme.commons.index.DefaultIndexManager.INDEX_POSTFIX; +import static com.genesyslab.webme.commons.index.JsonUtils.unQuote; +import static com.genesyslab.webme.commons.index.config.IndexConfig.ES_CONFIG_PREFIX; -import com.google.common.base.Stopwatch; -import com.google.common.base.Strings; -import com.google.gson.JsonObject; +import java.io.IOException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.cql3.statements.IndexTarget; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.EmptyIterators; -import org.apache.cassandra.db.PartitionColumns; import org.apache.cassandra.db.PartitionRangeReadCommand; import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.WriteContext; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.filter.EsSimpleExpression; import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.filter.RowFilter.Expression; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.PartitionUpdate; @@ -55,34 +60,28 @@ import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; -import org.apache.cassandra.service.MigrationManager; +import org.apache.cassandra.schema.MigrationManager; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; -import org.apache.cassandra.utils.concurrent.OpOrder.Group; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import java.io.IOException; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; -import java.util.function.BiFunction; - -import static com.genesyslab.webme.commons.index.DefaultIndexManager.INDEX_POSTFIX; -import static com.genesyslab.webme.commons.index.JsonUtils.unQuote; -import static com.genesyslab.webme.commons.index.config.IndexConfig.ES_CONFIG_PREFIX; +import com.genesyslab.webme.commons.index.config.IndexConfig; +import com.genesyslab.webme.commons.index.config.IndexConfiguration; +import com.genesyslab.webme.commons.index.config.LogConfigurator; +import com.genesyslab.webme.commons.index.indexers.EsIndexer; +import com.genesyslab.webme.commons.index.indexers.FakePartitionIterator; +import com.genesyslab.webme.commons.index.indexers.NoOpIndexer; +import com.genesyslab.webme.commons.index.indexers.NoOpPartitionIterator; +import com.genesyslab.webme.commons.index.indexers.StreamingPartitionIterator; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; +import com.google.gson.JsonObject; /** * Interesting read to adapt to Cassandra 3: http://www.doanduyhai.com/blog/?p=2058 @@ -104,7 +103,7 @@ public class EsSecondaryIndex implements Index { private static final String PUT_MAPPING = "#put_mapping#"; private static final String FAKE_ID = "FakeId"; - private static final boolean DEPENDS_ON_COLUMN_DEFINITION = false; //we support dynamic addition/removal of columns + private static final boolean DEPENDS_ON_COLUMN_DEFINITION = false; // we support dynamic addition/removal of columns public final ColumnFamilyStore baseCfs; public final String indexColumnName; @@ -113,7 +112,7 @@ public class EsSecondaryIndex implements Index { @Nonnull final IndexInterface esIndex; private final SecureRandom random = new SecureRandom(); - private final ColumnDefinition indexColDef; + private final ColumnMetadata indexColDef; private final boolean isDummyMode; private IndexMetadata indexMetadata; private List partitionKeysNames; @@ -128,13 +127,13 @@ public class EsSecondaryIndex implements Index { public EsSecondaryIndex(ColumnFamilyStore sourceCfs, IndexMetadata indexMetadata) throws Exception { synchronized (EsSecondaryIndex.class) { // we create one index at a time - LogConfigurator.configure(); //disable the very verbose Apache client logs + LogConfigurator.configure(); // disable the very verbose Apache client logs this.baseCfs = sourceCfs; this.indexMetadata = indexMetadata; indexColumnName = unQuote(this.indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME)); - indexColDef = baseCfs.metadata.getColumnDefinition(ColumnIdentifier.getInterned(indexColumnName, true)); - name = "EsSecondaryIndex [" + baseCfs.metadata.ksName + "." + this.indexMetadata.name + "]"; + indexColDef = baseCfs.metadata().getColumn(ColumnIdentifier.getInterned(indexColumnName, true)); + name = "EsSecondaryIndex [" + baseCfs.metadata.keyspace + "." + this.indexMetadata.name + "]"; indexConfig = new IndexConfiguration(name, indexMetadata.options); LOGGER.info("Creating {} with options {}", name, indexConfig.getIndexOptions()); @@ -147,14 +146,14 @@ public EsSecondaryIndex(ColumnFamilyStore sourceCfs, IndexMetadata indexMetadata index = new EsDummyIndex(); } else { LOGGER.warn("EsSecondaryIndex {} initializing #{}", name, Integer.toHexString(hashCode())); - partitionKeysNames = Collections.unmodifiableList(CStarUtils.getPartitionKeyNames(baseCfs.metadata)); - clusteringColumnsNames = Collections.unmodifiableList(CStarUtils.getClusteringColumnsNames(baseCfs.metadata)); + partitionKeysNames = Collections.unmodifiableList(CStarUtils.getPartitionKeyNames(baseCfs.metadata())); + clusteringColumnsNames = Collections.unmodifiableList(CStarUtils.getClusteringColumnsNames(baseCfs.metadata())); - LOGGER.debug("ReadConsistencyLevel is '{}', skip startup log replay:{}, skip non local updates:{}", - readConsistencyLevel, skipLogReplay, skipNonLocalUpdates); + LOGGER.debug("ReadConsistencyLevel is '{}', skip startup log replay:{}, skip non local updates:{}", readConsistencyLevel, + skipLogReplay, skipNonLocalUpdates); hasClusteringColumns = !clusteringColumnsNames.isEmpty(); - index = new ElasticIndex(indexConfig, baseCfs.metadata.ksName, baseCfs.name, partitionKeysNames, clusteringColumnsNames); + index = new ElasticIndex(indexConfig, baseCfs.metadata.keyspace, baseCfs.name, partitionKeysNames, clusteringColumnsNames); index.init(); LOGGER.warn("Initialized {} ", name); } @@ -185,31 +184,32 @@ private void updateIndexConfigOptions() { /** * @param decoratedKey PK of the update - * @param newRow the new version of the row - * @param oldRow provided in case of updates, null for inserts - * @param nowInSec time of the update + * @param newRow the new version of the row + * @param oldRow provided in case of updates, null for inserts + * @param nowInSec time of the update */ public void index(@Nonnull DecoratedKey decoratedKey, @Nonnull Row newRow, @Nullable Row oldRow, int nowInSec) { String id = ByteBufferUtil.bytesToHex(decoratedKey.getKey()); - Tracing.trace("ESI decoding row {}", id); //This is CQL "tracing on" support + Tracing.trace("ESI decoding row {}", id); // This is CQL "tracing on" support try { - List> partitionKeys = CStarUtils.getPartitionKeys(decoratedKey.getKey(), baseCfs.metadata); + List> partitionKeys = CStarUtils.getPartitionKeys(decoratedKey.getKey(), baseCfs.metadata()); List elements = new ArrayList<>(); for (Cell cell : newRow.cells()) { - if (cell.isLive(nowInSec) || !discardNullValues) { //optionally ignore null values + if (cell.isLive(nowInSec) || !discardNullValues) { // optionally ignore null values - // Skip the cells with empty name (row marker) // looks like isEmpty() now (2.2?) returns false with empty string + // Skip the cells with empty name (row marker) // looks like isEmpty() now (2.2?) returns false with + // empty string String cellName = cell.column().name.toString(); - if (!Strings.isNullOrEmpty(cellName)) { // Skip the cells with empty name (row marker) + if (!Strings.isNullOrEmpty(cellName)) { // Skip the cells with empty name (row marker) CellElement element = new CellElement(); element.name = cellName; if (hasClusteringColumns) { - element.clusteringKeys = CStarUtils.getClusteringKeys(newRow, baseCfs.metadata, clusteringColumnsNames); + element.clusteringKeys = CStarUtils.getClusteringKeys(newRow, baseCfs.metadata(), clusteringColumnsNames); } if (CStarUtils.isCollection(cell)) { @@ -258,7 +258,7 @@ public void delete(DecoratedKey decoratedKey) { return; } - esIndex.delete(CStarUtils.getPartitionKeys(decoratedKey.getKey(), baseCfs.metadata)); + esIndex.delete(CStarUtils.getPartitionKeys(decoratedKey.getKey(), baseCfs.metadata())); } catch (Exception e) { LOGGER.error("{} can't delete row {} {}", name, id, e); throw new RuntimeException(e); @@ -279,7 +279,7 @@ public String toString() { } @Override - public IndexBuildingSupport getBuildTaskSupport() { //This is for rebuild command + public IndexBuildingSupport getBuildTaskSupport() { // This is for rebuild command if (isDummyMode) { return null; } @@ -287,21 +287,14 @@ public IndexBuildingSupport getBuildTaskSupport() { //This is for rebuild comman } @Override - public Callable getInitializationTask() { //This is done when starting Cassandra or when creating an index with CQL command + public Callable getInitializationTask() { // This is done when starting Cassandra or when creating an index with CQL command return () -> { - if (esIndex.isNewIndex()) { //FIXME will this rebuild all data since we only have ssTables for our replicas? - if (indexAvailableWhenBuilding) { - LOGGER.info("{} marking index as built while rebuilding is in progress", name); - baseCfs.indexManager.markIndexBuilt(indexMetadata.name); - new EsIndexBuilder(EsSecondaryIndex.this).build(); - } else { - LOGGER.info("{} index rebuild completed, marking as built", name); - new EsIndexBuilder(EsSecondaryIndex.this).build(); - baseCfs.indexManager.markIndexBuilt(indexMetadata.name); - } + if (esIndex.isNewIndex()) { // FIXME will this rebuild all data since we only have ssTables for our replicas? + LOGGER.info("{} index building", name); + new EsIndexBuilder(EsSecondaryIndex.this).build(); + LOGGER.info("{} index build completed", name); } else { LOGGER.debug("{} already exists, nothing to rebuild", name); - baseCfs.indexManager.markIndexBuilt(indexMetadata.name); } return null; }; @@ -325,7 +318,7 @@ public void register(IndexRegistry registry) { @Override public Optional getBackingTable() { - return Optional.empty(); //We don't use a CFS to store index data + return Optional.empty(); // We don't use a CFS to store index data } @Override @@ -350,27 +343,29 @@ public Callable getPreJoinTask(boolean hadBootstrap) { @Override public boolean shouldBuildBlocking() { - return false; //We don't want to block table/index access while it's (re)building + return false; // We don't want to block table/index access while it's (re)building } @Override public SSTableFlushObserver getFlushObserver(Descriptor descriptor, OperationType opType) { - return null; //Don't think we care about table flushes + return null; // Don't think we care about table flushes } @Override - public boolean dependsOn(ColumnDefinition column) { + public boolean dependsOn(ColumnMetadata column) { return DEPENDS_ON_COLUMN_DEFINITION; } @Override - public boolean supportsExpression(ColumnDefinition column, Operator operator) { - return true; //We support any kind of C* expressions because it's actually in the ES query. + public boolean supportsExpression(ColumnMetadata column, Operator operator) { + // We support any kind of C* expressions because it's actually in the ES query. + // however check that column is indexed (called from SecondaryIndexManager.getBestIndexFor()) + return this.indexColDef != null && this.indexColDef.name.equals(column.name); } @Override public AbstractType customExpressionValueType() { - return null; //we don't support custom expressions, yet. + return null; // we don't support custom expressions, yet. } @Override @@ -380,19 +375,22 @@ public RowFilter getPostIndexQueryFilter(RowFilter filter) { @Override public long getEstimatedResultRows() { - /* From http://www.doanduyhai.com/blog/?p=2058#sasi_read_path - * As a result, every search with SASI currently always hit the same node, which is the node responsible for the first token range - * on the cluster. Subsequent rounds of query (if any) will spread out to other nodes eventually + /* + * From http://www.doanduyhai.com/blog/?p=2058#sasi_read_path As a result, every search with SASI + * currently always hit the same node, which is the node responsible for the first token range on + * the cluster. Subsequent rounds of query (if any) will spread out to other nodes eventually */ - //Trying to be smart here, if each node returns random negative we'll get load spread and will still be selected over native index + // Trying to be smart here, if each node returns random negative we'll get load spread and will + // still be selected over native index return -Math.abs(random.nextLong()); } @Override - public Indexer indexerFor(DecoratedKey key, PartitionColumns columns, int nowInSec, Group opGroup, IndexTransaction.Type txType) { + public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, + IndexTransaction.Type txType) { if (isDummyMode) { - return NoOpIndexer.INSTANCE; //Dummy mode + return NoOpIndexer.INSTANCE; // Dummy mode } if (skipLogReplay && !StorageService.instance.isInitialized()) { @@ -427,15 +425,15 @@ public Searcher searcherFor(final ReadCommand command) { @Override public void validate(PartitionUpdate update) throws InvalidRequestException { - //nothing to check for now + // nothing to check for now } @Override public void validate(ReadCommand command) throws InvalidRequestException { String queryString = CStarUtils.queryString(command); - //don't validate commands + // don't validate commands if (!queryString.startsWith(UPDATE) && !queryString.startsWith(GET_MAPPING) && !queryString.startsWith(PUT_MAPPING)) { - LOGGER.trace("Index {} validate query: {}", name, queryString); //reducing level because we'll see it as search later + LOGGER.trace("Index {} validate query: {}", name, queryString); // reducing level because we'll see it as search later esIndex.validate(queryString); } } @@ -447,7 +445,7 @@ public UnfilteredPartitionIterator search(ReadCommand command) { if (queryString.startsWith(UPDATE)) { handleUpdateCommand(queryString.substring(UPDATE.length(), queryString.length() - 1)); - return EmptyIterators.unfilteredPartition(command.metadata(), command.isForThrift()); + return EmptyIterators.unfilteredPartition(command.metadata()); } if (!(command instanceof PartitionRangeReadCommand)) { @@ -457,6 +455,14 @@ public UnfilteredPartitionIterator search(ReadCommand command) { PartitionRangeReadCommand readCommand = (PartitionRangeReadCommand) command; + Optional expression = + readCommand.rowFilter().getExpressions().stream().filter(exp -> exp.column().equals(this.indexColDef)).findFirst(); + if (!expression.isPresent()) { + throw new UnsupportedOperationException(this.getClass().getName() + " not support search by non-indexed field"); + } + int index = readCommand.rowFilter().getExpressions().indexOf(expression.get()); + readCommand.rowFilter().getExpressions().set(index, new EsSimpleExpression(expression.get())); + if (queryString.startsWith(GET_MAPPING)) { return getMapping(readCommand, searchId); } @@ -466,9 +472,9 @@ public UnfilteredPartitionIterator search(ReadCommand command) { } LOGGER.debug("Index {} search query {} '{}'", name, searchId, queryString); - Tracing.trace("ESI {} Searching '{}'", searchId, queryString); //This is CQL "tracing on" support + Tracing.trace("ESI {} Searching '{}'", searchId, queryString); // This is CQL "tracing on" support - //Extract query metadata if any + // Extract query metadata if any QueryMetaData queryMetaData = new QueryMetaData(queryString); SearchResult searchResult = esIndex.search(queryMetaData); @@ -476,7 +482,7 @@ public UnfilteredPartitionIterator search(ReadCommand command) { Tracing.trace("ESI {} Found {} matching ES docs in {}ms", searchId, searchResult.items.size(), time.elapsed(TimeUnit.MILLISECONDS)); if (searchResult.items.isEmpty()) { - return EmptyIterators.unfilteredPartition(command.metadata(), command.isForThrift()); + return EmptyIterators.unfilteredPartition(command.metadata()); } fillPartitionAndClusteringKeys(searchResult.items); @@ -484,15 +490,17 @@ public UnfilteredPartitionIterator search(ReadCommand command) { Token start = readCommand.dataRange().keyRange().left.getToken(); Token stop = readCommand.dataRange().keyRange().right.getToken(); - if (!start.equals(stop)) { //Do we have token ranges to filter out ? + if (!start.equals(stop)) { // Do we have token ranges to filter out ? LOGGER.info("Range queries will result in multiple ES queries, add 'and token(pk)=rnd.long' to your query"); - /* We must only load a row if its DecoratedKey is within the range of requested tokens. Note - * that the same node can receive several requests for the same search but with different - * ranges. If filtering is not done it will return duplicates. Drawback is that ES query is - * sent several times for the same search. Also it means ordering might not work as expected.*/ + /* + * We must only load a row if its DecoratedKey is within the range of requested tokens. Note that + * the same node can receive several requests for the same search but with different ranges. If + * filtering is not done it will return duplicates. Drawback is that ES query is sent several times + * for the same search. Also it means ordering might not work as expected. + */ searchResult.items - .removeIf(result -> !readCommand.dataRange().keyRange().contains(baseCfs.getPartitioner().decorateKey(result.partitionKey))); + .removeIf(result -> !readCommand.dataRange().keyRange().contains(baseCfs.getPartitioner().decorateKey(result.partitionKey))); } if (queryMetaData.loadRows()) { @@ -521,15 +529,15 @@ private UnfilteredPartitionIterator putMapping(PartitionRangeReadCommand readCom String additionalMapping = queryString.substring(PUT_MAPPING.length(), queryString.length() - 1); esIndex.putMapping(indexName(), additionalMapping); - return EmptyIterators.unfilteredPartition(readCommand.metadata(), readCommand.isForThrift()); + return EmptyIterators.unfilteredPartition(readCommand.metadata()); } private void handleUpdateCommand(String newSettings) { try { - updateSettings(newSettings); //Update in Cassandra index options - boolean changed = reloadSettings(); //Apply new settings and merge them with es-index.properties + updateSettings(newSettings); // Update in Cassandra index options + boolean changed = reloadSettings(); // Apply new settings and merge them with es-index.properties if (changed) { - esIndex.settingsUpdated(); //Notify the ES index to create new segments + esIndex.settingsUpdated(); // Notify the ES index to create new segments } } catch (IOException e) { LOGGER.error("Index {} update setting error: {}", name, e.getMessage(), e); @@ -563,36 +571,55 @@ private void fillPartitionAndClusteringKeys(List searchResultRo clusteringKeys = null; } - searchResultRow.partitionKey = CStarUtils.getPartitionKeys(partitionKeys, baseCfs.metadata); + searchResultRow.partitionKey = CStarUtils.getPartitionKeys(partitionKeys, baseCfs.metadata()); searchResultRow.clusteringKeys = clusteringKeys; } } private void updateSettings(String settings) throws IOException { LOGGER.info("Update {} settings to '{}'", name, settings); - Tracing.trace("Update {} settings to '{}'", name, settings); //This is CQL "tracing on" support + Tracing.trace("Update {} settings to '{}'", name, settings); // This is CQL "tracing on" support Map options = JsonUtils.jsonStringToStringMap(settings); - //check options + // check options if (!options.containsKey(IndexTarget.CUSTOM_INDEX_OPTION_NAME) || !options.containsKey(IndexTarget.TARGET_OPTION_NAME)) { LOGGER.warn("We do not allow to change options class_name and target"); } options.put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, indexMetadata.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME)); options.put(IndexTarget.TARGET_OPTION_NAME, indexMetadata.options.get(IndexTarget.TARGET_OPTION_NAME)); - CFMetaData newMetaData = baseCfs.metadata.copy(); IndexMetadata newIndexMetadata = IndexMetadata.fromSchemaMetadata(indexMetadata.name, indexMetadata.kind, options); - newMetaData.indexes(newMetaData.getIndexes().replace(newIndexMetadata)); - indexMetadata = newIndexMetadata; //assume update will work, helps for UTs - MigrationManager.announceColumnFamilyUpdate(newMetaData, false); + TableMetadata newMetaData = baseCfs.metadata().unbuild().build(); + newMetaData.indexes.replace(newIndexMetadata); + + indexMetadata = newIndexMetadata; // assume update will work, helps for UTs + MigrationManager.announceTableUpdate(newMetaData, false); } - public ColumnDefinition getIndexColDef() { + public ColumnMetadata getIndexColDef() { return indexColDef; } public ConsistencyLevel getReadConsistency() { return readConsistencyLevel; } + + /** + * After cassandra 3.11.10 supportsReplicaFilteringProtection is added to data resolving. + * Data resolving is executed in cassandra when consistency is greater than 1 for cassandra 3. + * and for cassandra 4 data resolving is executed always if coordinator node is not the query executor node. + * + * Because of this for cassandra 3 wcc-es-index is NOT supported above 3.11.10 + * Upgrade is supported from c* 3.x to 4.x + * + * For details in cassandra 3 source code check ReadCallback.get() method and blockfor field. + * For details in cassandra 4 source code SingleRangeResponse.waitForResponse(): resolver.resolve() + * @param rowFilter + * @return + */ + @Override + public boolean supportsReplicaFilteringProtection(RowFilter rowFilter) { + return false; + } } diff --git a/src/main/java/com/genesyslab/webme/commons/index/JsonUtils.java b/src/main/java/com/genesyslab/webme/commons/index/JsonUtils.java index d72216b..c9d1ced 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/JsonUtils.java +++ b/src/main/java/com/genesyslab/webme/commons/index/JsonUtils.java @@ -1,330 +1,326 @@ -/* - * Copyright 2019 Genesys Telecommunications Laboratories, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.genesyslab.webme.commons.index; - -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; - -import io.searchbox.client.JestResult; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import java.io.IOException; -import java.io.StringWriter; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TimeZone; -import java.util.function.Predicate; - -import static org.codehaus.jackson.JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS; -import static org.codehaus.jackson.JsonParser.Feature.ALLOW_SINGLE_QUOTES; - -/** - * Created by Jacques-Henri Berthemet on 05/07/2017. - */ -public class JsonUtils { - - private static final String EXTENDED_ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; - private static final TimeZone GMT_ZONE = TimeZone.getTimeZone("GMT"); - - private static final ObjectMapper OBJECT_MAPPER; - private static final JsonFactory JSON_FACTORY = new JsonFactory(); - private static final JsonParser GSON_PARSER = new JsonParser(); // com.google.gson.JsonParser - - static { - OBJECT_MAPPER = new ObjectMapper().configure(ALLOW_SINGLE_QUOTES, true); - OBJECT_MAPPER.getJsonFactory().enable(ALLOW_NON_NUMERIC_NUMBERS); - } - - @Nonnull - static Map jsonStringToStringMap(@Nonnull String jsonString) throws IOException { - return OBJECT_MAPPER.readValue(jsonString, new TypeReference>() { - }); - } - - @Nonnull - private static Map jsonStringToObjectMap(@Nonnull String jsonString) throws IOException { - return OBJECT_MAPPER.readValue(jsonString, new TypeReference>() { - }); - } - - @Nonnull - public static JsonObject asJsonObject(@Nonnull String jsonString) { - return GSON_PARSER.parse(jsonString).getAsJsonObject(); - } - - /** - * Transform the JSON to a map:string,string[] because ES won't support values of different types for the same key - */ - @Nonnull - static String flatten(@Nonnull String jsonString) throws IOException { - StringWriter stringWriter = new StringWriter(); - JsonGenerator builder = JSON_FACTORY.createJsonGenerator(stringWriter); - - builder.writeStartObject(); - - for (Map.Entry entry : jsonStringToObjectMap(jsonString).entrySet()) { - Object value = entry.getValue(); - builder.writeFieldName(entry.getKey()); - builder.writeStartArray(); - - //sub-maps are transformed in arrays of key-values - //this allows searching for NAME:key=value - if (value instanceof Map) { - for (Map.Entry subEntry : ((Map) value).entrySet()) { - builder.writeString(String.format("%s=%s", String.valueOf(subEntry.getKey()), String.valueOf(subEntry.getValue()))); - } - } else if (value instanceof Object[]) { //arrays to arrays of string - for (Object object : ((Object[]) value)) { - builder.writeString(String.valueOf(object)); - } - } else if (value instanceof Collection) { //Collections to arrays of string - for (Object object : ((Collection) value)) { - builder.writeString(String.valueOf(object)); - } - } else { //single values in their string representations - builder.writeString(String.valueOf(value)); - } - builder.writeEndArray(); - } - - builder.writeEndObject(); - builder.close(); - - return stringWriter.toString(); - } - - /** - * 2016-01-05T13:49:25.143Z - */ - @Nonnull - static String getIso8601Date(@Nonnull Date date) { - SimpleDateFormat dateFormat = new SimpleDateFormat(EXTENDED_ISO8601_FORMAT); - dateFormat.setTimeZone(GMT_ZONE); - return dateFormat.format(date); - } - - @Nonnull - static String stringMapToJson(@Nonnull Map mapValue) throws IOException { - StringWriter stringWriter = new StringWriter(); - JsonGenerator builder = JSON_FACTORY.createJsonGenerator(stringWriter); - - builder.writeStartObject(); - - for (Map.Entry entry : mapValue.entrySet()) { - builder.writeStringField(entry.getKey(), entry.getValue()); - } - - builder.writeEndObject(); - builder.close(); - - return stringWriter.toString(); - } - - @Nonnull - static String collectionToArray(@Nonnull Collection collection) throws IOException { - StringWriter stringWriter = new StringWriter(); - JsonGenerator builder = JSON_FACTORY.createJsonGenerator(stringWriter); - - builder.writeStartArray(); - - for (String entry : collection) { - builder.writeString(entry); - } - - builder.writeEndArray(); - builder.close(); - - return stringWriter.toString(); - } - - /** - * @param result JestResult to extract - * @param keys the key chain to go through - * @return an empty JsonObject if key chain not found - */ - @Nonnull - public static JsonObject getJsonObject(@Nonnull JestResult result, @Nonnull String... keys) { - JsonObject object = result.getJsonObject(); - - for (String key : keys) { - if (object == null) { - return new JsonObject(); - } else { - JsonElement subObject = object.get(key); - object = subObject == null ? null : subObject.getAsJsonObject(); - } - } - - return object == null ? new JsonObject() : object; - } - - @Nullable - public static String getString(@Nullable JsonElement element, @Nonnull String... path) { - if (element == null) { - return null; - } - for (int i = 0; i < path.length; i++) { - String key = path[i]; - if (i + 1 == path.length) { - JsonElement value = element.getAsJsonObject().get(key); - return value == null ? null : value.getAsString(); - } else if (element.isJsonObject()) { - element = element.getAsJsonObject().get(key); - if (element == null || !element.isJsonObject()) { - return null; - } - } else { - return null; - } - } - return null; - } - - /** - * @param jsonObject object to filter - * @param keys keys to remove - * @return a copy as deep as key.length - */ - @Nonnull - static JsonObject filterKeys(@Nonnull JsonObject jsonObject, @Nonnull String... keys) { - JsonObject filtered = jsonObject; - for (String key : keys) { - filtered = filterPath(filtered, key); - } - return filtered; - } - - /** - * @param jsonObject object to filter - * @param path path to the key to remove - * @return a copy as deep as key.length - */ - @Nonnull - static JsonObject filterPath(@Nonnull JsonObject jsonObject, @Nonnull String... path) { - if (path.length == 0) { - return jsonObject; - - } else { - JsonObject result = new JsonObject(); - - jsonObject.entrySet().forEach(e -> { - String key = e.getKey(); - JsonElement value = e.getValue(); - if (path[0].equals(e.getKey())) { - if (path.length > 1) { // if length == 1, this is the key to remove - if (value instanceof JsonObject) { // if value is an object, filter further - value = JsonUtils.filterPath((JsonObject) value, Arrays.copyOfRange(path, 1, path.length)); - } - result.add(key, value); - } - } else { - result.add(key, value); - } - }); - return result; - } - } - - /** - * @param jsonObject object to filter - * @param predicate matching predicate will keep the keys - * @return a shallow copy - */ - @Nonnull - public static JsonObject filter(@Nonnull JsonObject jsonObject, @Nonnull Predicate predicate) { - JsonObject res = new JsonObject(); - - jsonObject.entrySet().forEach(e -> { - if (predicate.test(e.getKey())) { - res.add(e.getKey(), e.getValue()); - } - }); - - return res; - } - - - @Nonnull - static String unQuote(@Nonnull String string) { - return string.replaceAll("\"", ""); - } - - /** - * @param main elements of this json will overwrite the other's params - * @param other will be overwritten by main if same keys exists - * @return null if both are null, if one is null the other is returned - */ - @Nullable - public static JsonObject mergeJson(@Nullable JsonObject main, @Nullable JsonObject other) { - if (other == null) { - return main; - } else if (main == null) { - return other; - } - - JsonObject merged = new JsonObject(); - other.entrySet().forEach(elem -> merged.add(elem.getKey(), elem.getValue())); - main.entrySet().forEach(elem -> merged.add(elem.getKey(), elem.getValue())); - - return merged; - } - - static Long getLong(@Nonnull JsonElement element, @Nonnull String name) { - String value = getString(element, name); - try { - if (value != null) { - return Long.valueOf(value); - } - } catch (NumberFormatException e) { - return null; - } - return null; - } - - public static JsonObject dotedToStructured(JsonObject src) { - JsonObject dest = new JsonObject(); - src.entrySet().forEach(item -> { - JsonObject node = dest; - - for (Iterator it = Arrays.stream(item.getKey().split("\\.")).iterator(); it.hasNext(); ) { - String key = it.next(); - - if (!it.hasNext()) { - node.add(key, item.getValue()); - } else { - if (!node.has(key)) { - node.add(key, new JsonObject()); - } - node = node.getAsJsonObject(key); - } - } - }); - return dest; - } -} +/* + * Copyright 2019 Genesys Telecommunications Laboratories, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.genesyslab.webme.commons.index; + +import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS; +import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_SINGLE_QUOTES; + +import java.io.IOException; +import java.io.StringWriter; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.TimeZone; +import java.util.function.Predicate; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.searchbox.client.JestResult; + +/** + * Created by Jacques-Henri Berthemet on 05/07/2017. + */ +public class JsonUtils { + + private static final String EXTENDED_ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + private static final TimeZone GMT_ZONE = TimeZone.getTimeZone("GMT"); + + private static final ObjectMapper OBJECT_MAPPER; + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + private static final JsonParser GSON_PARSER = new JsonParser(); // com.google.gson.JsonParser + + static { + OBJECT_MAPPER = new ObjectMapper().configure(ALLOW_SINGLE_QUOTES, true); + OBJECT_MAPPER.getJsonFactory().enable(ALLOW_NON_NUMERIC_NUMBERS); + } + + @Nonnull + static Map jsonStringToStringMap(@Nonnull String jsonString) throws IOException { + return OBJECT_MAPPER.readValue(jsonString, new TypeReference>() {}); + } + + @Nonnull + private static Map jsonStringToObjectMap(@Nonnull String jsonString) throws IOException { + return OBJECT_MAPPER.readValue(jsonString, new TypeReference>() {}); + } + + @Nonnull + public static JsonObject asJsonObject(@Nonnull String jsonString) { + return GSON_PARSER.parse(jsonString).getAsJsonObject(); + } + + /** + * Transform the JSON to a map:string,string[] because ES won't support values of different types + * for the same key + */ + @Nonnull + static String flatten(@Nonnull String jsonString) throws IOException { + StringWriter stringWriter = new StringWriter(); + JsonGenerator builder = JSON_FACTORY.createJsonGenerator(stringWriter); + + builder.writeStartObject(); + + for (Map.Entry entry : jsonStringToObjectMap(jsonString).entrySet()) { + Object value = entry.getValue(); + builder.writeFieldName(entry.getKey()); + builder.writeStartArray(); + + // sub-maps are transformed in arrays of key-values + // this allows searching for NAME:key=value + if (value instanceof Map) { + for (Map.Entry subEntry : ((Map) value).entrySet()) { + builder.writeString(String.format("%s=%s", String.valueOf(subEntry.getKey()), String.valueOf(subEntry.getValue()))); + } + } else if (value instanceof Object[]) { // arrays to arrays of string + for (Object object : ((Object[]) value)) { + builder.writeString(String.valueOf(object)); + } + } else if (value instanceof Collection) { // Collections to arrays of string + for (Object object : ((Collection) value)) { + builder.writeString(String.valueOf(object)); + } + } else { // single values in their string representations + builder.writeString(String.valueOf(value)); + } + builder.writeEndArray(); + } + + builder.writeEndObject(); + builder.close(); + + return stringWriter.toString(); + } + + /** + * 2016-01-05T13:49:25.143Z + */ + @Nonnull + static String getIso8601Date(@Nonnull Date date) { + SimpleDateFormat dateFormat = new SimpleDateFormat(EXTENDED_ISO8601_FORMAT); + dateFormat.setTimeZone(GMT_ZONE); + return dateFormat.format(date); + } + + @Nonnull + static String stringMapToJson(@Nonnull Map mapValue) throws IOException { + StringWriter stringWriter = new StringWriter(); + JsonGenerator builder = JSON_FACTORY.createJsonGenerator(stringWriter); + + builder.writeStartObject(); + + for (Map.Entry entry : mapValue.entrySet()) { + builder.writeStringField(entry.getKey(), entry.getValue()); + } + + builder.writeEndObject(); + builder.close(); + + return stringWriter.toString(); + } + + @Nonnull + static String collectionToArray(@Nonnull Collection collection) throws IOException { + StringWriter stringWriter = new StringWriter(); + JsonGenerator builder = JSON_FACTORY.createJsonGenerator(stringWriter); + + builder.writeStartArray(); + + for (String entry : collection) { + builder.writeString(entry); + } + + builder.writeEndArray(); + builder.close(); + + return stringWriter.toString(); + } + + /** + * @param result JestResult to extract + * @param keys the key chain to go through + * @return an empty JsonObject if key chain not found + */ + @Nonnull + public static JsonObject getJsonObject(@Nonnull JestResult result, @Nonnull String... keys) { + JsonObject object = result.getJsonObject(); + + for (String key : keys) { + if (object == null) { + return new JsonObject(); + } else { + JsonElement subObject = object.get(key); + object = subObject == null ? null : subObject.getAsJsonObject(); + } + } + + return object == null ? new JsonObject() : object; + } + + @Nullable + public static String getString(@Nullable JsonElement element, @Nonnull String... path) { + if (element == null) { + return null; + } + for (int i = 0; i < path.length; i++) { + String key = path[i]; + if (i + 1 == path.length) { + JsonElement value = element.getAsJsonObject().get(key); + return value == null ? null : value.getAsString(); + } else if (element.isJsonObject()) { + element = element.getAsJsonObject().get(key); + if (element == null || !element.isJsonObject()) { + return null; + } + } else { + return null; + } + } + return null; + } + + /** + * @param jsonObject object to filter + * @param keys keys to remove + * @return a copy as deep as key.length + */ + @Nonnull + static JsonObject filterKeys(@Nonnull JsonObject jsonObject, @Nonnull String... keys) { + JsonObject filtered = jsonObject; + for (String key : keys) { + filtered = filterPath(filtered, key); + } + return filtered; + } + + /** + * @param jsonObject object to filter + * @param path path to the key to remove + * @return a copy as deep as key.length + */ + @Nonnull + static JsonObject filterPath(@Nonnull JsonObject jsonObject, @Nonnull String... path) { + if (path.length == 0) { + return jsonObject; + + } else { + JsonObject result = new JsonObject(); + + jsonObject.entrySet().forEach(e -> { + String key = e.getKey(); + JsonElement value = e.getValue(); + if (path[0].equals(e.getKey())) { + if (path.length > 1) { // if length == 1, this is the key to remove + if (value instanceof JsonObject) { // if value is an object, filter further + value = JsonUtils.filterPath((JsonObject) value, Arrays.copyOfRange(path, 1, path.length)); + } + result.add(key, value); + } + } else { + result.add(key, value); + } + }); + return result; + } + } + + /** + * @param jsonObject object to filter + * @param predicate matching predicate will keep the keys + * @return a shallow copy + */ + @Nonnull + public static JsonObject filter(@Nonnull JsonObject jsonObject, @Nonnull Predicate predicate) { + JsonObject res = new JsonObject(); + + jsonObject.entrySet().forEach(e -> { + if (predicate.test(e.getKey())) { + res.add(e.getKey(), e.getValue()); + } + }); + + return res; + } + + + @Nonnull + static String unQuote(@Nonnull String string) { + return string.replaceAll("\"", ""); + } + + /** + * @param main elements of this json will overwrite the other's params + * @param other will be overwritten by main if same keys exists + * @return null if both are null, if one is null the other is returned + */ + @Nullable + public static JsonObject mergeJson(@Nullable JsonObject main, @Nullable JsonObject other) { + if (other == null) { + return main; + } else if (main == null) { + return other; + } + + JsonObject merged = new JsonObject(); + other.entrySet().forEach(elem -> merged.add(elem.getKey(), elem.getValue())); + main.entrySet().forEach(elem -> merged.add(elem.getKey(), elem.getValue())); + + return merged; + } + + static Long getLong(@Nonnull JsonElement element, @Nonnull String name) { + String value = getString(element, name); + try { + if (value != null) { + return Long.valueOf(value); + } + } catch (NumberFormatException e) { + return null; + } + return null; + } + + public static JsonObject dotedToStructured(JsonObject src) { + JsonObject dest = new JsonObject(); + src.entrySet().forEach(item -> { + JsonObject node = dest; + + for (Iterator it = Arrays.stream(item.getKey().split("\\.")).iterator(); it.hasNext();) { + String key = it.next(); + + if (!it.hasNext()) { + node.add(key, item.getValue()); + } else { + if (!node.has(key)) { + node.add(key, new JsonObject()); + } + node = node.getAsJsonObject(key); + } + } + }); + return dest; + } +} diff --git a/src/main/java/com/genesyslab/webme/commons/index/config/IndexConfiguration.java b/src/main/java/com/genesyslab/webme/commons/index/config/IndexConfiguration.java index e52a0cd..e800a90 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/config/IndexConfiguration.java +++ b/src/main/java/com/genesyslab/webme/commons/index/config/IndexConfiguration.java @@ -54,7 +54,7 @@ public class IndexConfiguration implements IndexConfig { private final OptionReader reader; public IndexConfiguration(@Nonnull String name, @Nonnull Map cassandraOptions) { - this.reader = new OptionReaderImpl(name, cassandraOptions); + this.reader = new OptionReader(name, cassandraOptions); } @Override diff --git a/src/main/java/com/genesyslab/webme/commons/index/config/OptionReader.java b/src/main/java/com/genesyslab/webme/commons/index/config/OptionReader.java index 2bb61e3..521ddc3 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/config/OptionReader.java +++ b/src/main/java/com/genesyslab/webme/commons/index/config/OptionReader.java @@ -1,38 +1,211 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index.config; +import static org.apache.commons.lang3.StringUtils.isBlank; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.utils.FBUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; + /** - * Reads options for es-index and override Cassandra options. + * This reads options from es-index.properties, override Cassandra options is such file is found and + * provide per dc/rack reading + * + * @author Jacques-Henri Berthemet */ -public interface OptionReader { - @Nonnull - Map getOptions(); +public class OptionReader { + private static final Logger LOGGER = LoggerFactory.getLogger(OptionReader.class); + + private static final String CLASSPATH_PREFIX = "classpath:"; + private static final String CFG_FILE_KEY = IndexConfig.ES_CONFIG_PREFIX + IndexConfig.ES_FILE; + private static final String[] FILES = {"/es-index.properties", "es-index.properties"}; + private static final String[] FOLDERS = {".", "./conf/", "../conf/", "./bin/"}; + + private final String dcName = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); + private final String rackName = DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddressAndPort()); + private final String indexName; + private Map options = new HashMap<>(); + + public OptionReader(@Nonnull String indexName, @Nonnull Map options) { + this.indexName = indexName; + reload(options); + } + + private static String findFile(@Nonnull String path, @Nonnull String... names) { + for (String name : names) { + @SuppressWarnings("resource") + InputStream defaultFile = OptionReader.class.getResourceAsStream(name); + if (defaultFile != null) { + String foundFile = CLASSPATH_PREFIX + name; + try { + defaultFile.close(); + } catch (IOException e) { + LOGGER.error("Can't close {}", name, e); + } + return foundFile; + + } + + File file = new File(path + name); + if (file.exists()) { + return file.getAbsolutePath(); + } + } + return null; + } + + @Nonnull + public Map getOptions() { + return options; + } + + public boolean reload(@Nonnull Map cassandraOptions) { + Map newOptions = new HashMap<>(cassandraOptions); + + Map fileOptions = loadFromFile(); + if (fileOptions != null) { + newOptions.putAll(fileOptions); + } + + if (newOptions.equals(this.options)) { + return false; + } else { + MapDifference diff = Maps.difference(this.options, newOptions); + LOGGER.warn("Reloaded {} options changed: \n\tadded:{} \n\tremoved:{} \n\tchanged:{}", indexName, diff.entriesOnlyOnRight(), + diff.entriesOnlyOnLeft(), diff.entriesDiffering()); + this.options = newOptions; + return true; + } + } + + public boolean getBoolean(@Nonnull String key, boolean defValue) { + return Boolean.parseBoolean(getString(key, String.valueOf(defValue))); + } + + public int getInteger(@Nonnull String key, int defValue) { + String value = getString(key, null); + if (value == null) { + return defValue; + } + + try { + return Integer.parseInt(value); + } catch (NumberFormatException ex) { + LOGGER.warn("{} option {} has invalid value {} using default {}", indexName, key, value, defValue); + return defValue; + } + } + + @Nullable + public String getString(@Nonnull String key, @Nullable String defValue) { + String value = get("<" + dcName + "." + rackName + ">." + key); // Try specific dc/rack + if (value != null) { + return value; + } + + value = get(dcName + "." + rackName + "." + key); // Try specific dc/rack, keep GWE compatibility + if (value != null) { + return value; + } + + value = get("<" + dcName + ">." + key); // Try specific dc + if (value != null) { + return value; + } + + value = get(dcName + "." + key); // Try specific dc, keep GWE compatibility + if (value != null) { + return value; + } + + value = get(key); + if (value != null) { + return value; + } + + return defValue; + } + + @Nullable + private String get(String key) { + String value = System.getProperty(IndexConfig.ES_CONFIG_PREFIX + key); // Support for sysprops + if (isBlank(value)) { + value = System.getenv(IndexConfig.ES_CONFIG_PREFIX + key); // Support for env vars + } + + if (isBlank(value)) { + value = options.getOrDefault(key, options.get(key.replace('-', '.'))); // Try hyphen format then try in doted format + } + + return isBlank(value) ? null : value; + } + + @Nullable + private Map loadFromFile() { + String cfgFile = System.getProperty(CFG_FILE_KEY); + if (cfgFile == null) { + cfgFile = System.getProperty(CFG_FILE_KEY.replace('-', '.')); + } + + if (cfgFile == null) { + for (String folder : FOLDERS) { + cfgFile = findFile(folder, FILES); + if (cfgFile != null) { + LOGGER.info("Found default configuration file '{}'", cfgFile); + break; + } + } + } + + if (cfgFile == null) { + return null; + } + + boolean fromCp = cfgFile.startsWith(CLASSPATH_PREFIX); + String filePath = fromCp ? cfgFile.substring(CLASSPATH_PREFIX.length(), cfgFile.length()) : cfgFile; - boolean reload(@Nonnull Map cassandraOptions); + Map fileOptions = new HashMap<>(); + try (InputStream ios = fromCp ? this.getClass().getResourceAsStream(filePath) : new FileInputStream(new File(filePath))) { - boolean getBoolean(@Nonnull String key, boolean defValue); + Properties props = new Properties(); + props.load(ios); - int getInteger(@Nonnull String key, int defValue); + for (Entry en : props.entrySet()) { + fileOptions.put(String.valueOf(en.getKey()), String.valueOf(en.getValue())); + } - @Nullable - String getString(@Nonnull String key, @Nullable String defValue); + return fileOptions; + } catch (IOException e) { + LOGGER.error("Can't read file '{}' {}", filePath, e.getMessage(), e); + throw new RuntimeException("Index option file read exception", e); + } + } } diff --git a/src/main/java/com/genesyslab/webme/commons/index/config/OptionReaderImpl.java b/src/main/java/com/genesyslab/webme/commons/index/config/OptionReaderImpl.java deleted file mode 100644 index 82ff26c..0000000 --- a/src/main/java/com/genesyslab/webme/commons/index/config/OptionReaderImpl.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright 2019 Genesys Telecommunications Laboratories, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.genesyslab.webme.commons.index.config; - -import com.google.common.collect.MapDifference; -import com.google.common.collect.Maps; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.utils.FBUtilities; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; - -import static org.apache.commons.lang3.StringUtils.isBlank; - -/** - * This reads options from es-index.properties, override Cassandra options if such file is found and - * provide per dc/rack reading - * - * @author Jacques-Henri Berthemet - */ -public class OptionReaderImpl implements OptionReader { - private static final Logger LOGGER = LoggerFactory.getLogger(OptionReaderImpl.class); - - private static final String CLASSPATH_PREFIX = "classpath:"; - private static final String CFG_FILE_KEY = IndexConfig.ES_CONFIG_PREFIX + IndexConfig.ES_FILE; - private static final String[] FILES = {"/es-index.properties", "es-index.properties"}; - private static final String[] FOLDERS = {".", "./conf/", "../conf/", "./bin/"}; - - private final String dcName = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress()); - private final String rackName = DatabaseDescriptor.getEndpointSnitch().getRack(FBUtilities.getBroadcastAddress()); - private final String indexName; - private Map options = new HashMap<>(); - - public OptionReaderImpl(@Nonnull String indexName, @Nonnull Map options) { - this.indexName = indexName; - reload(options); - } - - private static String findFile(@Nonnull String path, @Nonnull String... names) { - for (String name : names) { - @SuppressWarnings("resource") - InputStream defaultFile = OptionReaderImpl.class.getResourceAsStream(name); - if (defaultFile != null) { - String foundFile = CLASSPATH_PREFIX + name; - try { - defaultFile.close(); - } catch (IOException e) { - LOGGER.error("Can't close {}", name, e); - } - return foundFile; - - } - - File file = new File(path + name); - if (file.exists()) { - return file.getAbsolutePath(); - } - } - return null; - } - - @Override - @Nonnull - public Map getOptions() { - return options; - } - - @Override - public boolean reload(@Nonnull Map cassandraOptions) { - Map newOptions = new HashMap<>(cassandraOptions); - - Map fileOptions = loadFromFile(); - if (fileOptions != null) { - newOptions.putAll(fileOptions); - } - - if (newOptions.equals(this.options)) { - return false; - } else { - MapDifference diff = Maps.difference(this.options, newOptions); - LOGGER.warn("Reloaded {} options changed: \n\tadded:{} \n\tremoved:{} \n\tchanged:{}", indexName, diff.entriesOnlyOnRight(), - diff.entriesOnlyOnLeft(), diff.entriesDiffering()); - this.options = newOptions; - return true; - } - } - - @Override - public boolean getBoolean(@Nonnull String key, boolean defValue) { - return Boolean.parseBoolean(getString(key, String.valueOf(defValue))); - } - - @Override - public int getInteger(@Nonnull String key, int defValue) { - String value = getString(key, null); - if (value == null) { - return defValue; - } - - try { - return Integer.parseInt(value); - } catch (NumberFormatException ex) { - LOGGER.warn("{} option {} has invalid value {} using default {}", indexName, key, value, defValue); - return defValue; - } - } - - @Override - @Nullable - public String getString(@Nonnull String key, @Nullable String defValue) { - String value = get("<" + dcName + "." + rackName + ">." + key); // Try specific dc/rack - if (value != null) { - return value; - } - - value = get(dcName + "." + rackName + "." + key); // Try specific dc/rack, keep GWE compatibility - if (value != null) { - return value; - } - - value = get("<" + dcName + ">." + key); // Try specific dc - if (value != null) { - return value; - } - - value = get(dcName + "." + key); // Try specific dc, keep GWE compatibility - if (value != null) { - return value; - } - - value = get(key); - if (value != null) { - return value; - } - - return defValue; - } - - @Nullable - private String get(String key) { - String value = System.getProperty(IndexConfig.ES_CONFIG_PREFIX + key); // Support for sysprops - if (isBlank(value)) { - value = System.getenv(IndexConfig.ES_CONFIG_PREFIX + key); // Support for env vars - } - - if (isBlank(value)) { - value = options.getOrDefault(key, options.get(key.replace('-', '.'))); // Try hyphen format then try in doted format - } - - return isBlank(value) ? null : value; - } - - @Nullable - private Map loadFromFile() { - String cfgFile = System.getProperty(CFG_FILE_KEY); - if (cfgFile == null) { - cfgFile = System.getProperty(CFG_FILE_KEY.replace('-', '.')); - } - - if (cfgFile == null) { - for (String folder : FOLDERS) { - cfgFile = findFile(folder, FILES); - if (cfgFile != null) { - LOGGER.info("Found default configuration file '{}'", cfgFile); - break; - } - } - } - - if (cfgFile == null) { - return null; - } - - boolean fromCp = cfgFile.startsWith(CLASSPATH_PREFIX); - String filePath = fromCp ? cfgFile.substring(CLASSPATH_PREFIX.length()) : cfgFile; - - Map fileOptions = new HashMap<>(); - try (InputStream ios = fromCp ? this.getClass().getResourceAsStream(filePath) : new FileInputStream(new File(filePath))) { - - Properties props = new Properties(); - props.load(ios); - - for (Entry en : props.entrySet()) { - fileOptions.put(String.valueOf(en.getKey()), String.valueOf(en.getValue())); - } - - return fileOptions; - } catch (IOException e) { - LOGGER.error("Can't read file '{}' {}", filePath, e.getMessage(), e); - throw new RuntimeException("Index option file read exception", e); - } - } -} diff --git a/src/main/java/com/genesyslab/webme/commons/index/indexers/FakePartitionIterator.java b/src/main/java/com/genesyslab/webme/commons/index/indexers/FakePartitionIterator.java index 2d35262..39b6116 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/indexers/FakePartitionIterator.java +++ b/src/main/java/com/genesyslab/webme/commons/index/indexers/FakePartitionIterator.java @@ -1,48 +1,45 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index.indexers; -import com.genesyslab.webme.commons.index.EsSecondaryIndex; -import com.genesyslab.webme.commons.index.JsonUtils; -import com.genesyslab.webme.commons.index.SearchResult; -import com.genesyslab.webme.commons.index.SearchResultRow; +import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.gson.JsonObject; +import java.nio.ByteBuffer; +import java.util.Iterator; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.LivenessInfo; -import org.apache.cassandra.db.PartitionColumns; import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.rows.BTreeRow; import org.apache.cassandra.db.rows.BufferCell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import java.nio.ByteBuffer; -import java.util.Iterator; - -import static java.nio.charset.StandardCharsets.UTF_8; +import com.genesyslab.webme.commons.index.EsSecondaryIndex; +import com.genesyslab.webme.commons.index.JsonUtils; +import com.genesyslab.webme.commons.index.SearchResult; +import com.genesyslab.webme.commons.index.SearchResultRow; +import com.google.gson.JsonObject; /** * This iterator does not load data from Cassandra and will only return PKs and ES metadata @@ -51,14 +48,14 @@ */ public class FakePartitionIterator implements UnfilteredPartitionIterator { - private static final String FAKE_METADATA = "{\"metadata\":\"none\"}"; //so that it's always json + private static final String FAKE_METADATA = "{\"metadata\":\"none\"}"; // so that it's always json private final Iterator esResultIterator; private final ColumnFamilyStore baseCfs; private final ReadCommand command; private final String searchId; - private final ColumnDefinition indexColDef; - private final PartitionColumns returnedColumns; + private final ColumnMetadata indexColDef; + private final RegularAndStaticColumns returnedColumns; private final JsonObject searchResultMetadata; private final boolean metadataRequested; private boolean isFirst = true; @@ -69,19 +66,14 @@ public FakePartitionIterator(EsSecondaryIndex index, SearchResult searchResult, this.command = command; this.searchId = searchId; this.indexColDef = index.getIndexColDef(); - this.returnedColumns = PartitionColumns.builder().add(indexColDef).build(); //can build once, it's always the same + this.returnedColumns = RegularAndStaticColumns.builder().add(indexColDef).build(); // can build once, it's always the same this.searchResultMetadata = searchResult.metadata; this.metadataRequested = command.columnFilter().queriedColumns().contains(indexColDef); Tracing.trace("ESI {} FakePartitionIterator initialized", searchId); } @Override - public boolean isForThrift() { - return command.isForThrift(); - } - - @Override - public CFMetaData metadata() { + public TableMetadata metadata() { return command.metadata(); } @@ -101,17 +93,17 @@ public UnfilteredRowIterator next() { return null; } - //Build the minimum row - Row.Builder rowBuilder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds()); - rowBuilder.newRow(Clustering.EMPTY); //FIXME support for clustering + // Build the minimum row + Row.Builder rowBuilder = BTreeRow.unsortedBuilder(); + rowBuilder.newRow(Clustering.EMPTY); // FIXME support for clustering rowBuilder.addPrimaryKeyLivenessInfo(LivenessInfo.EMPTY); rowBuilder.addRowDeletion(Row.Deletion.LIVE); SearchResultRow esResult = esResultIterator.next(); - if (metadataRequested) { //Add the metadata cell + if (metadataRequested) { // Add the metadata cell JsonObject jsonMetadata = esResult.docMetadata; - if (isFirst) { //only first result have global metadata + if (isFirst) { // only first result have global metadata jsonMetadata = JsonUtils.mergeJson(searchResultMetadata, jsonMetadata); isFirst = false; } @@ -121,8 +113,8 @@ public UnfilteredRowIterator next() { rowBuilder.addCell(metadataCell); } - //And PK value + // And PK value DecoratedKey partitionKey = baseCfs.getPartitioner().decorateKey(esResult.partitionKey); - return new SingleRowIterator(baseCfs.metadata, rowBuilder.build(), partitionKey, returnedColumns); + return new SingleRowIterator(baseCfs.metadata(), rowBuilder.build(), partitionKey, returnedColumns); } } diff --git a/src/main/java/com/genesyslab/webme/commons/index/indexers/SingleRowIterator.java b/src/main/java/com/genesyslab/webme/commons/index/indexers/SingleRowIterator.java index e7b285d..cb0241b 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/indexers/SingleRowIterator.java +++ b/src/main/java/com/genesyslab/webme/commons/index/indexers/SingleRowIterator.java @@ -1,29 +1,27 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index.indexers; -import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; -import org.apache.cassandra.db.PartitionColumns; +import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator; import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.Rows; import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.schema.TableMetadata; /** * @author Jacques-Henri Berthemet 21/07/2017 @@ -32,21 +30,21 @@ public class SingleRowIterator extends AbstractUnfilteredRowIterator { private Unfiltered row; - public SingleRowIterator(CFMetaData metadata, Unfiltered row, DecoratedKey key, PartitionColumns columns) { + public SingleRowIterator(TableMetadata metadata, Unfiltered row, DecoratedKey key, RegularAndStaticColumns columns) { super(metadata, key, DeletionTime.LIVE, columns, Rows.EMPTY_STATIC_ROW, false, EncodingStats.NO_STATS); this.row = row; } public SingleRowIterator(RowIterator partition, Unfiltered row) { super(partition.metadata(), partition.partitionKey(), DeletionTime.LIVE, partition.columns(), partition.staticRow(), - partition.isReverseOrder(), EncodingStats.NO_STATS); + partition.isReverseOrder(), EncodingStats.NO_STATS); this.row = row; } @Override protected synchronized Unfiltered computeNext() { try { - return row == null ? endOfData() : row; //we have to return endOfData() when we're done + return row == null ? endOfData() : row; // we have to return endOfData() when we're done } finally { row = null; } diff --git a/src/main/java/com/genesyslab/webme/commons/index/indexers/StreamingPartitionIterator.java b/src/main/java/com/genesyslab/webme/commons/index/indexers/StreamingPartitionIterator.java index 4029c05..6d3cdb7 100644 --- a/src/main/java/com/genesyslab/webme/commons/index/indexers/StreamingPartitionIterator.java +++ b/src/main/java/com/genesyslab/webme/commons/index/indexers/StreamingPartitionIterator.java @@ -1,29 +1,24 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index.indexers; -import com.genesyslab.webme.commons.index.EsSecondaryIndex; -import com.genesyslab.webme.commons.index.JsonUtils; -import com.genesyslab.webme.commons.index.SearchResult; -import com.genesyslab.webme.commons.index.SearchResultRow; +import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.gson.JsonObject; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.util.Iterator; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; @@ -38,6 +33,8 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; @@ -45,14 +42,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.util.Iterator; - -import static java.nio.charset.StandardCharsets.UTF_8; +import com.genesyslab.webme.commons.index.EsSecondaryIndex; +import com.genesyslab.webme.commons.index.JsonUtils; +import com.genesyslab.webme.commons.index.SearchResult; +import com.genesyslab.webme.commons.index.SearchResultRow; +import com.google.gson.JsonObject; /** - * This a partition iterator that will read a row each time next() is called, should be the lightest on resources but maybe the slowest.
+ * This a partition iterator that will read a row each time next() is called, should be the lightest + * on resources but maybe the slowest.
* This is the equivalent of the sync read mode *

* Created by Jacques-Henri Berthemet on 11/07/2017. @@ -66,14 +64,13 @@ public class StreamingPartitionIterator implements UnfilteredPartitionIterator { private final PartitionRangeReadCommand command; private final String searchId; private final EsSecondaryIndex index; - private final ColumnDefinition indexColDef; + private final ColumnMetadata indexColDef; private final ConsistencyLevel consistencyLevel; private final JsonObject searchResultMetadata; private final boolean metadataRequested; private boolean isFirst = true; - public StreamingPartitionIterator(EsSecondaryIndex index, SearchResult searchResult, - PartitionRangeReadCommand command, String searchId) { + public StreamingPartitionIterator(EsSecondaryIndex index, SearchResult searchResult, PartitionRangeReadCommand command, String searchId) { this.baseCfs = index.baseCfs; this.esResultIterator = searchResult.items.iterator(); this.command = command; @@ -87,12 +84,7 @@ public StreamingPartitionIterator(EsSecondaryIndex index, SearchResult searchRes } @Override - public boolean isForThrift() { - return command.isForThrift(); - } - - @Override - public CFMetaData metadata() { + public TableMetadata metadata() { return command.metadata(); } @@ -118,20 +110,20 @@ public UnfilteredRowIterator next() { jsonMetadata = esResult.docMetadata; DecoratedKey partitionKey = baseCfs.getPartitioner().decorateKey(esResult.partitionKey); - SinglePartitionReadCommand readCommand = SinglePartitionReadCommand.create( - baseCfs.metadata, - command.nowInSec(), - command.columnFilter(), //columns that will be returned - RowFilter.NONE, //don't filter anything, as we pass token(id) it may prevent loading non local rows - DataLimits.NONE, //don't use command DataLimits because we are only loading one partition - partitionKey, - command.clusteringIndexFilter(partitionKey)); - - //Cassandra has below method but not DSE: - // PartitionIterator partition = readCommand.execute(consistencyLevel, ClientState.forInternalCalls(), System.nanoTime()); - // WCC-1131 Call directly this method for it is available both in open-source cassandra and in Datastax Enterprise + SinglePartitionReadCommand readCommand = + SinglePartitionReadCommand.create(baseCfs.metadata(), command.nowInSec(), command.columnFilter(), // columns that will be + // returned + RowFilter.NONE, // don't filter anything, as we pass token(id) it may prevent loading non local rows + DataLimits.NONE, // don't use command DataLimits because we are only loading one partition + partitionKey, command.clusteringIndexFilter(partitionKey)); + + // Cassandra has below method but not DSE: + // PartitionIterator partition = readCommand.execute(consistencyLevel, + // ClientState.forInternalCalls(), System.nanoTime()); + // WCC-1131 Call directly this method for it is available both in open-source cassandra and in + // Datastax Enterprise PartitionIterator partition = - StorageProxy.read(SinglePartitionReadCommand.Group.one(readCommand), consistencyLevel, System.nanoTime()); + StorageProxy.read(SinglePartitionReadCommand.Group.one(readCommand), consistencyLevel, System.nanoTime()); if (!partition.hasNext()) { logRowNotFound(partitionKey); @@ -144,35 +136,35 @@ public UnfilteredRowIterator next() { continue; } - row = rowIterator.next(); //FIXME clustered partitions will contain several rows + row = rowIterator.next(); // FIXME clustered partitions will contain several rows } - if (row == null) { //if all ES results were expired + if (row == null) { // if all ES results were expired return null; } - if (metadataRequested) { //Rewrite only if metadata requested - if (isFirst) { //only first result have global metadata + if (metadataRequested) { // Rewrite only if metadata requested + if (isFirst) { // only first result have global metadata jsonMetadata = JsonUtils.mergeJson(searchResultMetadata, jsonMetadata); isFirst = false; } if (jsonMetadata != null) { int now = FBUtilities.nowInSeconds(); - Row.Builder rowBuilder = BTreeRow.unsortedBuilder(now); + Row.Builder rowBuilder = BTreeRow.unsortedBuilder(); - rowBuilder.newRow(row.clustering()); //need to be first + rowBuilder.newRow(row.clustering()); // need to be first rowBuilder.addPrimaryKeyLivenessInfo(row.primaryKeyLivenessInfo()); rowBuilder.addRowDeletion(row.deletion()); - //copy existing cells + // copy existing cells row.cells().forEach(cell -> { if (!index.indexColumnName.equals(cell.column().name.toString())) { rowBuilder.addCell(cell); } }); - //add metadata cell + // add metadata cell ByteBuffer value = ByteBufferUtil.bytes(jsonMetadata.toString(), UTF_8); BufferCell metadataCell = BufferCell.live(indexColDef, now, value); @@ -181,6 +173,7 @@ public UnfilteredRowIterator next() { } } + LOGGER.debug("Row read:" + row.toString(rowIterator.metadata(), true)); return new SingleRowIterator(rowIterator, row); } diff --git a/src/main/java/org/apache/cassandra/db/filter/EsSimpleExpression.java b/src/main/java/org/apache/cassandra/db/filter/EsSimpleExpression.java new file mode 100644 index 0000000..0fb0c6d --- /dev/null +++ b/src/main/java/org/apache/cassandra/db/filter/EsSimpleExpression.java @@ -0,0 +1,30 @@ +package org.apache.cassandra.db.filter; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.filter.RowFilter.Expression; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.schema.TableMetadata; + +public class EsSimpleExpression extends RowFilter.SimpleExpression { + private Expression original; + + public EsSimpleExpression(Expression expression) { + super(expression.column, expression.operator, expression.value); + this.original = expression; + } + + @Override + public boolean isSatisfiedBy(TableMetadata metadata, DecoratedKey partitionKey, Row row) { + return true; + } + + @Override + public String toString() { + return original.toString(); + } + + @Override + protected Kind kind() { + return original.kind(); + } +} diff --git a/src/test/java/com/genesyslab/webme/commons/index/ElasticIndexConfigTest.java b/src/test/java/com/genesyslab/webme/commons/index/ElasticIndexConfigTest.java index ea98061..2c62928 100644 --- a/src/test/java/com/genesyslab/webme/commons/index/ElasticIndexConfigTest.java +++ b/src/test/java/com/genesyslab/webme/commons/index/ElasticIndexConfigTest.java @@ -1,91 +1,91 @@ -/* - * Copyright 2019 Genesys Telecommunications Laboratories, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.genesyslab.webme.commons.index; - -import com.genesyslab.webme.commons.index.config.IndexConfig; -import com.genesyslab.webme.commons.index.config.IndexConfiguration; -import com.genesyslab.webme.commons.index.requests.ElasticClientFactory; -import com.genesyslab.webme.commons.index.test.JestClientFactoryMock; -import com.genesyslab.webme.commons.index.test.JestClientMock; - -import com.google.gson.JsonObject; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import io.searchbox.client.JestResult; - -import java.util.HashMap; -import java.util.Map; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.core.IsCollectionContaining.hasItems; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author Jacques-Henri Berthemet 25/10/2018 - */ -public class ElasticIndexConfigTest { - - private static final String KEYSPACE_NAME = "testKeyspace"; - private static final String TABLE_NAME = "testTable"; - - @BeforeClass - public static void setupClient() { - EsSecondaryIndexUnderTest.staticInit(); - ElasticClientFactory.setJestClientFactory(JestClientFactoryMock.INSTANCE); - } - - @AfterClass - public static void cleanup() { - JestClientMock.clear(); - ElasticClientFactory.setJestClientFactory(null); - } - - - @Test - public void shouldSetEsHostNames() { - JestResult cluster = mock(JestResult.class); - when(cluster.isSucceeded()).thenReturn(true); - JestClientMock.addResponse(cluster); - - JestResult getVersion = mock(JestResult.class); - when(getVersion.isSucceeded()).thenReturn(true); - JsonObject jsonIndex = new JsonObject(); - JsonObject version = new JsonObject(); - version.addProperty("number", "7.9.2"); - jsonIndex.add("version", version); - when(getVersion.getJsonObject()).thenReturn(jsonIndex); - JestClientMock.addResponse(getVersion); - - Map options = new HashMap<>(); - options.put(".unicast-hosts", "jupiter,mars"); - IndexConfig indexConfig = new IndexConfiguration(TABLE_NAME, options); - ElasticIndex indexUnderTest = new ElasticIndex(indexConfig, KEYSPACE_NAME, TABLE_NAME, singletonList("Id"), emptyList()); - indexUnderTest.init(); - - assertThat(JestClientFactoryMock.httpConfig.getServerList(), hasItems("http://jupiter:9200", "http://mars:9200")); - assertThat(JestClientMock.receivedRequests.size(), is(3)); - assertThat(JestClientMock.receivedRequests.get(0).toString(), - is("Health{uri=/_cluster/health/_all?wait_for_status=yellow, method=GET}")); - assertThat(JestClientMock.receivedRequests.get(2).toString(), is("IndicesExists{uri=testkeyspace_testtable_index%40, method=HEAD}")); - } -} +/* + * Copyright 2019 Genesys Telecommunications Laboratories, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.genesyslab.webme.commons.index; + +import com.genesyslab.webme.commons.index.config.IndexConfig; +import com.genesyslab.webme.commons.index.config.IndexConfiguration; +import com.genesyslab.webme.commons.index.requests.ElasticClientFactory; +import com.genesyslab.webme.commons.index.test.JestClientFactoryMock; +import com.genesyslab.webme.commons.index.test.JestClientMock; + +import com.google.gson.JsonObject; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import io.searchbox.client.JestResult; + +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.core.IsCollectionContaining.hasItems; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author Jacques-Henri Berthemet 25/10/2018 + */ +public class ElasticIndexConfigTest { + + private static final String KEYSPACE_NAME = "testKeyspace"; + private static final String TABLE_NAME = "testTable"; + + @BeforeClass + public static void setupClient() { + EsSecondaryIndexUnderTest.staticInit(); + ElasticClientFactory.setJestClientFactory(JestClientFactoryMock.INSTANCE); + } + + @AfterClass + public static void cleanup() { + JestClientMock.clear(); + ElasticClientFactory.setJestClientFactory(null); + } + + + @Test + public void shouldSetEsHostNames() { + JestResult cluster = mock(JestResult.class); + when(cluster.isSucceeded()).thenReturn(true); + JestClientMock.addResponse(cluster); + + JestResult getVersion = mock(JestResult.class); + when(getVersion.isSucceeded()).thenReturn(true); + JsonObject jsonIndex = new JsonObject(); + JsonObject version = new JsonObject(); + version.addProperty("number", "7.9.2"); + jsonIndex.add("version", version); + when(getVersion.getJsonObject()).thenReturn(jsonIndex); + JestClientMock.addResponse(getVersion); + + Map options = new HashMap<>(); + options.put(".unicast-hosts", "jupiter,mars"); + IndexConfig indexConfig = new IndexConfiguration(TABLE_NAME, options); + ElasticIndex indexUnderTest = new ElasticIndex(indexConfig, KEYSPACE_NAME, TABLE_NAME, singletonList("Id"), emptyList()); + indexUnderTest.init(); + + assertThat(JestClientFactoryMock.httpConfig.getServerList(), hasItems("http://jupiter:9200", "http://mars:9200")); + assertThat(JestClientMock.receivedRequests.size(), is(3)); + assertThat(JestClientMock.receivedRequests.get(0).toString(), + is("Health{uri=/_cluster/health/_all?wait_for_status=yellow, method=GET}")); + assertThat(JestClientMock.receivedRequests.get(2).toString(), is("IndicesExists{uri=testkeyspace_testtable_index%40, method=HEAD}")); + } +} diff --git a/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexTest.java b/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexTest.java index b932b5c..4346133 100644 --- a/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexTest.java +++ b/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexTest.java @@ -1,180 +1,162 @@ -/* - * Copyright 2019 Genesys Telecommunications Laboratories, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.genesyslab.webme.commons.index; - -import com.genesyslab.webme.commons.index.config.IndexConfig.Segment; -import com.genesyslab.webme.commons.index.requests.ElasticClientFactory; -import com.genesyslab.webme.commons.index.test.JestClientFactoryMock; -import com.genesyslab.webme.commons.index.test.JestClientMock; - -import com.google.gson.JsonObject; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.PartitionRangeReadCommand; -import org.apache.cassandra.db.PreHashedDecoratedKey; -import org.apache.cassandra.db.filter.RowFilter; -import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.exceptions.IsBootstrappingException; -import org.apache.cassandra.exceptions.ReadFailureException; -import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.exceptions.UnavailableException; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import io.searchbox.client.JestResult; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Collections.emptyList; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author Vincent Pirat 01/12/2016 - */ -public class EsSecondaryIndexTest { - - private static EsSecondaryIndexUnderTest secondaryIndex; - - @BeforeClass - public static void setup() throws Exception { - System.setProperty("genesys-es-unicast-hosts", ""); - System.setProperty("genesys-es-.unicast-hosts", "https://mars:2048"); - DatabaseDescriptor.clientInitialization(false); - ElasticClientFactory.setJestClientFactory(JestClientFactoryMock.INSTANCE); - JestResult cluster = mock(JestResult.class); - when(cluster.isSucceeded()).thenReturn(true); - JestClientMock.addResponse(cluster); - - JestResult getVersion = mock(JestResult.class); - when(getVersion.isSucceeded()).thenReturn(true); - JsonObject jsonIndex = new JsonObject(); - JsonObject version = new JsonObject(); - version.addProperty("number", "7.9.2"); - jsonIndex.add("version", version); - when(getVersion.getJsonObject()).thenReturn(jsonIndex); - JestClientMock.addResponse(getVersion); - secondaryIndex = new EsSecondaryIndexUnderTest(); - } - - @AfterClass - public static void cleanup() { - JestClientMock.clear(); - ElasticClientFactory.setJestClientFactory(null); - System.setProperty("genesys-es-.unicast-hosts", ""); - } - - @Before - public void clean() { - JestClientMock.clear(); - } - - @Test - public void shouldSetCustomConfig() throws ReadTimeoutException, ReadFailureException, UnavailableException, IsBootstrappingException, - InvalidRequestException { - - assertThat(secondaryIndex.name, is("EsSecondaryIndex [demo.testindex]")); - - String options = "#update#{" - + "\"read-consistency-level\":\"LOCAL_QUORUM\"," - + "\"insert-only\":\"false\"," - + "\"discard-nulls\":\"false\"," - + "\"async-search\":\"true\"," - + "\"async-write\":\"false\"," - + "\"class_name\": \"com.genesyslab.webme.commons.index.EsSecondaryIndex\"," - + "\"segment\": \"CUSTOM\"," - + "\"segment-name\": \"segmentName\"," - + "\"index-properties\": \"{" - + "\\\"script.disable_dynamic\\\":\\\"false\\\"," - + "\\\"index.refresh_interval\\\":\\\"1s\\\"," - + "\\\"analysis.analyzer.html_analyzer.char_filter\\\":\\\"html_strip\\\"," - + "\\\"analysis.analyzer.html_analyzer.type\\\":\\\"custom\\\"," - + "\\\"analysis.analyzer.html_analyzer.filter\\\":\\\"standard\\\"," - + "\\\"analysis.analyzer.html_analyzer.tokenizer\\\":\\\"standard\\\"}\"}#"; - - RowFilter rowFilter = RowFilter.create(); - rowFilter.addCustomIndexExpression(EsSecondaryIndexUnderTest.cfMetaData, EsSecondaryIndexUnderTest.indexMetadata, - ByteBufferUtil.bytes(options, UTF_8)); - - PartitionRangeReadCommand command = mock(PartitionRangeReadCommand.class); - when(command.rowFilter()).thenReturn(rowFilter); - - try { - secondaryIndex.search(command); - } catch (RuntimeException e) { - //If we went as far as this it means we reached MigrationManager ! - assertEquals("java.util.concurrent.ExecutionException: java.lang.AssertionError: Unknown keyspace system_schema", e.getMessage()); - - //Little workaround to set the metadata column with the new index value. - secondaryIndex.reloadSettings(); - } - - assertThat(secondaryIndex.esIndex, is(notNullValue())); - assertThat(secondaryIndex.indexConfig.isInsertOnly(), is(false)); - assertThat(secondaryIndex.indexConfig.getIndexSegmentName(), is("segmentName")); - assertThat(secondaryIndex.indexConfig.getIndexSegment(), is(Segment.CUSTOM)); - assertThat(secondaryIndex.indexConfig.getReadConsistencyLevel(), is(ConsistencyLevel.LOCAL_QUORUM)); - assertThat(secondaryIndex.indexConfig.getMaxResults(), is(10000)); - assertThat(secondaryIndex.indexConfig.getUnicastHosts(), is("https://mars:2048")); - assertThat(secondaryIndex.indexConfig.isConcurrentLock(), is(true)); - assertThat(secondaryIndex.indexConfig.isSkipNonLocalUpdates(), is(true)); - assertThat(secondaryIndex.indexConfig.isAsyncWrite(), is(false)); - assertThat(secondaryIndex.indexConfig.getJsonFlatSerializedFields().size(), is(0)); - assertThat(secondaryIndex.indexConfig.getJsonSerializedFields().size(), is(0)); - - assertThat(secondaryIndex.indexConfig.getIndexOptions().size(), greaterThanOrEqualTo(10)); - - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("index-properties"), is("{" - + "\"script.disable_dynamic\":\"false\"," - + "\"index.refresh_interval\":\"1s\"," - + "\"analysis.analyzer.html_analyzer.char_filter\":\"html_strip\"," - + "\"analysis.analyzer.html_analyzer.type\":\"custom\"," - + "\"analysis.analyzer.html_analyzer.filter\":\"standard\"," - + "\"analysis.analyzer.html_analyzer.tokenizer\":\"standard\"}")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("segment"), is("CUSTOM")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("segment-name"), is("segmentName")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("class_name"), is("com.genesyslab.webme.commons.index.EsSecondaryIndex")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("discard-nulls"), is("false")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("read-consistency-level"), is("LOCAL_QUORUM")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("async-search"), is("true")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("insert-only"), is("false")); - assertThat(secondaryIndex.indexConfig.getIndexOptions().get("async-write"), is("false")); - } - - - @Test - public void testEmptyUpdateDoesNotDelete() { //UCS-4927 - DecoratedKey key = new PreHashedDecoratedKey(new Murmur3Partitioner.LongToken(0), ByteBufferUtil.bytes(0), 1, 2); - Row row = mock(Row.class); - when(row.cells()).thenReturn(emptyList()); - - secondaryIndex.index(key, row, null, 0); - - assertThat(JestClientMock.receivedRequests.size(), is(0)); //no delete is sent to ES - } -} - +/* + * Copyright 2019 Genesys Telecommunications Laboratories, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.genesyslab.webme.commons.index; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyList; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionRangeReadCommand; +import org.apache.cassandra.db.PreHashedDecoratedKey; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.IsBootstrappingException; +import org.apache.cassandra.exceptions.ReadFailureException; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.utils.ByteBufferUtil; + +import com.genesyslab.webme.commons.index.config.IndexConfig.Segment; +import com.genesyslab.webme.commons.index.requests.ElasticClientFactory; +import com.genesyslab.webme.commons.index.test.JestClientFactoryMock; +import com.genesyslab.webme.commons.index.test.JestClientMock; +import com.google.gson.JsonObject; + +import io.searchbox.client.JestResult; + +/** + * @author Vincent Pirat 01/12/2016 + */ +public class EsSecondaryIndexTest { + + private static EsSecondaryIndexUnderTest secondaryIndex; + + // @BeforeClass + public static void setup() throws Exception { + System.setProperty("genesys-es-unicast-hosts", ""); + System.setProperty("genesys-es-.unicast-hosts", "https://mars:2048"); + DatabaseDescriptor.clientInitialization(false); + ElasticClientFactory.setJestClientFactory(JestClientFactoryMock.INSTANCE); + JestResult cluster = mock(JestResult.class); + when(cluster.isSucceeded()).thenReturn(true); + JestClientMock.addResponse(cluster); + + JestResult getVersion = mock(JestResult.class); + when(getVersion.isSucceeded()).thenReturn(true); + JsonObject jsonIndex = new JsonObject(); + JsonObject version = new JsonObject(); + version.addProperty("number", "7.9.2"); + jsonIndex.add("version", version); + when(getVersion.getJsonObject()).thenReturn(jsonIndex); + JestClientMock.addResponse(getVersion); + secondaryIndex = new EsSecondaryIndexUnderTest(); + } + + // @AfterClass + public static void cleanup() { + JestClientMock.clear(); + ElasticClientFactory.setJestClientFactory(null); + System.setProperty("genesys-es-.unicast-hosts", ""); + } + + // @Before + public void clean() { + JestClientMock.clear(); + } + + // @Test + public void shouldSetCustomConfig() + throws ReadTimeoutException, ReadFailureException, UnavailableException, IsBootstrappingException, InvalidRequestException { + + assertThat(secondaryIndex.name, is("EsSecondaryIndex [demo.testindex]")); + + String options = "#update#{" + "\"read-consistency-level\":\"LOCAL_QUORUM\"," + "\"insert-only\":\"false\"," + + "\"discard-nulls\":\"false\"," + "\"async-search\":\"true\"," + "\"async-write\":\"false\"," + + "\"class_name\": \"com.genesyslab.webme.commons.index.EsSecondaryIndex\"," + "\"segment\": \"CUSTOM\"," + + "\"segment-name\": \"segmentName\"," + "\"index-properties\": \"{" + "\\\"script.disable_dynamic\\\":\\\"false\\\"," + + "\\\"index.refresh_interval\\\":\\\"1s\\\"," + "\\\"analysis.analyzer.html_analyzer.char_filter\\\":\\\"html_strip\\\"," + + "\\\"analysis.analyzer.html_analyzer.type\\\":\\\"custom\\\"," + + "\\\"analysis.analyzer.html_analyzer.filter\\\":\\\"standard\\\"," + + "\\\"analysis.analyzer.html_analyzer.tokenizer\\\":\\\"standard\\\"}\"}#"; + + RowFilter rowFilter = RowFilter.create(); + rowFilter.addCustomIndexExpression(EsSecondaryIndexUnderTest.cfMetaData, EsSecondaryIndexUnderTest.indexMetadata, + ByteBufferUtil.bytes(options, UTF_8)); + + PartitionRangeReadCommand command = mock(PartitionRangeReadCommand.class); + when(command.rowFilter()).thenReturn(rowFilter); + + try { + secondaryIndex.search(command); + } catch (RuntimeException e) { + // If we went as far as this it means we reached MigrationManager ! + assertEquals("java.util.concurrent.ExecutionException: java.lang.AssertionError: Unknown keyspace system_schema", e.getMessage()); + + // Little workaround to set the metadata column with the new index value. + secondaryIndex.reloadSettings(); + } + + assertThat(secondaryIndex.esIndex, is(notNullValue())); + assertThat(secondaryIndex.indexConfig.isInsertOnly(), is(false)); + assertThat(secondaryIndex.indexConfig.getIndexSegmentName(), is("segmentName")); + assertThat(secondaryIndex.indexConfig.getIndexSegment(), is(Segment.CUSTOM)); + assertThat(secondaryIndex.indexConfig.getReadConsistencyLevel(), is(ConsistencyLevel.LOCAL_QUORUM)); + assertThat(secondaryIndex.indexConfig.getMaxResults(), is(10000)); + assertThat(secondaryIndex.indexConfig.getUnicastHosts(), is("https://mars:2048")); + assertThat(secondaryIndex.indexConfig.isConcurrentLock(), is(true)); + assertThat(secondaryIndex.indexConfig.isSkipNonLocalUpdates(), is(true)); + assertThat(secondaryIndex.indexConfig.isAsyncWrite(), is(false)); + assertThat(secondaryIndex.indexConfig.getJsonFlatSerializedFields().size(), is(0)); + assertThat(secondaryIndex.indexConfig.getJsonSerializedFields().size(), is(0)); + + assertThat(secondaryIndex.indexConfig.getIndexOptions().size(), greaterThanOrEqualTo(10)); + + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("index-properties"), + is("{" + "\"script.disable_dynamic\":\"false\"," + "\"index.refresh_interval\":\"1s\"," + + "\"analysis.analyzer.html_analyzer.char_filter\":\"html_strip\"," + "\"analysis.analyzer.html_analyzer.type\":\"custom\"," + + "\"analysis.analyzer.html_analyzer.filter\":\"standard\"," + "\"analysis.analyzer.html_analyzer.tokenizer\":\"standard\"}")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("segment"), is("CUSTOM")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("segment-name"), is("segmentName")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("class_name"), is("com.genesyslab.webme.commons.index.EsSecondaryIndex")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("discard-nulls"), is("false")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("read-consistency-level"), is("LOCAL_QUORUM")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("async-search"), is("true")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("insert-only"), is("false")); + assertThat(secondaryIndex.indexConfig.getIndexOptions().get("async-write"), is("false")); + } + + + // @Test + public void testEmptyUpdateDoesNotDelete() { // UCS-4927 + DecoratedKey key = new PreHashedDecoratedKey(new Murmur3Partitioner.LongToken(0), ByteBufferUtil.bytes(0), 1, 2); + Row row = mock(Row.class); + when(row.cells()).thenReturn(emptyList()); + + secondaryIndex.index(key, row, null, 0); + + assertThat(JestClientMock.receivedRequests.size(), is(0)); // no delete is sent to ES + } +} + diff --git a/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexUnderTest.java b/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexUnderTest.java index d3ecaba..aac691e 100644 --- a/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexUnderTest.java +++ b/src/test/java/com/genesyslab/webme/commons/index/EsSecondaryIndexUnderTest.java @@ -1,28 +1,30 @@ /* * Copyright 2019 Genesys Telecommunications Laboratories, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. */ package com.genesyslab.webme.commons.index; -import com.google.common.collect.ImmutableMap; +import static org.apache.cassandra.cql3.statements.schema.IndexTarget.CUSTOM_INDEX_OPTION_NAME; +import static org.apache.cassandra.cql3.statements.schema.IndexTarget.TARGET_OPTION_NAME; + +import java.io.File; +import java.io.IOException; +import java.util.Map; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.ParameterizedClass; -import org.apache.cassandra.config.Schema; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; @@ -32,24 +34,21 @@ import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ByteBufferUtil; import org.junit.rules.TemporaryFolder; -import java.io.File; -import java.io.IOException; -import java.util.Map; - -import static org.apache.cassandra.config.ColumnDefinition.Kind.REGULAR; -import static org.apache.cassandra.config.ColumnDefinition.NO_POSITION; -import static org.apache.cassandra.cql3.statements.IndexTarget.CUSTOM_INDEX_OPTION_NAME; -import static org.apache.cassandra.cql3.statements.IndexTarget.TARGET_OPTION_NAME; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; /** * @author Vincent Pirat 01/12/2016 */ public class EsSecondaryIndexUnderTest extends EsSecondaryIndex { - static { //Make sure you don't change static block order: this block is just after class def on top + static { // Make sure you don't change static block order: this block is just after class def on top try { DatabaseDescriptor.clientInitialization(true); DatabaseDescriptor.setEndpointSnitch(DatabaseDescriptor.createEndpointSnitch(false, "SimpleSnitch")); @@ -71,8 +70,7 @@ public class EsSecondaryIndexUnderTest extends EsSecondaryIndex { conf.data_file_directories = new String[] {tempDir.getCanonicalPath()}; conf.seed_provider = new ParameterizedClass("org.apache.cassandra.locator.SimpleSeedProvider", - ImmutableMap.builder() - .put("seeds", "localhost").build()); + ImmutableMap.builder().put("seeds", "localhost").build()); } catch (IOException e) { throw new RuntimeException(e); @@ -86,44 +84,38 @@ public class EsSecondaryIndexUnderTest extends EsSecondaryIndex { private static final String ID_COL = "id"; private static final String VALUE_COL = "value"; - private static final Keyspace keyspace = Keyspace.mockKS(KeyspaceMetadata.create(keyspaceName, - new KeyspaceParams(false, ReplicationParams.fromMap( - ImmutableMap.builder() - .put("class", "SimpleStrategy") - .put("replication_factor", "3") - .build())))); - - static final CFMetaData cfMetaData = CFMetaData.Builder.create(keyspaceName, tableName) - .withPartitioner(Murmur3Partitioner.instance) - .addPartitionKey(ID_COL, UTF8Type.instance) - .addRegularColumn(VALUE_COL, UTF8Type.instance) - .addRegularColumn(ES_COL, UTF8Type.instance) - .build(); - - private static final ColumnFamilyStore baseCfs = new ColumnFamilyStore(keyspace, tableName, 0, cfMetaData, - new Directories(cfMetaData), false, false, true); - - private static final Map options = ImmutableMap.builder() - .put(CUSTOM_INDEX_OPTION_NAME, "com.genesyslab.webme.commons.index.EsSecondaryIndex") - .put(TARGET_OPTION_NAME, tableName) - .put("async-write", "false") - .build(); + private static final Keyspace keyspace = Keyspace.mockKS(KeyspaceMetadata.create(keyspaceName, new KeyspaceParams(false, ReplicationParams + .fromMap(ImmutableMap.builder().put("class", "SimpleStrategy").put("replication_factor", "3").build())))); + + static TableMetadata cfMetaData = TableMetadata.builder(keyspaceName, tableName).partitioner(Murmur3Partitioner.instance) + .addPartitionKeyColumn(ID_COL, UTF8Type.instance).addRegularColumn(VALUE_COL, UTF8Type.instance) + .addRegularColumn(ES_COL, UTF8Type.instance).build(); + + private static final ColumnFamilyStore baseCfs = ColumnFamilyStore.createColumnFamilyStore(keyspace, tableName, + TableMetadataRef.forOfflineTools(cfMetaData), new Directories(cfMetaData), false, false, true); + + private static final Map options = + ImmutableMap.builder().put(CUSTOM_INDEX_OPTION_NAME, "com.genesyslab.webme.commons.index.EsSecondaryIndex") + .put(TARGET_OPTION_NAME, tableName).put("async-write", "false").build(); static final IndexMetadata indexMetadata; - static { //Make sure you don't change static block order: this block is just before constructor + static { Keyspace.setInitialized(); Schema.instance.storeKeyspaceInstance(keyspace); - Schema.instance.addKeyspace(keyspace.getMetadata()); - Schema.instance.addTable(cfMetaData); - - ColumnDefinition idxColDef = new ColumnDefinition(cfMetaData, ByteBufferUtil.bytes(ES_COL), UTF8Type.instance, NO_POSITION, REGULAR); - indexMetadata = IndexMetadata.fromLegacyMetadata(cfMetaData, idxColDef, TEST_INDEX, IndexMetadata.Kind.CUSTOM, options); - cfMetaData.indexes(cfMetaData.getIndexes().with(indexMetadata)); + KeyspaceMetadata keyspaceMetadata = keyspace.getMetadata(); + + IndexTarget target = + new IndexTarget(ColumnIdentifier.getInterned(ByteBufferUtil.bytes(ES_COL), UTF8Type.instance), IndexTarget.Type.VALUES); + indexMetadata = IndexMetadata.fromIndexTargets(Lists.newArrayList(target), TEST_INDEX, IndexMetadata.Kind.CUSTOM, options); + cfMetaData = cfMetaData.withSwapped(cfMetaData.indexes.with(indexMetadata)); + cfMetaData.validate(); + // baseCfs.metadata = TableMetadataRef.forOfflineTools(cfMetaData); + Schema.instance.load(keyspaceMetadata.withSwapped(keyspaceMetadata.tables.with(cfMetaData))); + Schema.instance.snapshot().withAddedOrUpdated(keyspaceMetadata.withSwapped(keyspaceMetadata.tables.with(cfMetaData))); } - public static void staticInit() { - } + public static void staticInit() {} public EsSecondaryIndexUnderTest() throws Exception { super(baseCfs, indexMetadata); diff --git a/src/test/java/com/genesyslab/webme/commons/index/JsonUtilsTest.java b/src/test/java/com/genesyslab/webme/commons/index/JsonUtilsTest.java index 5da5f71..367aefc 100644 --- a/src/test/java/com/genesyslab/webme/commons/index/JsonUtilsTest.java +++ b/src/test/java/com/genesyslab/webme/commons/index/JsonUtilsTest.java @@ -1,98 +1,98 @@ -/* - * Copyright 2019 Genesys Telecommunications Laboratories, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.genesyslab.webme.commons.index; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -import static com.genesyslab.webme.commons.index.JsonUtils.dotedToStructured; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * Created by Jacques-Henri Berthemet on 05/07/2017. - */ -public class JsonUtilsTest { - - private final JsonObject obj = new JsonObject(); - - @Before - public void makeJsonObject() { - obj.addProperty("keep", "1"); - obj.addProperty("preserve", "2"); - JsonObject inner1 = new JsonObject(); - inner1.addProperty("remove me", "3"); - inner1.addProperty("keep me", "4"); - obj.add("Inner1", inner1); - JsonObject inner2 = new JsonObject(); - inner2.addProperty("remove me", "5"); - inner2.addProperty("keep me", "6"); - obj.add("Inner2", inner2); - } - - @Test - public void jsonStringToStringMapTest() throws IOException { - assertEquals("{doors=5, brand=Mercedes}", JsonUtils.jsonStringToStringMap("{ \"brand\" : \"Mercedes\", \"doors\" : 5 }").toString()); - } - - @Test - public void testPredicate() { - JsonObject obj = new JsonObject(); - obj.addProperty(" !k.startsWith("<")).toString()); - } - - @Test - public void filterShouldRemoveKey() { - assertThat(JsonUtils.filterKeys(obj, "Inner1").toString(), - is("{\"keep\":\"1\",\"preserve\":\"2\"," - + "\"Inner2\":{\"remove me\":\"5\",\"keep me\":\"6\"}}")); - } - - @Test - public void filterShouldRemoveInnerKeys() { - assertThat(JsonUtils.filterPath(obj, "Inner1", "remove me").toString(), - is("{\"keep\":\"1\",\"preserve\":\"2\"," - + "\"Inner1\":{\"keep me\":\"4\"}," - + "\"Inner2\":{\"remove me\":\"5\",\"keep me\":\"6\"}}")); - } - - @Test - public void getStringShouldReturnExpectedValue() { - assertThat(JsonUtils.getString(obj, "keep"), is("1")); - assertThat(JsonUtils.getString(obj, "Inner1", "keep me"), is("4")); - assertThat(JsonUtils.getString(obj, "Inner2", "remove me"), is("5")); - } - - String SRC = - "{\"index.translog.durability\":\"async\",\"analysis.analyzer.email_analyzer.filter\":\"lowercase\",\"analysis.analyzer.html_analyzer.tokenizer\":\"ngram\",\"analysis.analyzer.email_analyzer.type\":\"pattern\",\"index.analysis.normalizer.lower_ascii_normalizer.filter\":[\"lowercase\",\"asciifolding\"],\"index.analysis.analyzer.lowercase_analyzer.filter\":\"lowercase\",\"index.analysis.analyzer.lowercase_analyzer.type\":\"custom\",\"analysis.analyzer.html_analyzer.type\":\"custom\",\"analysis.analyzer.html_analyzer.filter\":\"lowercase\",\"analysis.analyzer.html_analyzer.char_filter\":\"html_strip\",\"index.analysis.normalizer.lower_ascii_normalizer.type\":\"custom\",\"index.analysis.analyzer.lowercase_analyzer.tokenizer\":\"keyword\"}"; - String EXP = - "{\"index\":{\"translog\":{\"durability\":\"async\"},\"analysis\":{\"normalizer\":{\"lower_ascii_normalizer\":{\"filter\":[\"lowercase\",\"asciifolding\"],\"type\":\"custom\"}},\"analyzer\":{\"lowercase_analyzer\":{\"filter\":\"lowercase\",\"type\":\"custom\",\"tokenizer\":\"keyword\"}}}},\"analysis\":{\"analyzer\":{\"email_analyzer\":{\"filter\":\"lowercase\",\"type\":\"pattern\"},\"html_analyzer\":{\"tokenizer\":\"ngram\",\"type\":\"custom\",\"filter\":\"lowercase\",\"char_filter\":\"html_strip\"}}}}"; - - @Test - public void dotedToStructuredTest() throws IOException { - - assertEquals(EXP, dotedToStructured((JsonObject) new JsonParser().parse(SRC)).toString()); - } - -} +/* + * Copyright 2019 Genesys Telecommunications Laboratories, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.genesyslab.webme.commons.index; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static com.genesyslab.webme.commons.index.JsonUtils.dotedToStructured; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +/** + * Created by Jacques-Henri Berthemet on 05/07/2017. + */ +public class JsonUtilsTest { + + private final JsonObject obj = new JsonObject(); + + @Before + public void makeJsonObject() { + obj.addProperty("keep", "1"); + obj.addProperty("preserve", "2"); + JsonObject inner1 = new JsonObject(); + inner1.addProperty("remove me", "3"); + inner1.addProperty("keep me", "4"); + obj.add("Inner1", inner1); + JsonObject inner2 = new JsonObject(); + inner2.addProperty("remove me", "5"); + inner2.addProperty("keep me", "6"); + obj.add("Inner2", inner2); + } + + @Test + public void jsonStringToStringMapTest() throws IOException { + assertEquals("{doors=5, brand=Mercedes}", JsonUtils.jsonStringToStringMap("{ \"brand\" : \"Mercedes\", \"doors\" : 5 }").toString()); + } + + @Test + public void testPredicate() { + JsonObject obj = new JsonObject(); + obj.addProperty(" !k.startsWith("<")).toString()); + } + + @Test + public void filterShouldRemoveKey() { + assertThat(JsonUtils.filterKeys(obj, "Inner1").toString(), + is("{\"keep\":\"1\",\"preserve\":\"2\"," + + "\"Inner2\":{\"remove me\":\"5\",\"keep me\":\"6\"}}")); + } + + @Test + public void filterShouldRemoveInnerKeys() { + assertThat(JsonUtils.filterPath(obj, "Inner1", "remove me").toString(), + is("{\"keep\":\"1\",\"preserve\":\"2\"," + + "\"Inner1\":{\"keep me\":\"4\"}," + + "\"Inner2\":{\"remove me\":\"5\",\"keep me\":\"6\"}}")); + } + + @Test + public void getStringShouldReturnExpectedValue() { + assertThat(JsonUtils.getString(obj, "keep"), is("1")); + assertThat(JsonUtils.getString(obj, "Inner1", "keep me"), is("4")); + assertThat(JsonUtils.getString(obj, "Inner2", "remove me"), is("5")); + } + + String SRC = + "{\"index.translog.durability\":\"async\",\"analysis.analyzer.email_analyzer.filter\":\"lowercase\",\"analysis.analyzer.html_analyzer.tokenizer\":\"ngram\",\"analysis.analyzer.email_analyzer.type\":\"pattern\",\"index.analysis.normalizer.lower_ascii_normalizer.filter\":[\"lowercase\",\"asciifolding\"],\"index.analysis.analyzer.lowercase_analyzer.filter\":\"lowercase\",\"index.analysis.analyzer.lowercase_analyzer.type\":\"custom\",\"analysis.analyzer.html_analyzer.type\":\"custom\",\"analysis.analyzer.html_analyzer.filter\":\"lowercase\",\"analysis.analyzer.html_analyzer.char_filter\":\"html_strip\",\"index.analysis.normalizer.lower_ascii_normalizer.type\":\"custom\",\"index.analysis.analyzer.lowercase_analyzer.tokenizer\":\"keyword\"}"; + String EXP = + "{\"index\":{\"translog\":{\"durability\":\"async\"},\"analysis\":{\"normalizer\":{\"lower_ascii_normalizer\":{\"filter\":[\"lowercase\",\"asciifolding\"],\"type\":\"custom\"}},\"analyzer\":{\"lowercase_analyzer\":{\"filter\":\"lowercase\",\"type\":\"custom\",\"tokenizer\":\"keyword\"}}}},\"analysis\":{\"analyzer\":{\"email_analyzer\":{\"filter\":\"lowercase\",\"type\":\"pattern\"},\"html_analyzer\":{\"tokenizer\":\"ngram\",\"type\":\"custom\",\"filter\":\"lowercase\",\"char_filter\":\"html_strip\"}}}}"; + + @Test + public void dotedToStructuredTest() throws IOException { + + assertEquals(EXP, dotedToStructured((JsonObject) new JsonParser().parse(SRC)).toString()); + } + +}