Add configurable truncation for string columns

docs/configuration/index.md

@@ -1431,6 +1431,7 @@ Additional Peon configs include:
 |`druid.indexer.task.tmpStorageBytesPerTask`|Maximum number of bytes per task to be used to store temporary files on disk. This config is generally intended for internal usage. Attempts to set it are very likely to be overwritten by the TaskRunner that executes the task, so be sure of what you expect to happen before directly adjusting this configuration parameter. The config is documented here primarily to provide an understanding of what it means if/when someone sees that it has been set. A value of -1 disables this limit.  |-1|
 |`druid.indexer.task.allowHadoopTaskExecution`|Conditional dictating if the cluster allows `index_hadoop` tasks to be executed. `index_hadoop` is deprecated, and defaulting to false will force cluster operators to acknowledge the deprecation and consciously opt in to using index_hadoop with the understanding that it will be removed in the future.|false|
 |`druid.indexer.server.maxChatRequests`|Maximum number of concurrent requests served by a task's chat handler. Set to 0 to disable limiting.|0|
+|`druid.indexing.formats.maxStringLength`|Maximum number of characters to store per string dimension value. Longer values are truncated during ingestion. Set to 0 to disable. Can be overridden per-dimension using `maxStringLength` in the [dimension object](../ingestion/ingestion-spec.md#dimension-objects).|0 (no truncation)|
 
 If the Peon is running in remote mode, there must be an Overlord up and running. Peons in remote mode can set the following configurations:
 

docs/ingestion/ingestion-spec.md

@@ -247,6 +247,7 @@ Dimension objects can have the following components:
 | name | The name of the dimension. This will be used as the field name to read from input records, as well as the column name stored in generated segments.<br /><br />Note that you can use a [`transformSpec`](#transformspec) if you want to rename columns during ingestion time. | none (required) |
 | createBitmapIndex | For `string` typed dimensions, whether or not bitmap indexes should be created for the column in generated segments. Creating a bitmap index requires more storage, but speeds up certain kinds of filtering (especially equality and prefix filtering). Only supported for `string` typed dimensions. | `true` |
 | multiValueHandling | For `string` typed dimensions, specifies the type of handling for [multi-value fields](../querying/multi-value-dimensions.md). Possible values are `array` (ingest string arrays as-is), `sorted_array` (sort string arrays during ingestion), and `sorted_set` (sort and de-duplicate string arrays during ingestion). This parameter is ignored for types other than `string`. | `sorted_array` |
+| maxStringLength | For `string` typed dimensions, the maximum number of characters to store per value. Longer values are truncated during ingestion. Set to 0 to disable. Overrides the global [`druid.indexing.formats.maxStringLength`](../configuration/index.md#additional-peon-configuration) property. | `0` (no truncation) |
 
 #### Inclusions and exclusions
 

processing/src/main/java/org/apache/druid/data/input/impl/StringDimensionSchema.java

@@ -21,15 +21,26 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.guice.BuiltInTypesModule;
 import org.apache.druid.segment.DimensionHandler;
 import org.apache.druid.segment.StringDimensionHandler;
 import org.apache.druid.segment.column.ColumnType;
 
+import javax.annotation.Nullable;
+
 public class StringDimensionSchema extends DimensionSchema
 {
   private static final boolean DEFAULT_CREATE_BITMAP_INDEX = true;
 
+  public static int getDefaultMaxStringLength()
+  {
+    return BuiltInTypesModule.getMaxStringLength();
+  }
+
+  private final int maxStringLength;
+
   @JsonCreator
   public static StringDimensionSchema create(String name)
   {
@@ -40,15 +51,33 @@ public static StringDimensionSchema create(String name)
   public StringDimensionSchema(
       @JsonProperty("name") String name,
       @JsonProperty("multiValueHandling") MultiValueHandling multiValueHandling,
-      @JsonProperty("createBitmapIndex") Boolean createBitmapIndex
+      @JsonProperty("createBitmapIndex") Boolean createBitmapIndex,
+      @JsonProperty("maxStringLength") @Nullable Integer maxStringLength
   )
   {
     super(name, multiValueHandling, createBitmapIndex == null ? DEFAULT_CREATE_BITMAP_INDEX : createBitmapIndex);
+    this.maxStringLength = maxStringLength != null && maxStringLength > 0 ? maxStringLength : getDefaultMaxStringLength();
+  }
+
+  public StringDimensionSchema(
+      String name,
+      MultiValueHandling multiValueHandling,
+      Boolean createBitmapIndex
+  )
+  {
+    this(name, multiValueHandling, createBitmapIndex, getDefaultMaxStringLength());
   }
 
   public StringDimensionSchema(String name)
   {
-    this(name, null, DEFAULT_CREATE_BITMAP_INDEX);
+    this(name, null, DEFAULT_CREATE_BITMAP_INDEX, getDefaultMaxStringLength());
+  }
+
+  @JsonProperty
+  @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+  public int getMaxStringLength()
+  {
+    return maxStringLength;
   }
 
   @Override
@@ -73,6 +102,6 @@ public boolean canBeMultiValued()
   @Override
   public DimensionHandler getDimensionHandler()
   {
-    return new StringDimensionHandler(getName(), getMultiValueHandling(), hasBitmapIndex(), false);
+    return new StringDimensionHandler(getName(), getMultiValueHandling(), hasBitmapIndex(), false, maxStringLength);
   }
 }

processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java

@@ -53,6 +53,7 @@ public class BuiltInTypesModule implements DruidModule
    */
   private static DimensionSchema.MultiValueHandling STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY;
   private static IndexSpec DEFAULT_INDEX_SPEC = IndexSpec.builder().build();
+  private static int MAX_STRING_LENGTH = 0;
 
   /**
    * @return the configured string multi value handling mode from the system config if set; otherwise, returns
@@ -89,6 +90,7 @@ public void configure(Binder binder)
   public SideEffectRegisterer initDimensionHandlerAndMvHandlingMode(DefaultColumnFormatConfig formatsConfig)
   {
     setStringMultiValueHandlingModeIfConfigured(formatsConfig.getStringMultiValueHandlingMode());
+    setMaxStringLengthIfConfigured(formatsConfig.getMaxStringLength());
     setIndexSpecDefaults(formatsConfig.getIndexSpec());
     setNestedColumnDefaults(formatsConfig);
 
@@ -128,6 +130,24 @@ private static void registerSerde()
     }
   }
 
+  private static void setMaxStringLengthIfConfigured(@Nullable Integer maxStringLength)
+  {
+    if (maxStringLength != null) {
+      MAX_STRING_LENGTH = maxStringLength;
+    }
+  }
+
+  @VisibleForTesting
+  public static void setMaxStringLength(int maxStringLength)
+  {
+    MAX_STRING_LENGTH = maxStringLength;
+  }
+
+  public static int getMaxStringLength()
+  {
+    return MAX_STRING_LENGTH;
+  }
+
   private static void setStringMultiValueHandlingModeIfConfigured(@Nullable String stringMultiValueHandlingMode)
   {
     if (stringMultiValueHandlingMode != null) {

processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java

@@ -68,6 +68,21 @@ private static String validateMultiValueHandlingMode(
     return stringMultiValueHandlingMode;
   }
 
+  @Nullable
+  private static Integer validateMaxStringLength(@Nullable Integer maxStringLength)
+  {
+    if (maxStringLength != null && maxStringLength <= 0) {
+      throw DruidException.forPersona(DruidException.Persona.OPERATOR)
+                          .ofCategory(DruidException.Category.INVALID_INPUT)
+                          .build(
+                              "Invalid value[%s] specified for 'druid.indexing.formats.maxStringLength'."
+                              + " Value must be a positive integer.",
+                              maxStringLength
+                          );
+    }
+    return maxStringLength;
+  }
+
   @JsonProperty("stringMultiValueHandlingMode")
   @Nullable
   private final Integer nestedColumnFormatVersion;
@@ -80,11 +95,16 @@ private static String validateMultiValueHandlingMode(
   @Nullable
   private final IndexSpec indexSpec;
 
+  @JsonProperty("maxStringLength")
+  @Nullable
+  private final Integer maxStringLength;
+
   @JsonCreator
   public DefaultColumnFormatConfig(
       @JsonProperty("stringMultiValueHandlingMode") @Nullable String stringMultiValueHandlingMode,
       @JsonProperty("nestedColumnFormatVersion") @Nullable Integer nestedColumnFormatVersion,
-      @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec
+      @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
+      @JsonProperty("maxStringLength") @Nullable Integer maxStringLength
   )
   {
     validateMultiValueHandlingMode(stringMultiValueHandlingMode);
@@ -93,6 +113,7 @@ public DefaultColumnFormatConfig(
     this.stringMultiValueHandlingMode = validateMultiValueHandlingMode(stringMultiValueHandlingMode);
     this.nestedColumnFormatVersion = nestedColumnFormatVersion;
     this.indexSpec = indexSpec;
+    this.maxStringLength = validateMaxStringLength(maxStringLength);
   }
 
   @Nullable
@@ -116,6 +137,13 @@ public IndexSpec getIndexSpec()
     return indexSpec;
   }
 
+  @Nullable
+  @JsonProperty("maxStringLength")
+  public Integer getMaxStringLength()
+  {
+    return maxStringLength;
+  }
+
   @Override
   public boolean equals(Object o)
   {
@@ -128,13 +156,14 @@ public boolean equals(Object o)
     DefaultColumnFormatConfig that = (DefaultColumnFormatConfig) o;
     return Objects.equals(nestedColumnFormatVersion, that.nestedColumnFormatVersion)
            && Objects.equals(stringMultiValueHandlingMode, that.stringMultiValueHandlingMode)
-           && Objects.equals(indexSpec, that.indexSpec);
+           && Objects.equals(indexSpec, that.indexSpec)
+           && Objects.equals(maxStringLength, that.maxStringLength);
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(nestedColumnFormatVersion, stringMultiValueHandlingMode, indexSpec);
+    return Objects.hash(nestedColumnFormatVersion, stringMultiValueHandlingMode, indexSpec, maxStringLength);
   }
 
   @Override
@@ -144,6 +173,7 @@ public String toString()
            "stringMultiValueHandlingMode=" + stringMultiValueHandlingMode +
            ", nestedColumnFormatVersion=" + nestedColumnFormatVersion +
            ", indexSpec=" + indexSpec +
+           ", maxStringLength=" + maxStringLength +
            '}';
   }
 }

processing/src/main/java/org/apache/druid/segment/StringDimensionHandler.java

@@ -104,18 +104,31 @@ private static IndexedInts getRow(ColumnValueSelector s)
   private final MultiValueHandling multiValueHandling;
   private final boolean hasBitmapIndexes;
   private final boolean hasSpatialIndexes;
+  private final int maxStringLength;
 
   public StringDimensionHandler(
       String dimensionName,
       MultiValueHandling multiValueHandling,
       boolean hasBitmapIndexes,
       boolean hasSpatialIndexes
   )
+  {
+    this(dimensionName, multiValueHandling, hasBitmapIndexes, hasSpatialIndexes, StringDimensionSchema.getDefaultMaxStringLength());
+  }
+
+  public StringDimensionHandler(
+      String dimensionName,
+      MultiValueHandling multiValueHandling,
+      boolean hasBitmapIndexes,
+      boolean hasSpatialIndexes,
+      int maxStringLength
+  )
   {
     this.dimensionName = dimensionName;
     this.multiValueHandling = multiValueHandling;
     this.hasBitmapIndexes = hasBitmapIndexes;
     this.hasSpatialIndexes = hasSpatialIndexes;
+    this.maxStringLength = maxStringLength;
   }
 
   @Override
@@ -160,7 +173,7 @@ public SettableColumnValueSelector makeNewSettableEncodedValueSelector()
   @Override
   public DimensionIndexer<Integer, int[], String> makeIndexer()
   {
-    return new StringDimensionIndexer(multiValueHandling, hasBitmapIndexes, hasSpatialIndexes);
+    return new StringDimensionIndexer(multiValueHandling, hasBitmapIndexes, hasSpatialIndexes, maxStringLength);
   }
 
   @Override

processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java

@@ -24,6 +24,7 @@
 import org.apache.druid.collections.bitmap.BitmapFactory;
 import org.apache.druid.collections.bitmap.MutableBitmap;
 import org.apache.druid.data.input.impl.DimensionSchema.MultiValueHandling;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
@@ -57,18 +58,38 @@ public class StringDimensionIndexer extends DictionaryEncodedColumnIndexer<int[]
   private final MultiValueHandling multiValueHandling;
   private final boolean hasBitmapIndexes;
   private final boolean hasSpatialIndexes;
+  private final int maxStringLength;
   private volatile boolean hasMultipleValues = false;
 
   public StringDimensionIndexer(
       @Nullable MultiValueHandling multiValueHandling,
       boolean hasBitmapIndexes,
       boolean hasSpatialIndexes
   )
+  {
+    this(multiValueHandling, hasBitmapIndexes, hasSpatialIndexes, StringDimensionSchema.getDefaultMaxStringLength());
+  }
+
+  public StringDimensionIndexer(
+      @Nullable MultiValueHandling multiValueHandling,
+      boolean hasBitmapIndexes,
+      boolean hasSpatialIndexes,
+      int maxStringLength
+  )
   {
     super(new StringDimensionDictionary());
     this.multiValueHandling = multiValueHandling == null ? MultiValueHandling.ofDefault() : multiValueHandling;
     this.hasBitmapIndexes = hasBitmapIndexes;
     this.hasSpatialIndexes = hasSpatialIndexes;
+    this.maxStringLength = maxStringLength;
+  }
+
+  private String truncateIfNeeded(String value)
+  {
+    if (maxStringLength > 0 && value != null && value.length() > maxStringLength) {
+      return value.substring(0, maxStringLength);
+    }
+    return value;
   }
 
   @Override
@@ -92,7 +113,7 @@ public EncodedKeyComponent<int[]> processRowValsToUnsortedEncodedKeyComponent(@N
         dimLookup.add(null);
         encodedDimensionValues = IntArrays.EMPTY_ARRAY;
       } else if (dimValuesList.size() == 1) {
-        encodedDimensionValues = new int[]{dimLookup.add(Evals.asString(dimValuesList.get(0)))};
+        encodedDimensionValues = new int[]{dimLookup.add(truncateIfNeeded(Evals.asString(dimValuesList.get(0))))};
       } else {
         hasMultipleValues = true;
         final String[] dimensionValues = new String[dimValuesList.size()];
@@ -125,7 +146,7 @@ public EncodedKeyComponent<int[]> processRowValsToUnsortedEncodedKeyComponent(@N
       encodedDimensionValues =
           new int[]{dimLookup.add(Evals.asString(StringUtils.encodeBase64String((byte[]) dimValues)))};
     } else {
-      encodedDimensionValues = new int[]{dimLookup.add(Evals.asString(dimValues))};
+      encodedDimensionValues = new int[]{dimLookup.add(truncateIfNeeded(Evals.asString(dimValues)))};
     }
 
     // If dictionary size has changed, the sorted lookup is no longer valid.

processing/src/test/java/org/apache/druid/data/input/impl/StringDimensionSchemaTest.java

@@ -54,9 +54,11 @@ public void testDeserializeFromJson() throws JsonProcessingException
     final String json = "{\n"
                         + "  \"name\" : \"dim\",\n"
                         + "  \"multiValueHandling\" : \"SORTED_SET\",\n"
-                        + "  \"createBitmapIndex\" : false\n"
+                        + "  \"createBitmapIndex\" : false,\n"
+                        + "  \"maxStringLength\" : 200\n"
                         + "}";
     final StringDimensionSchema schema = (StringDimensionSchema) jsonMapper.readValue(json, DimensionSchema.class);
     Assert.assertEquals(new StringDimensionSchema("dim", MultiValueHandling.SORTED_SET, false), schema);
+    Assert.assertEquals(200, schema.getMaxStringLength());
   }
 }