Merge remote-tracking branch 'apache/master' into remove-createsmfn-c…

…putime

.github/labeler.yml

@@ -88,7 +88,7 @@
 'Kubernetes':
   - changed-files:
     - any-glob-to-any-file:
-      - 'extensions-contrib/kubernetes-overlord-extensions/**'
+      - 'extensions-core/kubernetes-overlord-extensions/**'
 
 'GHA':
   - changed-files:

.github/workflows/static-checks.yml

@@ -41,8 +41,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429
-        java: [ '11', '17', '21.0.4' ]
+        java: [ '11', '17', '21' ]
     runs-on: ubuntu-latest
     steps:
       - name: checkout branch

.github/workflows/unit-and-integration-tests-unified.yml

@@ -79,8 +79,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429
-        jdk: [ '11', '17', '21.0.4' ]
+        jdk: [ '11', '17', '21' ]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
@@ -211,8 +210,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429
-        jdk: [ '11', '21.0.4' ]
+        jdk: [ '11', '21' ]
     name: "unit tests (jdk${{ matrix.jdk }})"
     uses: ./.github/workflows/unit-tests.yml
     needs: [unit-tests-unapproved, check-approval]

benchmarks/src/test/java/org/apache/druid/benchmark/compression/BaseColumnarLongsBenchmark.java

@@ -318,7 +318,7 @@ static ColumnarLongs createColumnarLongs(String encoding, ByteBuffer buffer)
       case "none-longs":
       case "zstd-auto":
       case "zstd-longs":
-        return CompressedColumnarLongsSupplier.fromByteBuffer(buffer, ByteOrder.LITTLE_ENDIAN).get();
+        return CompressedColumnarLongsSupplier.fromByteBuffer(buffer, ByteOrder.LITTLE_ENDIAN, null).get();
     }
 
     throw new IllegalArgumentException("unknown encoding");

benchmarks/src/test/java/org/apache/druid/benchmark/compression/CompressedColumnarIntsBenchmark.java

@@ -82,7 +82,8 @@ public void setup() throws IOException
     );
     this.compressed = CompressedVSizeColumnarIntsSupplier.fromByteBuffer(
         bufferCompressed,
-        ByteOrder.nativeOrder()
+        ByteOrder.nativeOrder(),
+        null
     ).get();
 
     final ByteBuffer bufferUncompressed = serialize(VSizeColumnarInts.fromArray(vals));

benchmarks/src/test/java/org/apache/druid/benchmark/compression/CompressedVSizeColumnarMultiIntsBenchmark.java

@@ -95,7 +95,8 @@ public void setup() throws IOException
     );
     this.compressed = CompressedVSizeColumnarMultiIntsSupplier.fromByteBuffer(
         bufferCompressed,
-        ByteOrder.nativeOrder()
+        ByteOrder.nativeOrder(),
+        null
     ).get();
 
     final ByteBuffer bufferUncompressed = serialize(

benchmarks/src/test/java/org/apache/druid/benchmark/compression/FloatCompressionBenchmark.java

@@ -75,7 +75,7 @@ public void setup() throws Exception
     File compFile = new File(dir, file + "-" + strategy);
     bufferHandler = FileUtils.map(compFile);
     ByteBuffer buffer = bufferHandler.get();
-    supplier = CompressedColumnarFloatsSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder());
+    supplier = CompressedColumnarFloatsSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null);
   }
 
   @TearDown

benchmarks/src/test/java/org/apache/druid/benchmark/compression/LongCompressionBenchmark.java

@@ -79,7 +79,7 @@ public void setup() throws Exception
     File compFile = new File(dir, file + "-" + strategy + "-" + format);
     bufferHandler = FileUtils.map(compFile);
     ByteBuffer buffer = bufferHandler.get();
-    supplier = CompressedColumnarLongsSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder());
+    supplier = CompressedColumnarLongsSupplier.fromByteBuffer(buffer, ByteOrder.nativeOrder(), null);
   }
 
   @TearDown

benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java

@@ -139,7 +139,23 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark
       "SELECT ARRAY_OVERLAP(\"multi-string3\", ARRAY[1, 2, 10, 11, 20, 22, 30, 33, 40, 44, 50, 55, 100]) FROM expressions",
       // 46: filters with random orders
       "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND string1 = '1000' GROUP BY 1 ORDER BY 2",
-      "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) AND string1 = '1000' GROUP BY 1 ORDER BY 2"
+      "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) AND string1 = '1000' GROUP BY 1 ORDER BY 2",
+      // 48-57 nvl tests
+      // lower cardinality
+      "SELECT NVL(string2, string1), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(string2, CONCAT(string1, '-', long2)), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // higher cardinality
+      "SELECT NVL(string5, string3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(string5, CONCAT(string3, '-', long2)), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // no lhs nulls
+      "SELECT NVL(string1, string3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(string1, CONCAT(string3, '-', long2)), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // numeric
+      "SELECT NVL(long5, long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(long5, long1 + long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      // numeric no lhs null
+      "SELECT NVL(long1, long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
+      "SELECT NVL(long1, long5 + long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2"
   );
 
   @Param({
@@ -200,7 +216,17 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark
       "44",
       "45",
       "46",
-      "47"
+      "47",
+      "48",
+      "49",
+      "50",
+      "51",
+      "52",
+      "53",
+      "54",
+      "55",
+      "56",
+      "57"
   })
   private String query;
 

cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java

@@ -61,49 +61,58 @@ private void cleanProperties()
     properties.clear();
   }
 
+  private Injector createInjector()
+  {
+    return Guice.createInjector(
+      binder -> {
+        binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/test/redis");
+        binder.bindConstant().annotatedWith(Names.named("servicePort")).to(0);
+        binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(-1);
+        binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
+        binder.bindScope(LazySingleton.class, Scopes.SINGLETON);
+        binder.bind(JsonConfigurator.class).in(LazySingleton.class);
+        binder.bind(Properties.class).toInstance(properties);
+        JsonConfigProvider.bind(binder, PROPERTY_PREFIX, AWSCredentialsConfig.class);
+      }
+    );
+  }
+
+  @Test
+  public void testDefaultValues()
+  {
+    final Injector injector = createInjector();
+    final AWSCredentialsConfig credentialsConfig = injector.getInstance(AWSCredentialsConfig.class);
+    Assert.assertEquals("", credentialsConfig.getAccessKey().getPassword());
+    Assert.assertEquals("", credentialsConfig.getSecretKey().getPassword());
+  }
+
   @Test
   public void testStringProperty()
   {
+    final String filePath = "/path/to/credentials";
+    properties.put(PROPERTY_PREFIX + ".fileSessionCredentials", filePath);
     properties.put(PROPERTY_PREFIX + ".accessKey", SOME_SECRET);
     properties.put(PROPERTY_PREFIX + ".secretKey", SOME_SECRET);
 
-    final Injector injector = Guice.createInjector(
-        binder -> {
-          binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/test/redis");
-          binder.bindConstant().annotatedWith(Names.named("servicePort")).to(0);
-          binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(-1);
-          binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
-          binder.bindScope(LazySingleton.class, Scopes.SINGLETON);
-          binder.bind(JsonConfigurator.class).in(LazySingleton.class);
-          binder.bind(Properties.class).toInstance(properties);
-          JsonConfigProvider.bind(binder, PROPERTY_PREFIX, AWSCredentialsConfig.class);
-        }
-    );
+    final Injector injector = createInjector();
     final AWSCredentialsConfig credentialsConfig = injector.getInstance(AWSCredentialsConfig.class);
+    Assert.assertEquals(filePath, credentialsConfig.getFileSessionCredentials());
     Assert.assertEquals(SOME_SECRET, credentialsConfig.getAccessKey().getPassword());
     Assert.assertEquals(SOME_SECRET, credentialsConfig.getSecretKey().getPassword());
   }
 
   @Test
   public void testJsonProperty() throws Exception
   {
+    final String filePath = "/path/to/credentials";
     final String someSecret = new ObjectMapper().writeValueAsString(new DefaultPasswordProvider(SOME_SECRET));
+    properties.put(PROPERTY_PREFIX + ".fileSessionCredentials", filePath);
     properties.put(PROPERTY_PREFIX + ".accessKey", someSecret);
     properties.put(PROPERTY_PREFIX + ".secretKey", someSecret);
 
-    final Injector injector = Guice.createInjector(
-        binder -> {
-          binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/test/redis");
-          binder.bindConstant().annotatedWith(Names.named("servicePort")).to(0);
-          binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(-1);
-          binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
-          binder.bindScope(LazySingleton.class, Scopes.SINGLETON);
-          binder.bind(JsonConfigurator.class).in(LazySingleton.class);
-          binder.bind(Properties.class).toInstance(properties);
-          JsonConfigProvider.bind(binder, PROPERTY_PREFIX, AWSCredentialsConfig.class);
-        }
-    );
+    final Injector injector = createInjector();
     final AWSCredentialsConfig credentialsConfig = injector.getInstance(AWSCredentialsConfig.class);
+    Assert.assertEquals(filePath, credentialsConfig.getFileSessionCredentials());
     Assert.assertEquals(SOME_SECRET, credentialsConfig.getAccessKey().getPassword());
     Assert.assertEquals(SOME_SECRET, credentialsConfig.getSecretKey().getPassword());
   }

distribution/docker/druid.sh

@@ -138,8 +138,8 @@ then
     setKey _common druid.zk.service.host "${ZOOKEEPER}"
 fi
 
-DRUID_SET_HOST_IP=${DRUID_SET_HOST_IP:-0}
-if [ "${DRUID_SET_HOST_IP}" = "1" ]
+DRUID_SET_HOST=${DRUID_SET_HOST:-1}
+if [ "${DRUID_SET_HOST}" = "1" ]
 then
     setKey $SERVICE druid.host $(ip r get 1 | awk '{print $7;exit}')
 fi

distribution/docker/peon.sh

@@ -97,8 +97,8 @@ then
     setKey _common druid.zk.service.host "${ZOOKEEPER}"
 fi
 
-DRUID_SET_HOST_IP=${DRUID_SET_HOST_IP:-0}
-if [ "${DRUID_SET_HOST_IP}" = "1" ]
+DRUID_SET_HOST=${DRUID_SET_HOST:-1}
+if [ "${DRUID_SET_HOST}" = "1" ]
 then
     setKey $SERVICE druid.host $(ip r get 1 | awk '{print $7;exit}')
 fi

distribution/pom.xml

@@ -254,6 +254,8 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
                                         <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-kubernetes-overlord-extensions</argument>
+                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-catalog</argument>
                                         <argument>${druid.distribution.pulldeps.opts}</argument>
                                     </arguments>
@@ -413,8 +415,6 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:kafka-emitter</argument>
                                         <argument>-c</argument>
-                                        <argument>org.apache.druid.extensions.contrib:druid-kubernetes-overlord-extensions</argument>
-                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:materialized-view-maintenance</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:materialized-view-selection</argument>

docs/configuration/extensions.md

@@ -39,6 +39,7 @@ Core extensions are maintained by Druid committers.
 |druid-azure-extensions|Microsoft Azure deep storage.|[link](../development/extensions-core/azure.md)|
 |druid-basic-security|Support for Basic HTTP authentication and role-based access control.|[link](../development/extensions-core/druid-basic-security.md)|
 |druid-bloom-filter|Support for providing Bloom filters in druid queries.|[link](../development/extensions-core/bloom-filter.md)|
+|druid-catalog|This extension allows users to configure, update, retrieve, and manage metadata stored in Druid's catalog. |[link](../development/extensions-core/catalog.md)|
 |druid-datasketches|Support for approximate counts and set operations with [Apache DataSketches](https://datasketches.apache.org/).|[link](../development/extensions-core/datasketches-extension.md)|
 |druid-google-extensions|Google Cloud Storage deep storage.|[link](../development/extensions-core/google.md)|
 |druid-hdfs-storage|HDFS deep storage.|[link](../development/extensions-core/hdfs.md)|
@@ -62,6 +63,7 @@ Core extensions are maintained by Druid committers.
 |simple-client-sslcontext|Simple SSLContext provider module to be used by Druid's internal HttpClient when talking to other Druid processes over HTTPS.|[link](../development/extensions-core/simple-client-sslcontext.md)|
 |druid-pac4j|OpenID Connect authentication for druid processes.|[link](../development/extensions-core/druid-pac4j.md)|
 |druid-kubernetes-extensions|Druid cluster deployment on Kubernetes without Zookeeper.|[link](../development/extensions-core/kubernetes.md)|
+|druid-kubernetes-overlord-extensions|Support for launching tasks in k8s without Middle Managers|[link](../development/extensions-core/k8s-jobs.md)|
 
 ## Community extensions
 
@@ -100,7 +102,6 @@ All of these community extensions can be downloaded using [pull-deps](../operati
 |druid-tdigestsketch|Support for approximate sketch aggregators based on [T-Digest](https://github.com/tdunning/t-digest)|[link](../development/extensions-contrib/tdigestsketch-quantiles.md)|
 |gce-extensions|GCE Extensions|[link](../development/extensions-contrib/gce-extensions.md)|
 |prometheus-emitter|Exposes [Druid metrics](../operations/metrics.md) for Prometheus server collection (<https://prometheus.io/>)|[link](../development/extensions-contrib/prometheus.md)|
-|druid-kubernetes-overlord-extensions|Support for launching tasks in k8s without Middle Managers|[link](../development/extensions-contrib/k8s-jobs.md)|
 |druid-spectator-histogram|Support for efficient approximate percentile queries|[link](../development/extensions-contrib/spectator-histogram.md)|
 |druid-rabbit-indexing-service|Support for creating and managing [RabbitMQ](https://www.rabbitmq.com/) indexing tasks|[link](../development/extensions-contrib/rabbit-stream-ingestion.md)|
 |druid-ranger-security|Support for access control through Apache Ranger.|[link](../development/extensions-contrib/druid-ranger-security.md)|

docs/configuration/index.md

@@ -882,6 +882,7 @@ These Coordinator static configurations can be defined in the `coordinator/runti
 |`druid.coordinator.kill.ignoreDurationToRetain`|A way to override `druid.coordinator.kill.durationToRetain` and tell the coordinator that you do not care about the end date of unused segment intervals when it comes to killing them. If true, the coordinator considers all unused segments as eligible to be killed.|false|
 |`druid.coordinator.kill.bufferPeriod`|The amount of time that a segment must be unused before it is able to be permanently removed from metadata and deep storage. This can serve as a buffer period to prevent data loss if data ends up being needed after being marked unused.|`P30D`|
 |`druid.coordinator.kill.maxSegments`|The number of unused segments to kill per kill task. This number must be greater than 0. This only applies when `druid.coordinator.kill.on=true`.|100|
+|`druid.coordinator.kill.maxInterval`|The largest interval, as an [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations), of segments to delete per kill task. Set to zero, e.g. `PT0S`, for unlimited. This only applies when `druid.coordinator.kill.on=true`.|`P30D`|
 |`druid.coordinator.balancer.strategy`|The [balancing strategy](../design/coordinator.md#balancing-segments-in-a-tier) used by the Coordinator to distribute segments among the Historical servers in a tier. The `cost` strategy distributes segments by minimizing a cost function, `diskNormalized` weights these costs with the disk usage ratios of the servers and `random` distributes segments randomly.|`cost`|
 |`druid.coordinator.loadqueuepeon.http.repeatDelay`|The start and repeat delay (in milliseconds) for the load queue peon, which manages the load/drop queue of segments for any server.|1 minute|
 |`druid.coordinator.loadqueuepeon.http.batchSize`|Number of segment load/drop requests to batch in one HTTP request. Note that it must be smaller than `druid.segmentCache.numLoadingThreads` config on Historical service.|1|

docs/design/architecture.md

@@ -107,7 +107,7 @@ forking separate JVM processes per-task, the Indexer runs tasks as individual th
 
 The Indexer is designed to be easier to configure and deploy compared to the MiddleManager + Peon system and to better enable resource sharing across tasks, which can help streaming ingestion. The Indexer is currently designated [experimental](../development/experimental.md).
 
-Typically, you would deploy one of the following: MiddleManagers, [MiddleManager-less ingestion using Kubernetes](../development/extensions-contrib/k8s-jobs.md), or Indexers. You wouldn't deploy more than one of these options.
+Typically, you would deploy one of the following: MiddleManagers, [MiddleManager-less ingestion using Kubernetes](../development/extensions-core/k8s-jobs.md), or Indexers. You wouldn't deploy more than one of these options.
 
 ## Colocation of services
 

docs/design/indexer.md

@@ -24,7 +24,7 @@ sidebar_label: "Indexer"
   -->
 
 :::info
- The Indexer is an optional and experimental feature. If you're primarily performing batch ingestion, we recommend you use either the MiddleManager and Peon task execution system or [MiddleManager-less ingestion using Kubernetes](../development/extensions-contrib/k8s-jobs.md). If you're primarily doing streaming ingestion, you may want to try either [MiddleManager-less ingestion using Kubernetes](../development/extensions-contrib/k8s-jobs.md) or the Indexer service.
+ The Indexer is an optional and experimental feature. If you're primarily performing batch ingestion, we recommend you use either the MiddleManager and Peon task execution system or [MiddleManager-less ingestion using Kubernetes](../development/extensions-core/k8s-jobs.md). If you're primarily doing streaming ingestion, you may want to try either [MiddleManager-less ingestion using Kubernetes](../development/extensions-core/k8s-jobs.md) or the Indexer service.
 :::
 
 The Apache Druid Indexer service is an alternative to the Middle Manager + Peon task execution system. Instead of forking a separate JVM process per-task, the Indexer runs tasks as separate threads within a single JVM process.