Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable dataset support for quidem tests #17699

Merged
merged 76 commits into from
Feb 10, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
76 commits
Select commit Hold shift + click to select a range
8fbbf73
first relevant commit w.r.t use cachingclientquerss
kgyrtkirk Jan 30, 2025
49377df
inline
kgyrtkirk Jan 30, 2025
7fbf287
try1
kgyrtkirk Jan 30, 2025
1c58dc8
Revert "try1"
kgyrtkirk Jan 30, 2025
95506a3
separate methopd
kgyrtkirk Jan 30, 2025
012fc9b
move emptywalker to guice module
kgyrtkirk Jan 30, 2025
a3a03b4
factor
kgyrtkirk Jan 30, 2025
567d0c8
extract
kgyrtkirk Jan 30, 2025
41baee4
inline impl
kgyrtkirk Jan 30, 2025
cbf16d0
inline
kgyrtkirk Jan 30, 2025
d8af313
one step closer
kgyrtkirk Jan 30, 2025
4f074fa
rely on @Inject
kgyrtkirk Jan 30, 2025
dcc72d4
rely on Inject instead of wiore in
kgyrtkirk Jan 30, 2025
2dd95c5
rely on Inject instead of wiore in
kgyrtkirk Jan 30, 2025
0c3a7ee
this far
kgyrtkirk Jan 30, 2025
336b7c0
xt
kgyrtkirk Jan 30, 2025
880d27b
Revert "xt"
kgyrtkirk Jan 30, 2025
30bd4bb
Merge commit 'f4912d1c6620caa40cbd02cbd8ef91fd68187a1b' into test-dat…
kgyrtkirk Feb 3, 2025
746c2d2
stuff from oteher branch
kgyrtkirk Feb 3, 2025
0dd0973
up
kgyrtkirk Feb 3, 2025
cdca192
a
kgyrtkirk Feb 3, 2025
7924d64
prep ingest
kgyrtkirk Feb 3, 2025
c76e376
update
kgyrtkirk Feb 3, 2025
57c96a1
make it run
kgyrtkirk Feb 4, 2025
292e3a5
add dep
kgyrtkirk Feb 4, 2025
cf7298e
format
kgyrtkirk Feb 4, 2025
f9e9f38
x
kgyrtkirk Feb 4, 2025
3d7a6df
u
kgyrtkirk Feb 4, 2025
8704ee0
closer
kgyrtkirk Feb 4, 2025
7006c3a
prepare to dissolve
kgyrtkirk Feb 4, 2025
608c0e7
x
kgyrtkirk Feb 4, 2025
396b022
f
kgyrtkirk Feb 4, 2025
1f5116c
talk
kgyrtkirk Feb 4, 2025
9bf97bb
remove
kgyrtkirk Feb 4, 2025
3d67ea7
tableconfig0
kgyrtkirk Feb 5, 2025
5a625b8
Revert "tableconfig0"
kgyrtkirk Feb 5, 2025
590d317
x
kgyrtkirk Feb 5, 2025
13ee04a
a
kgyrtkirk Feb 5, 2025
10623b0
au
kgyrtkirk Feb 5, 2025
f3cd723
s
kgyrtkirk Feb 5, 2025
0781a34
mx
kgyrtkirk Feb 5, 2025
c02433c
crap
kgyrtkirk Feb 5, 2025
74d6d0e
Revert "crap"
kgyrtkirk Feb 5, 2025
29a45d6
s
kgyrtkirk Feb 5, 2025
8d3ea2e
s
kgyrtkirk Feb 5, 2025
7923543
wood-wood
kgyrtkirk Feb 5, 2025
ac85179
move crap out
kgyrtkirk Feb 5, 2025
f478ee5
Reapply "tableconfig0"
kgyrtkirk Feb 5, 2025
8d856f8
a
kgyrtkirk Feb 5, 2025
2c78cad
finally
kgyrtkirk Feb 5, 2025
6f9d566
make more wood
kgyrtkirk Feb 5, 2025
aa60263
update
kgyrtkirk Feb 5, 2025
4ca3806
make relative work
kgyrtkirk Feb 5, 2025
dfd1b6d
rename option to datasets
kgyrtkirk Feb 5, 2025
e390cb2
move
kgyrtkirk Feb 5, 2025
871fa91
tries
kgyrtkirk Feb 5, 2025
07e139a
cleanup
kgyrtkirk Feb 5, 2025
b7a758d
fix tests
kgyrtkirk Feb 5, 2025
24c19d6
up
kgyrtkirk Feb 5, 2025
8c3a019
cleanup
kgyrtkirk Feb 5, 2025
e3d9b6f
fix hashcode/equals
kgyrtkirk Feb 5, 2025
0966a12
remove comment
kgyrtkirk Feb 5, 2025
5efb3c7
remove unnecessary changes
kgyrtkirk Feb 5, 2025
0e8cc13
fix style
kgyrtkirk Feb 5, 2025
78e3784
cleanup
kgyrtkirk Feb 5, 2025
d48cd79
x
kgyrtkirk Feb 5, 2025
8430ebd
Merge remote-tracking branch 'apache/master' into test-dataset-0
kgyrtkirk Feb 6, 2025
8c4f082
add test
kgyrtkirk Feb 6, 2025
7d1f9be
remove ex
kgyrtkirk Feb 6, 2025
46080b4
mandate directory for datasets
kgyrtkirk Feb 6, 2025
d028d04
move customdataset to sql
kgyrtkirk Feb 6, 2025
7677415
use TestSegmentsBroker
kgyrtkirk Feb 7, 2025
97ca2d3
clenaup
kgyrtkirk Feb 7, 2025
83ae6ac
alternates
kgyrtkirk Feb 7, 2025
8ae065b
Revert "alternates"
kgyrtkirk Feb 7, 2025
2d42004
checkstyle fix
kgyrtkirk Feb 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,13 @@
package org.apache.druid.benchmark.query;

import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.msq.sql.MSQTaskSqlEngine;
import org.apache.druid.msq.test.ExtractResultsFactory;
import org.apache.druid.msq.test.MSQTestOverlordServiceClient;
import org.apache.druid.msq.test.StandardMSQComponentSupplier;
import org.apache.druid.msq.util.MultiStageQueryContext;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
import org.apache.druid.sql.calcite.QueryTestBuilder;
Expand Down Expand Up @@ -202,18 +199,11 @@ public MSQComponentSupplier(TempDirProducer tempFolderProducer)
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
QueryRunnerFactoryConglomerate conglomerate,
JoinableFactoryWrapper joinableFactory,
Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
final SpecificSegmentsQuerySegmentWalker retVal = super.createQuerySegmentWalker(
conglomerate,
joinableFactory,
injector);
TestDataBuilder.attachIndexesForBenchmarkDatasource(retVal);
return retVal;
walker = super.addSegmentsToWalker(walker);
TestDataBuilder.attachIndexesForBenchmarkDatasource(walker);
return walker;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
package org.apache.druid.compressedbigdecimal;

import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import org.apache.druid.compressedbigdecimal.CompressedBigDecimalSqlAggregatorTestBase.CompressedBigDecimalComponentSupplier;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.InputRowSchema;
Expand All @@ -30,14 +29,12 @@
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
Expand Down Expand Up @@ -84,11 +81,7 @@ public DruidModule getCoreModule()
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
QueryableIndex index =
IndexBuilder.create()
Expand All @@ -106,7 +99,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(ROWS1)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(CalciteTests.DATASOURCE1)
.interval(index.getDataInterval())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,11 @@
package org.apache.druid.query.aggregation.tdigestsketch.sql;

import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
Expand All @@ -42,7 +40,6 @@
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
Expand Down Expand Up @@ -77,11 +74,7 @@ public DruidModule getCoreModule()
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
TDigestSketchModule.registerSerde();

Expand All @@ -106,7 +99,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(TestDataBuilder.ROWS1)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(CalciteTests.DATASOURCE1)
.interval(index.getDataInterval())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import org.apache.druid.error.DruidException;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.Intervals;
Expand All @@ -34,7 +33,6 @@
import org.apache.druid.query.BaseQuery;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
Expand Down Expand Up @@ -71,7 +69,6 @@
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
Expand Down Expand Up @@ -261,13 +258,8 @@ public DruidModule getCoreModule()
return DruidModuleCollection.of(super.getCoreModule(), new HllSketchModule());
}

@SuppressWarnings("resource")
@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
HllSketchModule.registerSerde();
final QueryableIndex index = IndexBuilder
Expand All @@ -292,7 +284,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(CalciteTests.DATASOURCE1)
.interval(index.getDataInterval())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.granularity.Granularities;
Expand All @@ -30,7 +29,6 @@
import org.apache.druid.query.JoinDataSource;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
Expand Down Expand Up @@ -59,7 +57,6 @@
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinType;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
Expand Down Expand Up @@ -97,11 +94,7 @@ public DruidModule getCoreModule()
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
DoublesSketchModule.registerSerde();

Expand All @@ -126,7 +119,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(TestDataBuilder.ROWS1)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(CalciteTests.DATASOURCE1)
.interval(index.getDataInterval())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import org.apache.druid.error.DruidException;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.Intervals;
Expand All @@ -31,7 +30,6 @@
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryDataSource;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
Expand All @@ -58,7 +56,6 @@
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
Expand Down Expand Up @@ -120,11 +117,7 @@ public DruidModule getCoreModule()
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
SketchModule.registerSerde();

Expand All @@ -151,7 +144,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(TestDataBuilder.ROWS1)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(DATA_SOURCE)
.interval(index.getDataInterval())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,11 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.aggregation.datasketches.tuple.ArrayOfDoublesSketchAggregatorFactory;
Expand All @@ -43,7 +41,6 @@
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
Expand Down Expand Up @@ -118,11 +115,7 @@ public DruidModule getCoreModule()
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
ArrayOfDoublesSketchModule.registerSerde();

Expand Down Expand Up @@ -150,7 +143,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(ROWS)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(DATA_SOURCE)
.interval(index.getDataInterval())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,11 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.guice.BloomFilterExtensionModule;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
Expand All @@ -44,7 +42,6 @@
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
Expand Down Expand Up @@ -83,11 +80,7 @@ public DruidModule getCoreModule()
}

@Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
)
public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker)
{
final QueryableIndex index =
IndexBuilder.create()
Expand All @@ -106,7 +99,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
.rows(TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS)
.buildMMappedIndex();

return SpecificSegmentsQuerySegmentWalker.createWalker(injector, conglomerate).add(
return walker.add(
DataSegment.builder()
.dataSource(DATA_SOURCE)
.interval(index.getDataInterval())
Expand Down
Loading
Loading