From c8077182870523b1a697c099a21708ba139a96b4 Mon Sep 17 00:00:00 2001
From: jinhelin <linjinhe33@gmail.com>
Date: Mon, 30 Dec 2024 14:23:39 +0800
Subject: [PATCH] *: Refine MutableSupport and DeltaMergeDefines (#9746)
ref pingcap/tiflash#9673
---
.../AddExtraTableIDColumnTransformAction.cpp | 4 +-
dbms/src/DataTypes/DataTypeString.cpp | 12 ++
dbms/src/DataTypes/DataTypeString.h | 5 +-
dbms/src/Databases/test/gtest_database.cpp | 20 +--
.../Debug/MockExecutor/TableScanBinder.cpp | 6 +-
dbms/src/Debug/dbgQueryCompiler.cpp | 6 +-
.../Coprocessor/DAGStorageInterpreter.cpp | 10 +-
.../Flash/Coprocessor/GenSchemaAndColumn.cpp | 36 +++--
dbms/src/Flash/Coprocessor/RemoteRequest.cpp | 8 +-
.../tests/gtest_trigger_pipeline_writer.cpp | 1 +
.../tests/bench_aggregation_hash_map.cpp | 3 +-
dbms/src/Interpreters/ExpressionAnalyzer.cpp | 6 +-
.../Interpreters/InterpreterAlterQuery.cpp | 15 +--
.../Interpreters/InterpreterDescribeQuery.cpp | 6 +-
.../Interpreters/InterpreterInsertQuery.cpp | 3 -
.../Interpreters/InterpreterManageQuery.cpp | 2 +-
dbms/src/Server/DTTool/DTToolBench.cpp | 14 +-
dbms/src/Server/DTTool/DTToolInspect.cpp | 3 +-
.../DeltaMerge/ColumnFile/ColumnFileBig.h | 4 +-
.../ColumnFileTinyVectorIndexReader.cpp | 2 +-
.../ColumnFileTinyVectorIndexWriter.cpp | 4 +-
.../Storages/DeltaMerge/DMDecoratorStreams.h | 2 +-
.../DMVersionFilterBlockInputStream.h | 6 +-
.../Decode/SSTFilesToDTFilesOutputStream.cpp | 10 +-
dbms/src/Storages/DeltaMerge/DeltaMerge.h | 4 +-
.../Storages/DeltaMerge/DeltaMergeDefines.cpp | 95 ++++++++++++++
.../Storages/DeltaMerge/DeltaMergeDefines.h | 113 +++-------------
.../Storages/DeltaMerge/DeltaMergeHelpers.h | 2 +-
.../Storages/DeltaMerge/DeltaMergeStore.cpp | 36 ++---
.../src/Storages/DeltaMerge/DeltaMergeStore.h | 10 +-
.../DeltaMerge/DeltaMergeStore_Ingest.cpp | 1 +
.../src/Storages/DeltaMerge/File/ColumnStat.h | 2 +
dbms/src/Storages/DeltaMerge/File/DMFile.cpp | 1 +
.../Storages/DeltaMerge/File/DMFileMetaV2.cpp | 2 +-
.../DeltaMerge/File/DMFilePackFilter.cpp | 6 +-
.../File/DMFilePackFilterResult.cpp | 1 +
.../DeltaMerge/File/DMFilePackFilterResult.h | 18 +--
.../Storages/DeltaMerge/File/DMFileReader.cpp | 20 +--
.../File/DMFileVectorIndexReader.cpp | 2 +-
.../File/DMFileVectorIndexWriter.cpp | 4 +-
.../DMFileWithVectorIndexBlockInputStream.cpp | 2 +-
.../Storages/DeltaMerge/File/DMFileWriter.cpp | 12 +-
.../Storages/DeltaMerge/File/DMFileWriter.h | 2 +-
.../Storages/DeltaMerge/Filter/FilterHelper.h | 7 +-
.../DeltaMerge/Filter/PushDownExecutor.cpp | 8 +-
.../Storages/DeltaMerge/Index/MinMaxIndex.cpp | 2 +-
.../DeltaMerge/PKSquashingBlockInputStream.h | 4 +-
dbms/src/Storages/DeltaMerge/RowKeyRange.h | 1 +
dbms/src/Storages/DeltaMerge/Segment.cpp | 31 ++---
.../DeltaMerge/convertColumnTypeHelpers.cpp | 2 +-
.../src/Storages/DeltaMerge/tests/DMTestEnv.h | 96 +++++++-------
.../DeltaMerge/tests/gtest_column_filter.cpp | 8 +-
.../DeltaMerge/tests/gtest_data_streams.cpp | 6 +-
.../DeltaMerge/tests/gtest_dm_column_file.cpp | 16 +--
.../tests/gtest_dm_delta_merge_store.cpp | 88 +++++++------
...est_dm_delta_merge_store_for_fast_scan.cpp | 6 +-
...test_dm_delta_merge_store_vector_index.cpp | 6 +-
.../DeltaMerge/tests/gtest_dm_file.cpp | 18 +--
.../tests/gtest_dm_meta_version.cpp | 44 +++----
.../DeltaMerge/tests/gtest_dm_segment.cpp | 4 +-
.../tests/gtest_dm_segment_common_handle.cpp | 72 +++++-----
.../tests/gtest_dm_simple_pk_test_basic.cpp | 4 +-
.../tests/gtest_dm_storage_delta_merge.cpp | 6 +-
.../DeltaMerge/tests/gtest_dm_utils.cpp | 12 +-
.../tests/gtest_dm_vector_index.cpp | 4 +-
.../tests/gtest_segment_read_task.cpp | 2 +-
.../tests/gtest_segment_read_task_pool.cpp | 1 +
.../gtest_segment_replace_stable_data.cpp | 22 ++--
.../tests/gtest_segment_test_basic.cpp | 44 ++++---
.../tests/gtest_sst_files_stream.cpp | 18 +--
.../DeltaMerge/tests/gtest_version_filter.cpp | 8 +-
.../DeltaMerge/workload/DTWorkload.cpp | 4 +-
.../DeltaMerge/workload/DataGenerator.cpp | 18 ++-
.../DeltaMerge/workload/TableGenerator.cpp | 4 +-
.../Storages/DeltaMerge/workload/Utils.cpp | 4 +-
dbms/src/Storages/ITableDeclaration.cpp | 4 +-
.../Decode/DecodingStorageSchemaSnapshot.cpp | 13 +-
.../KVStore/Decode/RegionBlockReader.cpp | 10 +-
.../KVStore/MultiRaft/PrehandleSnapshot.cpp | 2 +-
dbms/src/Storages/KVStore/Types.h | 8 --
.../tests/bench_region_block_reader.cpp | 10 +-
.../tests/gtest_region_block_reader.cpp | 40 +++---
.../Storages/KVStore/tests/gtest_spill.cpp | 6 +-
.../KVStore/tests/gtest_sync_status.cpp | 2 +-
dbms/src/Storages/MutableSupport.cpp | 57 +++++---
dbms/src/Storages/MutableSupport.h | 73 +++--------
dbms/src/Storages/S3/FileCache.cpp | 6 +-
.../src/Storages/S3/tests/gtest_filecache.cpp | 17 ++-
dbms/src/Storages/StorageDeltaMerge.cpp | 124 ++++++++++--------
.../System/StorageSystemDTLocalIndexes.cpp | 2 +-
.../System/StorageSystemDTSegments.cpp | 2 +-
.../Storages/System/StorageSystemDTTables.cpp | 2 +-
.../Storages/System/StorageSystemTables.cpp | 11 +-
dbms/src/TestUtils/InputStreamTestUtils.h | 2 +-
dbms/src/TiDB/Decode/RowCodec.cpp | 2 +-
dbms/src/TiDB/Schema/SchemaBuilder.cpp | 6 +-
dbms/src/TiDB/Schema/TiDB.cpp | 12 +-
dbms/src/TiDB/tests/RowCodecTestUtils.h | 26 ++--
dbms/src/TiDB/tests/gtest_row_v2.cpp | 30 ++---
99 files changed, 814 insertions(+), 744 deletions(-)
create mode 100644 dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp
diff --git a/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp b/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp
index 47b52d9eb98..a819743e9c7 100644
--- a/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp
+++ b/dbms/src/DataStreams/AddExtraTableIDColumnTransformAction.cpp
@@ -21,7 +21,7 @@ namespace DB
Block AddExtraTableIDColumnTransformAction::buildHeader(const Block & inner_header_, int extra_table_id_index)
{
auto header = inner_header_.cloneEmpty();
- if (extra_table_id_index != InvalidColumnID)
+ if (extra_table_id_index != MutSup::invalid_col_id)
{
const auto & extra_table_id_col_define = DM::getExtraTableIDColumnDefine();
ColumnWithTypeAndName col{
@@ -67,7 +67,7 @@ bool AddExtraTableIDColumnTransformAction::transform(Block & block, TableID phys
if (unlikely(!block))
return true;
- if (extra_table_id_index != InvalidColumnID)
+ if (extra_table_id_index != MutSup::invalid_col_id)
{
const auto & extra_table_id_col_define = DM::getExtraTableIDColumnDefine();
ColumnWithTypeAndName col{
diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp
index 939e1827a8a..864f388e870 100644
--- a/dbms/src/DataTypes/DataTypeString.cpp
+++ b/dbms/src/DataTypes/DataTypeString.cpp
@@ -24,6 +24,8 @@
#include <IO/WriteHelpers.h>
#include <Storages/FormatVersion.h>
+#include <magic_enum.hpp>
+
#if __SSE2__
#include <emmintrin.h>
#endif
@@ -517,4 +519,14 @@ String DataTypeString::getNullableDefaultName()
return fmt::format("Nullable({})", getDefaultName());
}
+std::span<const std::pair<String, DataTypePtr>> DataTypeString::getTiDBPkColumnStringNameAndTypes()
+{
+ static const auto name_and_types = std::array{
+ std::make_pair(NameV2, DataTypeFactory::instance().getOrSet(NameV2)),
+ std::make_pair(LegacyName, DataTypeFactory::instance().getOrSet(LegacyName)),
+ };
+ // Minus one for ignoring SerdesFormat::None.
+ static_assert(magic_enum::enum_count<SerdesFormat>() - 1 == name_and_types.size());
+ return name_and_types;
+}
} // namespace DB
diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h
index 8687e945864..c0611a58895 100644
--- a/dbms/src/DataTypes/DataTypeString.h
+++ b/dbms/src/DataTypes/DataTypeString.h
@@ -13,9 +13,10 @@
// limitations under the License.
#pragma once
-
#include <DataTypes/IDataType.h>
+#include <span>
+
namespace DB
{
class DataTypeString final : public IDataType
@@ -98,6 +99,8 @@ class DataTypeString final : public IDataType
static String getDefaultName();
static String getNullableDefaultName();
+ static std::span<const std::pair<String, DataTypePtr>> getTiDBPkColumnStringNameAndTypes();
+
explicit DataTypeString(SerdesFormat serdes_fmt_ = SerdesFormat::None);
private:
diff --git a/dbms/src/Databases/test/gtest_database.cpp b/dbms/src/Databases/test/gtest_database.cpp
index 8e13e69b558..f5021874de2 100644
--- a/dbms/src/Databases/test/gtest_database.cpp
+++ b/dbms/src/Databases/test/gtest_database.cpp
@@ -173,7 +173,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -257,7 +257,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -272,7 +272,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -373,7 +373,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -391,7 +391,7 @@ try
auto storage = db2->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -581,7 +581,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -597,7 +597,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -632,7 +632,7 @@ try
// Get storage from database
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -744,7 +744,7 @@ try
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
@@ -793,7 +793,7 @@ try
const auto * tbl_name = "t_45";
auto storage = db->tryGetTable(*ctx, tbl_name);
ASSERT_NE(storage, nullptr);
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), tbl_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
diff --git a/dbms/src/Debug/MockExecutor/TableScanBinder.cpp b/dbms/src/Debug/MockExecutor/TableScanBinder.cpp
index 833c3b064fe..fc33bdc6500 100644
--- a/dbms/src/Debug/MockExecutor/TableScanBinder.cpp
+++ b/dbms/src/Debug/MockExecutor/TableScanBinder.cpp
@@ -61,7 +61,7 @@ TableID TableScanBinder::getTableId() const
void TableScanBinder::setTipbColumnInfo(tipb::ColumnInfo * ci, const DAGColumnInfo & dag_column_info) const
{
auto names = splitQualifiedName(dag_column_info.first);
- if (names.column_name == MutableSupport::tidb_pk_column_name)
+ if (names.column_name == MutSup::extra_handle_column_name)
ci->set_column_id(-1);
else
ci->set_column_id(table_info.getColumnID(names.column_name));
@@ -142,10 +142,10 @@ ExecutorBinderPtr compileTableScan(
{
TiDB::ColumnInfo ci;
ci.tp = TiDB::TypeLongLong;
- ci.id = TiDBPkColumnID;
+ ci.id = MutSup::extra_handle_id;
ci.setPriKeyFlag();
ci.setNotNullFlag();
- ts_output.emplace_back(std::make_pair(MutableSupport::tidb_pk_column_name, std::move(ci)));
+ ts_output.emplace_back(std::make_pair(MutSup::extra_handle_column_name, std::move(ci)));
}
return std::make_shared<mock::TableScanBinder>(executor_index, ts_output, table_info, keep_order);
diff --git a/dbms/src/Debug/dbgQueryCompiler.cpp b/dbms/src/Debug/dbgQueryCompiler.cpp
index b8bd24253b0..1acc4ef7634 100644
--- a/dbms/src/Debug/dbgQueryCompiler.cpp
+++ b/dbms/src/Debug/dbgQueryCompiler.cpp
@@ -125,7 +125,7 @@ std::pair<ExecutorBinderPtr, bool> compileQueryBlock(
{
if (auto * identifier = typeid_cast<ASTIdentifier *>(expr.get()))
{
- if (identifier->getColumnName() == MutableSupport::tidb_pk_column_name)
+ if (identifier->getColumnName() == MutSup::extra_handle_column_name)
{
append_pk_column = true;
}
@@ -179,7 +179,7 @@ std::pair<ExecutorBinderPtr, bool> compileQueryBlock(
if (auto * identifier = typeid_cast<ASTIdentifier *>(expr.get()))
{
auto [db_name, table_name, column_name] = splitQualifiedName(identifier->getColumnName());
- if (column_name == MutableSupport::tidb_pk_column_name)
+ if (column_name == MutSup::extra_handle_column_name)
{
if (table_name.empty())
{
@@ -423,4 +423,4 @@ QueryTasks queryPlanToQueryTasks(
}
return tasks;
}
-} // namespace DB
\ No newline at end of file
+} // namespace DB
diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp
index 0dd45d77f5b..8fa8ac976be 100644
--- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp
+++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp
@@ -557,7 +557,7 @@ std::tuple<bool, String> compareColumns(
for (const auto & column : columns)
{
- // Exclude virtual columns, including EXTRA_HANDLE_COLUMN_ID, VERSION_COLUMN_ID,TAG_COLUMN_ID,EXTRA_TABLE_ID_COLUMN_ID
+ // Exclude virtual columns, including MutSup::extra_handle_id, MutSup::version_col_id,MutSup::delmark_col_id,MutSup::extra_table_id_col_id
if (column.id < 0)
{
continue;
@@ -1482,7 +1482,7 @@ std::unordered_map<TableID, DAGStorageInterpreter::StorageWithStructureLock> DAG
std::pair<Names, std::vector<UInt8>> DAGStorageInterpreter::getColumnsForTableScan()
{
// Get handle column name.
- String handle_column_name = MutableSupport::tidb_pk_column_name;
+ String handle_column_name = MutSup::extra_handle_column_name;
if (auto pk_handle_col = storage_for_logical_table->getTableInfo().getPKHandleColumn())
handle_column_name = pk_handle_col->get().name;
@@ -1504,10 +1504,10 @@ std::pair<Names, std::vector<UInt8>> DAGStorageInterpreter::getColumnsForTableSc
}
// Column ID -1 return the handle column
String name;
- if (cid == TiDBPkColumnID)
+ if (cid == MutSup::extra_handle_id)
name = handle_column_name;
- else if (cid == ExtraTableIDColumnID)
- name = MutableSupport::extra_table_id_column_name;
+ else if (cid == MutSup::extra_table_id_col_id)
+ name = MutSup::extra_table_id_column_name;
else
name = storage_for_logical_table->getTableInfo().getColumnName(cid);
required_columns_tmp.emplace_back(std::move(name));
diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp
index d7df2262cfb..b37a2984602 100644
--- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp
+++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp
@@ -26,13 +26,13 @@ namespace
DataTypePtr getPkType(const TiDB::ColumnInfo & column_info)
{
const auto & pk_data_type = getDataTypeByColumnInfoForComputingLayer(column_info);
- /// primary key type must be tidb_pk_column_int_type or tidb_pk_column_string_type.
+ /// primary key type must be getTiDBPkColumnIntType or getTiDBPkColumnStringType.
RUNTIME_CHECK(
- pk_data_type->equals(*MutableSupport::tidb_pk_column_int_type)
- || pk_data_type->equals(*MutableSupport::tidb_pk_column_string_type),
+ pk_data_type->equals(*MutSup::getExtraHandleColumnIntType())
+ || pk_data_type->equals(*MutSup::getExtraHandleColumnStringType()),
pk_data_type->getName(),
- MutableSupport::tidb_pk_column_int_type->getName(),
- MutableSupport::tidb_pk_column_string_type->getName());
+ MutSup::getExtraHandleColumnIntType()->getName(),
+ MutSup::getExtraHandleColumnStringType()->getName());
return pk_data_type;
}
} // namespace
@@ -70,13 +70,11 @@ NamesAndTypes genNamesAndTypes(const TiDB::ColumnInfos & column_infos, const Str
const auto & column_info = column_infos[i];
switch (column_info.id)
{
- case TiDBPkColumnID:
- names_and_types.emplace_back(MutableSupport::tidb_pk_column_name, getPkType(column_info));
+ case MutSup::extra_handle_id:
+ names_and_types.emplace_back(MutSup::extra_handle_column_name, getPkType(column_info));
break;
- case ExtraTableIDColumnID:
- names_and_types.emplace_back(
- MutableSupport::extra_table_id_column_name,
- MutableSupport::extra_table_id_column_type);
+ case MutSup::extra_table_id_col_id:
+ names_and_types.emplace_back(MutSup::extra_table_id_column_name, MutSup::getExtraTableIdColumnType());
break;
default:
names_and_types.emplace_back(
@@ -94,7 +92,7 @@ NamesAndTypes genNamesAndTypes(const TiDBTableScan & table_scan, const StringRef
std::tuple<DM::ColumnDefinesPtr, int> genColumnDefinesForDisaggregatedRead(const TiDBTableScan & table_scan)
{
auto column_defines = std::make_shared<DM::ColumnDefines>();
- int extra_table_id_index = InvalidColumnID;
+ int extra_table_id_index = MutSup::invalid_col_id;
column_defines->reserve(table_scan.getColumnSize());
for (Int32 i = 0; i < table_scan.getColumnSize(); ++i)
{
@@ -106,18 +104,18 @@ std::tuple<DM::ColumnDefinesPtr, int> genColumnDefinesForDisaggregatedRead(const
const auto output_name = genNameForExchangeReceiver(i);
switch (column_info.id)
{
- case TiDBPkColumnID:
+ case MutSup::extra_handle_id:
column_defines->emplace_back(DM::ColumnDefine{
- TiDBPkColumnID,
- output_name, // MutableSupport::tidb_pk_column_name
+ MutSup::extra_handle_id,
+ output_name, // MutSup::extra_handle_column_name
getPkType(column_info)});
break;
- case ExtraTableIDColumnID:
+ case MutSup::extra_table_id_col_id:
{
column_defines->emplace_back(DM::ColumnDefine{
- ExtraTableIDColumnID,
- output_name, // MutableSupport::extra_table_id_column_name
- MutableSupport::extra_table_id_column_type});
+ MutSup::extra_table_id_col_id,
+ output_name, // MutSup::extra_table_id_column_name
+ MutSup::getExtraTableIdColumnType()});
extra_table_id_index = i;
break;
}
diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp
index a6a05feb09f..13bca3793db 100644
--- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp
+++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp
@@ -44,7 +44,7 @@ RemoteRequest RemoteRequest::build(
auto * mutable_table_scan = ts_exec->mutable_tbl_scan();
table_scan.constructTableScanForRemoteRead(mutable_table_scan, table_info.id);
- String handle_column_name = MutableSupport::tidb_pk_column_name;
+ String handle_column_name = MutSup::extra_handle_column_name;
if (auto pk_handle_col = table_info.getPKHandleColumn())
handle_column_name = pk_handle_col->get().name;
@@ -53,7 +53,7 @@ RemoteRequest RemoteRequest::build(
const auto & col = table_scan.getColumns()[i];
auto col_id = col.id;
- if (col_id == DB::TiDBPkColumnID)
+ if (col_id == MutSup::extra_handle_id)
{
TiDB::ColumnInfo ci;
ci.tp = TiDB::TypeLongLong;
@@ -61,11 +61,11 @@ RemoteRequest RemoteRequest::build(
ci.setNotNullFlag();
schema.emplace_back(std::make_pair(handle_column_name, std::move(ci)));
}
- else if (col_id == ExtraTableIDColumnID)
+ else if (col_id == MutSup::extra_table_id_col_id)
{
TiDB::ColumnInfo ci;
ci.tp = TiDB::TypeLongLong;
- schema.emplace_back(std::make_pair(MutableSupport::extra_table_id_column_name, std::move(ci)));
+ schema.emplace_back(std::make_pair(MutSup::extra_table_id_column_name, std::move(ci)));
}
else
{
diff --git a/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp b/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp
index f4c1db357ba..b453aa69f11 100644
--- a/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp
+++ b/dbms/src/Flash/Mpp/tests/gtest_trigger_pipeline_writer.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <DataTypes/DataTypesNumber.h>
#include <Flash/Coprocessor/DAGContext.h>
#include <Flash/Coprocessor/DAGResponseWriter.h>
#include <Flash/Coprocessor/WaitResult.h>
diff --git a/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp b/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp
index 39c711a03c5..21a339c2a1f 100644
--- a/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp
+++ b/dbms/src/Flash/tests/bench_aggregation_hash_map.cpp
@@ -14,6 +14,7 @@
#include <AggregateFunctions/registerAggregateFunctions.h>
#include <Columns/ColumnDecimal.h>
+#include <Common/Stopwatch.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDecimal.h>
#include <DataTypes/DataTypeString.h>
@@ -310,7 +311,7 @@ try
LOG_DEBUG(log, "build_side_watch: {}, hashmap size: {}", build_side_watch.elapsed(), data_variants->size());
std::vector<AggregatedDataVariantsPtr> variants{data_variants};
- auto merging_buckets = aggregator->mergeAndConvertToBlocks(variants, /*final=*/true, /*max_thread=*/1);
+ auto merging_buckets = aggregator->mergeAndConvertToBlocks(variants, /*final=*/true, /*max_threads=*/1);
std::vector<Block> res_block;
Stopwatch probe_side_watch;
diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
index 765e8e224cf..8601df0110b 100644
--- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp
@@ -524,7 +524,7 @@ void ExpressionAnalyzer::analyzeAggregation()
{
NameSet unique_keys;
ASTs & group_asts = select_query->group_expression_list->children;
- for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i)
+ for (ssize_t i = 0; i < static_cast<ssize_t>(group_asts.size()); ++i)
{
ssize_t size = group_asts.size();
getRootActions(group_asts[i], true, false, temp_actions);
@@ -1037,7 +1037,7 @@ void ExpressionAnalyzer::normalizeTreeImpl(
if (storage && select_query && !select_query->raw_for_mutable)
{
// LOG_DEBUG(&Poco::Logger::get("ExpressionAnalyzer"), "Filter hidden columns for mutable table.");
- filtered_names = MutableSupport::instance().hiddenColumns(storage->getName());
+ filtered_names = MutSup::instance().hiddenColumns(storage->getName());
}
}
@@ -1392,7 +1392,7 @@ void ExpressionAnalyzer::optimizeGroupBy()
}
select_query->group_expression_list = std::make_shared<ASTExpressionList>();
- select_query->group_expression_list->children.emplace_back(std::make_shared<ASTLiteral>(UInt64(unused_column)));
+ select_query->group_expression_list->children.emplace_back(std::make_shared<ASTLiteral>(unused_column));
}
}
diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
index 240b43042c0..01c00793a03 100644
--- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
@@ -106,7 +106,7 @@ void InterpreterAlterQuery::parseAlter(
{
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
- OrderedNameSet filtered_names = MutableSupport::instance().hiddenColumns(table->getName());
+ OrderedNameSet filtered_names = MutSup::instance().hiddenColumns(table->getName());
for (const auto & params : params_container)
{
@@ -117,8 +117,7 @@ void InterpreterAlterQuery::parseAlter(
const auto & ast_col_decl = typeid_cast<const ASTColumnDeclaration &>(*params.col_decl);
- if (ast_col_decl.name == MutableSupport::version_column_name
- || ast_col_decl.name == MutableSupport::delmark_column_name)
+ if (ast_col_decl.name == MutSup::version_column_name || ast_col_decl.name == MutSup::delmark_column_name)
throw Exception(
"Internal column name can not be used: " + ast_col_decl.name,
ErrorCodes::ARGUMENT_OUT_OF_BOUND);
@@ -150,8 +149,7 @@ void InterpreterAlterQuery::parseAlter(
const Field & column_name = typeid_cast<const ASTIdentifier &>(*(params.column)).name;
- if (column_name == MutableSupport::version_column_name
- || column_name == MutableSupport::delmark_column_name)
+ if (column_name == MutSup::version_column_name || column_name == MutSup::delmark_column_name)
{
FieldVisitorToString to_string;
auto err_msg = "Internal column name can not be dropped: " + applyVisitor(to_string, column_name);
@@ -170,8 +168,8 @@ void InterpreterAlterQuery::parseAlter(
command.type = AlterCommand::DROP_COLUMN;
command.column_name = typeid_cast<const ASTIdentifier &>(*(params.column)).name;
- if (command.column_name == MutableSupport::version_column_name
- || command.column_name == MutableSupport::delmark_column_name)
+ if (command.column_name == MutSup::version_column_name
+ || command.column_name == MutSup::delmark_column_name)
throw Exception(
"Internal column name can not be dropped: " + command.column_name,
ErrorCodes::ARGUMENT_OUT_OF_BOUND);
@@ -186,8 +184,7 @@ void InterpreterAlterQuery::parseAlter(
const auto & ast_col_decl = typeid_cast<const ASTColumnDeclaration &>(*params.col_decl);
- if (ast_col_decl.name == MutableSupport::version_column_name
- || ast_col_decl.name == MutableSupport::delmark_column_name)
+ if (ast_col_decl.name == MutSup::version_column_name || ast_col_decl.name == MutSup::delmark_column_name)
throw Exception(
"Internal column name can not be modified: " + ast_col_decl.name,
ErrorCodes::ARGUMENT_OUT_OF_BOUND);
diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
index 36e4b9363aa..b7a87734ef8 100644
--- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -71,7 +71,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
ColumnDefaults column_defaults;
StoragePtr table;
- auto table_expression = typeid_cast<const ASTTableExpression *>(ast.table_expression.get());
+ const auto * table_expression = typeid_cast<const ASTTableExpression *>(ast.table_expression.get());
if (table_expression->subquery)
{
@@ -82,7 +82,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
{
if (table_expression->table_function)
{
- auto table_function = typeid_cast<const ASTFunction *>(table_expression->table_function.get());
+ const auto * table_function = typeid_cast<const ASTFunction *>(table_expression->table_function.get());
/// Get the table function
TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function->name, context);
/// Run it and remember the result
@@ -123,7 +123,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl()
Block sample_block = getSampleBlock();
MutableColumns res_columns = sample_block.cloneEmptyColumns();
- const OrderedNameSet filtered_names = MutableSupport::instance().hiddenColumns(table->getName());
+ const OrderedNameSet filtered_names = MutSup::instance().hiddenColumns(table->getName());
for (const auto & column : columns)
{
diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
index 71ea49e3e02..8a3b4253715 100644
--- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp
@@ -111,9 +111,6 @@ BlockIO InterpreterInsertQuery::execute()
checkAccess(query);
StoragePtr table = getTable(query);
- // if (table->getName() == MutableSupport::txn_storage_name)
- // throw Exception(MutableSupport::txn_storage_name + " doesn't support Insert", ErrorCodes::LOGICAL_ERROR);
-
auto table_lock = table->lockStructureForShare(context.getCurrentQueryId());
NamesAndTypesList required_columns = table->getColumns().getAllPhysical();
diff --git a/dbms/src/Interpreters/InterpreterManageQuery.cpp b/dbms/src/Interpreters/InterpreterManageQuery.cpp
index 765e87480e7..b572853720e 100644
--- a/dbms/src/Interpreters/InterpreterManageQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterManageQuery.cpp
@@ -28,7 +28,7 @@ BlockIO InterpreterManageQuery::execute()
StoragePtr table = context.getTable(ast.database, ast.table);
StorageDeltaMerge * manageable_storage;
- if (table->getName() == MutableSupport::delta_tree_storage_name)
+ if (table->getName() == MutSup::delta_tree_storage_name)
{
manageable_storage = &dynamic_cast<StorageDeltaMerge &>(*table);
}
diff --git a/dbms/src/Server/DTTool/DTToolBench.cpp b/dbms/src/Server/DTTool/DTToolBench.cpp
index c5085d7b3e6..040a668708c 100644
--- a/dbms/src/Server/DTTool/DTToolBench.cpp
+++ b/dbms/src/Server/DTTool/DTToolBench.cpp
@@ -108,7 +108,7 @@ DB::Block createBlock(
Block block;
//PK
{
- ColumnWithTypeAndName pk_col(nullptr, EXTRA_HANDLE_COLUMN_INT_TYPE, "id", EXTRA_HANDLE_COLUMN_ID);
+ ColumnWithTypeAndName pk_col(nullptr, MutSup::getExtraHandleColumnIntType(), "id", MutSup::extra_handle_id);
IColumn::MutablePtr m_col = pk_col.type->createColumn();
for (size_t i = 0; i < row_number; i++)
{
@@ -121,7 +121,11 @@ DB::Block createBlock(
}
// Version
{
- ColumnWithTypeAndName version_col({}, VERSION_COLUMN_TYPE, VERSION_COLUMN_NAME, VERSION_COLUMN_ID);
+ ColumnWithTypeAndName version_col(
+ {},
+ MutSup::getVersionColumnType(),
+ MutSup::version_column_name,
+ MutSup::version_col_id);
IColumn::MutablePtr m_col = version_col.type->createColumn();
for (size_t i = 0; i < row_number; ++i)
{
@@ -135,7 +139,11 @@ DB::Block createBlock(
//Tag
{
- ColumnWithTypeAndName tag_col(nullptr, TAG_COLUMN_TYPE, TAG_COLUMN_NAME, TAG_COLUMN_ID);
+ ColumnWithTypeAndName tag_col(
+ nullptr,
+ MutSup::getDelmarkColumnType(),
+ MutSup::delmark_column_name,
+ MutSup::delmark_col_id);
IColumn::MutablePtr m_col = tag_col.type->createColumn();
auto & column_data = typeid_cast<ColumnVector<UInt8> &>(*m_col).getData();
column_data.resize(row_number);
diff --git a/dbms/src/Server/DTTool/DTToolInspect.cpp b/dbms/src/Server/DTTool/DTToolInspect.cpp
index 4ab155b5c95..c906d15c36b 100644
--- a/dbms/src/Server/DTTool/DTToolInspect.cpp
+++ b/dbms/src/Server/DTTool/DTToolInspect.cpp
@@ -150,7 +150,8 @@ int inspectServiceMain(DB::Context & context, const InspectArgs & args)
{
for (const auto & c : all_cols)
{
- if (c.id == DB::TiDBPkColumnID || c.id == DB::VersionColumnID || c.id == DB::DelMarkColumnID)
+ if (c.id == DB::MutSup::extra_handle_id || c.id == DB::MutSup::version_col_id
+ || c.id == DB::MutSup::delmark_col_id)
cols_to_dump.emplace_back(c);
}
}
diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h
index eb9f6f90f01..0e2b515bb07 100644
--- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h
+++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileBig.h
@@ -175,14 +175,14 @@ class ColumnFileBigReader : public ColumnFileReader
{
if (col_defs_->size() == 1)
{
- if ((*col_defs)[0].id == EXTRA_HANDLE_COLUMN_ID)
+ if ((*col_defs)[0].id == MutSup::extra_handle_id)
{
pk_ver_only = true;
}
}
else if (col_defs_->size() == 2)
{
- if ((*col_defs)[0].id == EXTRA_HANDLE_COLUMN_ID && (*col_defs)[1].id == VERSION_COLUMN_ID)
+ if ((*col_defs)[0].id == MutSup::extra_handle_id && (*col_defs)[1].id == MutSup::version_col_id)
{
pk_ver_only = true;
}
diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp
index fe07ad9da12..59d96035d5c 100644
--- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp
+++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexReader.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/EventRecorder.h>
#include <IO/Buffer/ReadBufferFromString.h>
#include <IO/Compression/CompressedReadBuffer.h>
#include <Storages/DeltaMerge/ColumnFile/ColumnFileDataProvider.h>
@@ -20,7 +21,6 @@
#include <Storages/DeltaMerge/Index/VectorIndexCache.h>
#include <Storages/DeltaMerge/Index/VectorSearchPerf.h>
-
namespace DB::DM
{
diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp
index f7a8e014847..5903092652f 100644
--- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp
+++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFileTinyVectorIndexWriter.cpp
@@ -144,7 +144,7 @@ ColumnFileTinyPtr ColumnFileTinyVectorIndexWriter::buildIndexForFile(
break;
RUNTIME_CHECK(block.columns() == read_columns->size());
- RUNTIME_CHECK(block.getByPosition(0).column_id == TAG_COLUMN_ID);
+ RUNTIME_CHECK(block.getByPosition(0).column_id == MutSup::delmark_col_id);
auto del_mark_col = block.safeGetByPosition(0).column;
RUNTIME_CHECK(del_mark_col != nullptr);
@@ -218,7 +218,7 @@ ColumnFileTinys ColumnFileTinyVectorIndexWriter::build(ProceedCheckFn should_pro
column_defines = getColumnDefinesFromBlock(schema->getSchema());
const auto del_cd_iter
= std::find_if(column_defines.cbegin(), column_defines.cend(), [](const ColumnDefine & cd) {
- return cd.id == TAG_COLUMN_ID;
+ return cd.id == MutSup::delmark_col_id;
});
RUNTIME_CHECK_MSG(
del_cd_iter != column_defines.cend(),
diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h
index d24404ecc74..897b2e6c119 100644
--- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h
+++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h
@@ -40,7 +40,7 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream
, log(Logger::get(tracing_id))
{
children.emplace_back(input);
- delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME);
+ delete_col_pos = input->getHeader().getPositionByName(MutSup::delmark_column_name);
}
~DMDeleteFilterBlockInputStream() override
{
diff --git a/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h b/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h
index 40d00927d01..d4c63be93ce 100644
--- a/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h
+++ b/dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h
@@ -62,9 +62,9 @@ class DMVersionFilterBlockInputStream : public IBlockInputStream
auto input_header = input->getHeader();
- handle_col_pos = input_header.getPositionByName(EXTRA_HANDLE_COLUMN_NAME);
- version_col_pos = input_header.getPositionByName(VERSION_COLUMN_NAME);
- delete_col_pos = input_header.getPositionByName(TAG_COLUMN_NAME);
+ handle_col_pos = input_header.getPositionByName(MutSup::extra_handle_column_name);
+ version_col_pos = input_header.getPositionByName(MutSup::version_column_name);
+ delete_col_pos = input_header.getPositionByName(MutSup::delmark_column_name);
}
~DMVersionFilterBlockInputStream() override
diff --git a/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp b/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp
index 3a4a47975a7..53136529b60 100644
--- a/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp
+++ b/dbms/src/Storages/DeltaMerge/Decode/SSTFilesToDTFilesOutputStream.cpp
@@ -275,8 +275,8 @@ void SSTFilesToDTFilesOutputStream<ChildStream>::write()
{
// Check whether rows are sorted by handle & version in ascending order.
SortDescription sort;
- sort.emplace_back(MutableSupport::tidb_pk_column_name, 1, 0);
- sort.emplace_back(MutableSupport::version_column_name, 1, 0);
+ sort.emplace_back(MutSup::extra_handle_column_name, 1, 0);
+ sort.emplace_back(MutSup::version_column_name, 1, 0);
if (unlikely(block.rows() > 1 && !isAlreadySorted(block, sort)))
{
@@ -288,8 +288,8 @@ void SSTFilesToDTFilesOutputStream<ChildStream>::write()
const size_t nrows = block.rows();
for (size_t i = 0; i < nrows; ++i)
{
- const auto & pk_col = block.getByName(MutableSupport::tidb_pk_column_name);
- const auto & ver_col = block.getByName(MutableSupport::version_column_name);
+ const auto & pk_col = block.getByName(MutSup::extra_handle_column_name);
+ const auto & ver_col = block.getByName(MutSup::version_column_name);
LOG_ERROR(
log,
"[Row={}/{}] [pk={}] [ver={}]",
@@ -385,7 +385,7 @@ void SSTFilesToDTFilesOutputStream<ChildStream>::updateRangeFromNonEmptyBlock(Bl
RUNTIME_CHECK(block.rows() > 0);
- const auto & pk_col = block.getByName(MutableSupport::tidb_pk_column_name);
+ const auto & pk_col = block.getByName(MutSup::extra_handle_column_name);
const auto rowkey_column = RowKeyColumnContainer(pk_col.column, schema_snap->is_common_handle);
auto & current_file_range = ingest_files_range.back();
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMerge.h b/dbms/src/Storages/DeltaMerge/DeltaMerge.h
index c3178e71825..167c7a794b9 100644
--- a/dbms/src/Storages/DeltaMerge/DeltaMerge.h
+++ b/dbms/src/Storages/DeltaMerge/DeltaMerge.h
@@ -217,8 +217,8 @@ class DeltaMergeBlockInputStream final
{
// In some cases like Segment::getSplitPointSlow, only handle column in block.
if (block.columns() < 2 //
- || block.getByPosition(0).column_id != EXTRA_HANDLE_COLUMN_ID //
- || block.getByPosition(1).column_id != VERSION_COLUMN_ID)
+ || block.getByPosition(0).column_id != MutSup::extra_handle_id //
+ || block.getByPosition(1).column_id != MutSup::version_col_id)
return;
++num_read;
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp
new file mode 100644
index 00000000000..66f38de11ab
--- /dev/null
+++ b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.cpp
@@ -0,0 +1,95 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <DataTypes/DataTypeString.h>
+#include <Storages/DeltaMerge/DeltaMergeDefines.h>
+
+namespace DB::DM
+{
+
+const ColumnDefine & getExtraIntHandleColumnDefine()
+{
+ static const ColumnDefine int_handle_cd{
+ MutSup::extra_handle_id,
+ MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnIntType()};
+ return int_handle_cd;
+}
+
+namespace
+{
+auto getExtraStringHandleColumnDefines()
+{
+ const auto names_and_types = ::DB::DataTypeString::getTiDBPkColumnStringNameAndTypes();
+ std::vector<std::pair<String, ColumnDefine>> name_and_column_defines;
+ name_and_column_defines.reserve(names_and_types.size());
+ for (const auto & name_and_type : names_and_types)
+ {
+ name_and_column_defines.emplace_back(
+ name_and_type.first,
+ ColumnDefine{MutSup::extra_handle_id, MutSup::extra_handle_column_name, name_and_type.second});
+ }
+ return name_and_column_defines;
+}
+} // namespace
+const ColumnDefine & getExtraStringHandleColumnDefine()
+{
+ static const auto name_and_column_defines = getExtraStringHandleColumnDefines();
+
+ const auto default_name = ::DB::DataTypeString::getDefaultName();
+ auto itr = std::find_if(
+ name_and_column_defines.begin(),
+ name_and_column_defines.end(),
+ [&default_name](const auto & name_and_column_define) { return name_and_column_define.first == default_name; });
+ RUNTIME_CHECK_MSG(
+ itr != name_and_column_defines.end(),
+ "Cannot find '{}' for TiDB primary key column string type",
+ default_name);
+ return itr->second;
+}
+
+const ColumnDefine & getExtraHandleColumnDefine(bool is_common_handle)
+{
+ if (is_common_handle)
+ return getExtraStringHandleColumnDefine();
+ return getExtraIntHandleColumnDefine();
+}
+
+const ColumnDefine & getVersionColumnDefine()
+{
+ static const ColumnDefine version_cd{
+ MutSup::version_col_id,
+ MutSup::version_column_name,
+ MutSup::getVersionColumnType()};
+ return version_cd;
+}
+
+const ColumnDefine & getTagColumnDefine()
+{
+ static const ColumnDefine delmark_cd{
+ MutSup::delmark_col_id,
+ MutSup::delmark_column_name,
+ MutSup::getDelmarkColumnType()};
+ return delmark_cd;
+}
+
+const ColumnDefine & getExtraTableIDColumnDefine()
+{
+ static const ColumnDefine table_id_cd{
+ MutSup::extra_table_id_col_id,
+ MutSup::extra_table_id_column_name,
+ MutSup::getExtraTableIdColumnType()};
+ return table_id_cd;
+}
+} // namespace DB::DM
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h
index 3c8c7256a00..3a95b7e4aee 100644
--- a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h
+++ b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h
@@ -16,20 +16,15 @@
#include <Common/Allocator.h>
#include <Common/ArenaWithFreeLists.h>
-#include <Common/EventRecorder.h>
-#include <Core/NamesAndTypes.h>
+#include <Core/Block.h>
#include <Core/Types.h>
-#include <DataTypes/DataTypeFactory.h>
+#include <DataTypes/IDataType.h>
#include <Storages/DeltaMerge/ColumnDefine_fwd.h>
-#include <Storages/DeltaMerge/Range.h>
-#include <Storages/FormatVersion.h>
#include <Storages/KVStore/Types.h>
#include <Storages/MutableSupport.h>
-#include <TiDB/Schema/VectorIndex.h>
#include <limits>
#include <memory>
-#include <mutex>
#include <optional>
namespace TiDB
@@ -37,9 +32,7 @@ namespace TiDB
struct TableInfo;
} // namespace TiDB
-namespace DB
-{
-namespace DM
+namespace DB::DM
{
/// S: Scale factor of delta tree node.
/// M: Capacity factor of leaf node.
@@ -49,8 +42,6 @@ static constexpr size_t DT_S = 3;
static constexpr size_t DT_M = 55;
static constexpr size_t DT_F = 20;
-using Ids = std::vector<UInt64>;
-
template <class ValueSpace, size_t M, size_t F, size_t S = 3, typename TAllocator = Allocator<false>>
class DeltaTree;
@@ -60,7 +51,6 @@ class DTEntryIterator;
template <size_t M, size_t F, size_t S, typename TAllocator = Allocator<false>>
class DTEntriesCopy;
-
struct EmptyValueSpace
{
void removeFromInsert(UInt64) {}
@@ -70,16 +60,10 @@ using EntryIterator = DTEntryIterator<DT_M, DT_F, DT_S>;
using DefaultDeltaTree = DeltaTree<EmptyValueSpace, DT_M, DT_F, DT_S, ArenaWithFreeLists>;
using DeltaTreePtr = std::shared_ptr<DefaultDeltaTree>;
using BlockPtr = std::shared_ptr<Block>;
-
-using RowId = UInt64;
using ColId = DB::ColumnID;
-using Handle = DB::HandleID;
-
using ColIds = std::vector<ColId>;
-using HandlePair = std::pair<Handle, Handle>;
-
+using Handle = DB::HandleID;
using RowsAndBytes = std::pair<size_t, size_t>;
-
using OptionTableInfoConstRef = std::optional<std::reference_wrapper<const TiDB::TableInfo>>;
struct ColumnDefine
@@ -97,70 +81,21 @@ struct ColumnDefine
{}
};
-using ColumnMap = std::unordered_map<ColId, ColumnPtr>;
-using MutableColumnMap = std::unordered_map<ColId, MutableColumnPtr>;
-using LockGuard = std::lock_guard<std::mutex>;
-
-inline static const UInt64 INITIAL_EPOCH = 0;
-
-// TODO maybe we should use those variables instead of macros?
-#define EXTRA_HANDLE_COLUMN_NAME ::DB::MutableSupport::tidb_pk_column_name
-#define VERSION_COLUMN_NAME ::DB::MutableSupport::version_column_name
-#define TAG_COLUMN_NAME ::DB::MutableSupport::delmark_column_name
-#define EXTRA_TABLE_ID_COLUMN_NAME ::DB::MutableSupport::extra_table_id_column_name
-
-#define EXTRA_HANDLE_COLUMN_ID ::DB::TiDBPkColumnID
-#define VERSION_COLUMN_ID ::DB::VersionColumnID
-#define TAG_COLUMN_ID ::DB::DelMarkColumnID
-#define EXTRA_TABLE_ID_COLUMN_ID ::DB::ExtraTableIDColumnID
-
-#define EXTRA_HANDLE_COLUMN_INT_TYPE ::DB::MutableSupport::tidb_pk_column_int_type
-#define EXTRA_HANDLE_COLUMN_STRING_TYPE ::DB::MutableSupport::tidb_pk_column_string_type
-#define VERSION_COLUMN_TYPE ::DB::MutableSupport::version_column_type
-#define TAG_COLUMN_TYPE ::DB::MutableSupport::delmark_column_type
-#define EXTRA_TABLE_ID_COLUMN_TYPE ::DB::MutableSupport::extra_table_id_column_type
-
-inline const ColumnDefine & getExtraIntHandleColumnDefine()
-{
- static ColumnDefine EXTRA_HANDLE_COLUMN_DEFINE_{
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_INT_TYPE};
- return EXTRA_HANDLE_COLUMN_DEFINE_;
-}
-inline const ColumnDefine & getExtraStringHandleColumnDefine()
+inline auto format_as(const ColumnDefine & cd)
{
- static ColumnDefine EXTRA_HANDLE_COLUMN_DEFINE_{
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_STRING_TYPE};
- return EXTRA_HANDLE_COLUMN_DEFINE_;
-}
-inline const ColumnDefine & getExtraHandleColumnDefine(bool is_common_handle)
-{
- if (is_common_handle)
- return getExtraStringHandleColumnDefine();
- return getExtraIntHandleColumnDefine();
-}
-inline const ColumnDefine & getVersionColumnDefine()
-{
- static ColumnDefine VERSION_COLUMN_DEFINE_{VERSION_COLUMN_ID, VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE};
- return VERSION_COLUMN_DEFINE_;
-}
-inline const ColumnDefine & getTagColumnDefine()
-{
- static ColumnDefine TAG_COLUMN_DEFINE_{TAG_COLUMN_ID, TAG_COLUMN_NAME, TAG_COLUMN_TYPE};
- return TAG_COLUMN_DEFINE_;
-}
-inline const ColumnDefine & getExtraTableIDColumnDefine()
-{
- static ColumnDefine EXTRA_TABLE_ID_COLUMN_DEFINE_{
- EXTRA_TABLE_ID_COLUMN_ID,
- EXTRA_TABLE_ID_COLUMN_NAME,
- EXTRA_TABLE_ID_COLUMN_TYPE};
- return EXTRA_TABLE_ID_COLUMN_DEFINE_;
+ return fmt::format("{}/{}", cd.id, cd.type->getName());
}
+inline static constexpr UInt64 INITIAL_EPOCH = 0;
+inline static constexpr bool DM_RUN_CHECK = true;
+
+ALWAYS_INLINE const ColumnDefine & getExtraIntHandleColumnDefine();
+ALWAYS_INLINE const ColumnDefine & getExtraStringHandleColumnDefine();
+ALWAYS_INLINE const ColumnDefine & getExtraHandleColumnDefine(bool is_common_handle);
+ALWAYS_INLINE const ColumnDefine & getVersionColumnDefine();
+ALWAYS_INLINE const ColumnDefine & getTagColumnDefine();
+ALWAYS_INLINE const ColumnDefine & getExtraTableIDColumnDefine();
+
static_assert(
static_cast<Int64>(static_cast<UInt64>(std::numeric_limits<Int64>::min())) == std::numeric_limits<Int64>::min(),
"Unsupported compiler!");
@@ -168,8 +103,6 @@ static_assert(
static_cast<Int64>(static_cast<UInt64>(std::numeric_limits<Int64>::max())) == std::numeric_limits<Int64>::max(),
"Unsupported compiler!");
-static constexpr bool DM_RUN_CHECK = true;
-
struct Attr
{
String col_name;
@@ -177,16 +110,4 @@ struct Attr
DataTypePtr type;
};
using Attrs = std::vector<Attr>;
-
-} // namespace DM
-} // namespace DB
-
-template <>
-struct fmt::formatter<DB::DM::ColumnDefine>
-{
- template <typename FormatContext>
- auto format(const DB::DM::ColumnDefine & cd, FormatContext & ctx) const -> decltype(ctx.out())
- {
- return fmt::format_to(ctx.out(), "{}/{}", cd.id, cd.type->getName());
- }
-};
+} // namespace DB::DM
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h
index 0c0cc49c54e..ca50f38bde5 100644
--- a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h
+++ b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h
@@ -79,7 +79,7 @@ inline SortDescription getPkSort(const ColumnDefine & handle)
{
SortDescription sort;
sort.emplace_back(handle.name, 1, 1);
- sort.emplace_back(VERSION_COLUMN_NAME, 1, 1);
+ sort.emplace_back(MutSup::version_column_name, 1, 1);
return sort;
}
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
index 17a3cc29df6..3b61f57b502 100644
--- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
+++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/EventRecorder.h>
#include <Common/Exception.h>
#include <Common/FailPoint.h>
#include <Common/FmtUtils.h>
@@ -199,7 +200,8 @@ ColumnDefinesPtr generateStoreColumns(const ColumnDefines & table_columns, bool
// Add other columns
for (const auto & col : table_columns)
{
- if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME)
+ if (col.name != MutSup::extra_handle_column_name && col.name != MutSup::version_column_name
+ && col.name != MutSup::delmark_column_name)
columns->emplace_back(col);
}
return columns;
@@ -265,7 +267,8 @@ DeltaMergeStore::DeltaMergeStore(
original_table_columns.emplace_back(getTagColumnDefine());
for (const auto & col : columns)
{
- if (col.id != original_table_handle_define.id && col.id != VERSION_COLUMN_ID && col.id != TAG_COLUMN_ID)
+ if (col.id != original_table_handle_define.id && col.id != MutSup::version_col_id
+ && col.id != MutSup::delmark_col_id)
original_table_columns.emplace_back(col);
}
@@ -559,15 +562,15 @@ Block DeltaMergeStore::addExtraColumnIfNeed(
{
if (pkIsHandle(handle_define))
{
- if (!EXTRA_HANDLE_COLUMN_INT_TYPE->equals(*handle_define.type))
+ if (!MutSup::getExtraHandleColumnIntType()->equals(*handle_define.type))
{
auto handle_pos = getPosByColumnId(block, handle_define.id);
addColumnToBlock(
block, //
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
- EXTRA_HANDLE_COLUMN_INT_TYPE->createColumn());
+ MutSup::extra_handle_id,
+ MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnIntType(),
+ MutSup::getExtraHandleColumnIntType()->createColumn());
// Fill the new handle column with data in column[handle_pos] by applying cast.
DefaultExecutable(FunctionToInt64::create(db_context)).execute(block, {handle_pos}, block.columns() - 1);
}
@@ -580,9 +583,9 @@ Block DeltaMergeStore::addExtraColumnIfNeed(
ColumnPtr handle_column = pk_column->cloneResized(pk_column->size());
addColumnToBlock(
block, //
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnIntType(),
handle_column);
}
}
@@ -607,7 +610,7 @@ DM::WriteResult DeltaMergeStore::write(
if (db_context.getSettingsRef().dt_log_record_version)
{
- const auto & ver_col = block.getByName(VERSION_COLUMN_NAME).column;
+ const auto & ver_col = block.getByName(MutSup::version_column_name).column;
const auto * ver = toColumnVectorDataPtr<UInt64>(ver_col);
std::unordered_set<UInt64> dedup_ver;
for (auto v : *ver)
@@ -616,7 +619,7 @@ DM::WriteResult DeltaMergeStore::write(
}
std::unordered_set<Int64> dedup_handles;
- auto extra_handle_col = tryGetByColumnId(block, EXTRA_HANDLE_COLUMN_ID);
+ auto extra_handle_col = tryGetByColumnId(block, MutSup::extra_handle_id);
if (extra_handle_col.column)
{
if (extra_handle_col.type->getTypeId() == TypeIndex::Int64)
@@ -643,8 +646,8 @@ DM::WriteResult DeltaMergeStore::write(
{
// Sort the block by handle & version in ascending order.
SortDescription sort;
- sort.emplace_back(EXTRA_HANDLE_COLUMN_NAME, 1, 0);
- sort.emplace_back(VERSION_COLUMN_NAME, 1, 0);
+ sort.emplace_back(MutSup::extra_handle_column_name, 1, 0);
+ sort.emplace_back(MutSup::version_column_name, 1, 0);
if (rows > 1 && !isAlreadySorted(block, sort))
stableSortBlock(block, sort);
@@ -654,7 +657,7 @@ DM::WriteResult DeltaMergeStore::write(
size_t offset = 0;
size_t limit;
- const auto handle_column = block.getByName(EXTRA_HANDLE_COLUMN_NAME).column;
+ const auto handle_column = block.getByName(MutSup::extra_handle_column_name).column;
auto rowkey_column = RowKeyColumnContainer(handle_column, is_common_handle);
// Write block by segments
@@ -1986,7 +1989,8 @@ void DeltaMergeStore::applySchemaChanges(TiDB::TableInfo & table_info)
while (iter != new_original_table_columns.end())
{
// remove the extra columns
- if (iter->id == EXTRA_HANDLE_COLUMN_ID || iter->id == VERSION_COLUMN_ID || iter->id == TAG_COLUMN_ID)
+ if (iter->id == MutSup::extra_handle_id || iter->id == MutSup::version_col_id
+ || iter->id == MutSup::delmark_col_id)
{
iter++;
continue;
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h
index d72041946df..df8bbb90924 100644
--- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h
+++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h
@@ -435,7 +435,7 @@ class DeltaMergeStore
size_t num_streams,
bool keep_order,
const SegmentIdSet & read_segments = {},
- size_t extra_table_id_index = InvalidColumnID);
+ size_t extra_table_id_index = MutSup::invalid_col_id);
/// Read all rows without MVCC filtering
void readRaw(
@@ -447,7 +447,7 @@ class DeltaMergeStore
size_t num_streams,
bool keep_order,
const SegmentIdSet & read_segments = {},
- size_t extra_table_id_index = InvalidColumnID);
+ size_t extra_table_id_index = MutSup::invalid_col_id);
/// Read rows in two modes:
/// when is_fast_scan == false, we will read rows with MVCC filtering, del mark !=0 filter and sorted merge.
@@ -468,7 +468,7 @@ class DeltaMergeStore
bool is_fast_scan = false,
size_t expected_block_size = DEFAULT_BLOCK_SIZE,
const SegmentIdSet & read_segments = {},
- size_t extra_table_id_index = InvalidColumnID,
+ size_t extra_table_id_index = MutSup::invalid_col_id,
ScanContextPtr scan_context = nullptr);
@@ -493,7 +493,7 @@ class DeltaMergeStore
bool is_fast_scan = false,
size_t expected_block_size = DEFAULT_BLOCK_SIZE,
const SegmentIdSet & read_segments = {},
- size_t extra_table_id_index = InvalidColumnID,
+ size_t extra_table_id_index = MutSup::invalid_col_id,
ScanContextPtr scan_context = nullptr);
Remote::DisaggPhysicalTableReadSnapshotPtr writeNodeBuildRemoteReadSnapshot(
@@ -618,7 +618,7 @@ class DeltaMergeStore
const String & tracing_id = "",
ScanContextPtr scan_context = nullptr);
- static bool pkIsHandle(const ColumnDefine & handle_define) { return handle_define.id != EXTRA_HANDLE_COLUMN_ID; }
+ static bool pkIsHandle(const ColumnDefine & handle_define) { return handle_define.id != MutSup::extra_handle_id; }
/// Try to stall the writing. It will suspend the current thread if flow control is necessary.
/// There are roughly two flow control mechanisms:
diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp
index 55354d28078..02aa0d8d3bb 100644
--- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp
+++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore_Ingest.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/EventRecorder.h>
#include <Common/FailPoint.h>
#include <Common/TiFlashMetrics.h>
#include <Interpreters/Context.h>
diff --git a/dbms/src/Storages/DeltaMerge/File/ColumnStat.h b/dbms/src/Storages/DeltaMerge/File/ColumnStat.h
index 13eb4078e01..a6a6cd8fadf 100644
--- a/dbms/src/Storages/DeltaMerge/File/ColumnStat.h
+++ b/dbms/src/Storages/DeltaMerge/File/ColumnStat.h
@@ -14,10 +14,12 @@
#pragma once
+#include <DataTypes/DataTypeFactory.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Storages/DeltaMerge/DeltaMergeDefines.h>
#include <Storages/DeltaMerge/dtpb/dmfile.pb.h>
+#include <Storages/FormatVersion.h>
#include <Storages/KVStore/Types.h>
namespace DB::DM
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp
index 21048578a4b..b9750c77983 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp
@@ -13,6 +13,7 @@
// limitations under the License.
#include <Common/FailPoint.h>
+#include <Common/typeid_cast.h>
#include <IO/FileProvider/FileProvider.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp
index a5f3f9810e6..231809c6662 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileMetaV2.cpp
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <DataTypes/DataTypeNullable.h>
#include <IO/Buffer/ReadBufferFromString.h>
#include <IO/FileProvider/FileProvider.h>
#include <Storages/DeltaMerge/File/DMFileMetaV2.h>
-
namespace DB::ErrorCodes
{
extern const int CORRUPTED_DATA;
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp
index d18c79108d8..a532c9d1ad9 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.cpp
@@ -13,6 +13,7 @@
// limitations under the License.
#include <Common/Exception.h>
+#include <Common/Stopwatch.h>
#include <Common/TiFlashMetrics.h>
#include <IO/FileProvider/ChecksumReadBufferBuilder.h>
#include <Storages/DeltaMerge/File/DMFilePackFilter.h>
@@ -21,7 +22,6 @@
#include <Storages/DeltaMerge/RowKeyRange.h>
#include <Storages/DeltaMerge/ScanContext.h>
-
namespace DB::DM
{
@@ -56,7 +56,7 @@ DMFilePackFilterResultPtr DMFilePackFilter::load()
auto read_all_packs = (rowkey_ranges.size() == 1 && rowkey_ranges[0].all()) || rowkey_ranges.empty();
if (!read_all_packs)
{
- tryLoadIndex(result.param, EXTRA_HANDLE_COLUMN_ID);
+ tryLoadIndex(result.param, MutSup::extra_handle_id);
std::vector<RSOperatorPtr> handle_filters;
for (auto & rowkey_range : rowkey_ranges)
handle_filters.emplace_back(toFilter(rowkey_range));
@@ -65,7 +65,7 @@ DMFilePackFilterResultPtr DMFilePackFilter::load()
if (!rowkey_ranges.empty())
{
bool is_common_handle = rowkey_ranges.begin()->is_common_handle;
- auto handle_col_type = dmfile->getColumnStat(EXTRA_HANDLE_COLUMN_ID).type;
+ auto handle_col_type = dmfile->getColumnStat(MutSup::extra_handle_id).type;
if (is_common_handle)
RUNTIME_CHECK_MSG(
handle_col_type->getTypeId() == TypeIndex::String,
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp
index b5778efeb86..9d809930761 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/Stopwatch.h>
#include <Storages/DeltaMerge/File/DMFilePackFilter.h>
#include <Storages/DeltaMerge/File/DMFilePackFilterResult.h>
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h
index dadf054fd88..49314f13205 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h
+++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilterResult.h
@@ -46,9 +46,9 @@ class DMFilePackFilterResult
const FileProviderPtr & file_provider,
const ScanContextPtr & scan_context) const
{
- if (!param.indexes.count(EXTRA_HANDLE_COLUMN_ID))
- tryLoadIndex(dmfile, EXTRA_HANDLE_COLUMN_ID, file_provider, scan_context);
- auto & minmax_index = param.indexes.find(EXTRA_HANDLE_COLUMN_ID)->second.minmax;
+ if (!param.indexes.count(MutSup::extra_handle_id))
+ tryLoadIndex(dmfile, MutSup::extra_handle_id, file_provider, scan_context);
+ auto & minmax_index = param.indexes.find(MutSup::extra_handle_id)->second.minmax;
return minmax_index->getIntMinMax(pack_id).first;
}
@@ -58,9 +58,9 @@ class DMFilePackFilterResult
const FileProviderPtr & file_provider,
const ScanContextPtr & scan_context) const
{
- if (!param.indexes.count(EXTRA_HANDLE_COLUMN_ID))
- tryLoadIndex(dmfile, EXTRA_HANDLE_COLUMN_ID, file_provider, scan_context);
- auto & minmax_index = param.indexes.find(EXTRA_HANDLE_COLUMN_ID)->second.minmax;
+ if (!param.indexes.count(MutSup::extra_handle_id))
+ tryLoadIndex(dmfile, MutSup::extra_handle_id, file_provider, scan_context);
+ auto & minmax_index = param.indexes.find(MutSup::extra_handle_id)->second.minmax;
return minmax_index->getStringMinMax(pack_id).first;
}
@@ -70,9 +70,9 @@ class DMFilePackFilterResult
const FileProviderPtr & file_provider,
const ScanContextPtr & scan_context) const
{
- if (!param.indexes.count(VERSION_COLUMN_ID))
- tryLoadIndex(dmfile, VERSION_COLUMN_ID, file_provider, scan_context);
- auto & minmax_index = param.indexes.find(VERSION_COLUMN_ID)->second.minmax;
+ if (!param.indexes.count(MutSup::version_col_id))
+ tryLoadIndex(dmfile, MutSup::version_col_id, file_provider, scan_context);
+ auto & minmax_index = param.indexes.find(MutSup::version_col_id)->second.minmax;
return minmax_index->getUInt64MinMax(pack_id).second;
}
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp
index b0afccc6d91..e8b64bb10f4 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp
@@ -227,7 +227,7 @@ Block DMFileReader::readWithFilter(const IColumn::Filter & filter)
bool DMFileReader::isCacheableColumn(const ColumnDefine & cd)
{
- return cd.id == EXTRA_HANDLE_COLUMN_ID || cd.id == VERSION_COLUMN_ID;
+ return cd.id == MutSup::extra_handle_id || cd.id == MutSup::version_col_id;
}
Block DMFileReader::read()
@@ -257,7 +257,7 @@ Block DMFileReader::readImpl(const ReadBlockInfo & read_info)
/// 2. Find packs can do clean read.
const bool need_read_extra_columns = std::any_of(read_columns.cbegin(), read_columns.cend(), [](const auto & cd) {
- return cd.id == EXTRA_HANDLE_COLUMN_ID || cd.id == TAG_COLUMN_ID || cd.id == VERSION_COLUMN_ID;
+ return cd.id == MutSup::extra_handle_id || cd.id == MutSup::delmark_col_id || cd.id == MutSup::version_col_id;
});
const auto & pack_stats = dmfile->getPackStats();
const auto & pack_properties = dmfile->getPackProperties();
@@ -334,13 +334,13 @@ Block DMFileReader::readImpl(const ReadBlockInfo & read_info)
// For handle, tag and version column, we can try to do clean read.
switch (cd.id)
{
- case EXTRA_HANDLE_COLUMN_ID:
+ case MutSup::extra_handle_id:
col = readExtraColumn(cd, start_pack_id, pack_count, read_rows, handle_column_clean_read_packs);
break;
- case TAG_COLUMN_ID:
+ case MutSup::delmark_col_id:
col = readExtraColumn(cd, start_pack_id, pack_count, read_rows, del_column_clean_read_packs);
break;
- case VERSION_COLUMN_ID:
+ case MutSup::version_col_id:
col = readExtraColumn(cd, start_pack_id, pack_count, read_rows, version_column_clean_read_packs);
break;
default:
@@ -371,7 +371,7 @@ ColumnPtr DMFileReader::cleanRead(
{
switch (cd.id)
{
- case EXTRA_HANDLE_COLUMN_ID:
+ case MutSup::extra_handle_id:
{
if (is_common_handle)
{
@@ -384,11 +384,11 @@ ColumnPtr DMFileReader::cleanRead(
return cd.type->createColumnConst(rows_count, Field(min_handle));
}
}
- case TAG_COLUMN_ID:
+ case MutSup::delmark_col_id:
{
return cd.type->createColumnConst(rows_count, Field(static_cast<UInt64>(pack_stats[range.first].first_tag)));
}
- case VERSION_COLUMN_ID:
+ case MutSup::version_col_id:
{
return cd.type->createColumnConst(
rows_count,
@@ -412,11 +412,11 @@ ColumnPtr DMFileReader::readExtraColumn(
size_t read_rows,
const std::vector<size_t> & clean_read_packs)
{
- assert(cd.id == EXTRA_HANDLE_COLUMN_ID || cd.id == TAG_COLUMN_ID || cd.id == VERSION_COLUMN_ID);
+ assert(cd.id == MutSup::extra_handle_id || cd.id == MutSup::delmark_col_id || cd.id == MutSup::version_col_id);
const auto & pack_stats = dmfile->getPackStats();
auto read_strategy = ColumnCache::getCleanReadStrategy(start_pack_id, pack_count, clean_read_packs);
- if (read_strategy.size() != 1 && cd.id == EXTRA_HANDLE_COLUMN_ID)
+ if (read_strategy.size() != 1 && cd.id == MutSup::extra_handle_id)
{
// If size of read_strategy is not 1, handle can not do clean read.
read_strategy.clear();
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp
index f25090b390a..6e1f987ddbc 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/Stopwatch.h>
#include <Common/TiFlashMetrics.h>
#include <Storages/DeltaMerge/File/DMFile.h>
#include <Storages/DeltaMerge/File/DMFileVectorIndexReader.h>
@@ -21,7 +22,6 @@
#include <Storages/S3/FileCache.h>
#include <Storages/S3/FileCachePerf.h>
-
namespace DB::ErrorCodes
{
extern const int S3_ERROR;
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp
index 78f415206a4..3b69242cd8e 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexWriter.cpp
@@ -90,7 +90,7 @@ size_t DMFileVectorIndexWriter::buildIndexForFile(const DMFilePtr & dm_file_muta
{
const auto column_defines = dm_file_mutable->getColumnDefines();
const auto del_cd_iter = std::find_if(column_defines.cbegin(), column_defines.cend(), [](const ColumnDefine & cd) {
- return cd.id == TAG_COLUMN_ID;
+ return cd.id == MutSup::delmark_col_id;
});
RUNTIME_CHECK_MSG(
del_cd_iter != column_defines.cend(),
@@ -175,7 +175,7 @@ size_t DMFileVectorIndexWriter::buildIndexForFile(const DMFilePtr & dm_file_muta
break;
RUNTIME_CHECK(block.columns() == num_cols);
- RUNTIME_CHECK(block.getByPosition(0).column_id == TAG_COLUMN_ID);
+ RUNTIME_CHECK(block.getByPosition(0).column_id == MutSup::delmark_col_id);
auto del_mark_col = block.safeGetByPosition(0).column;
RUNTIME_CHECK(del_mark_col != nullptr);
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp
index a7e65ab9461..b1c07a04275 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.cpp
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/Stopwatch.h>
#include <Storages/DeltaMerge/File/DMFileWithVectorIndexBlockInputStream.h>
#include <Storages/DeltaMerge/ScanContext.h>
-
namespace DB::DM
{
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp
index fe8a6b3d505..43300ce3cda 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.cpp
@@ -62,7 +62,7 @@ DMFileWriter::DMFileWriter(
// TODO: currently we only generate index for Integers, Date, DateTime types, and this should be configurable by user.
/// for handle column always generate index
auto type = removeNullable(cd.type);
- bool do_index = cd.id == EXTRA_HANDLE_COLUMN_ID || type->isInteger() || type->isDateOrDateTime();
+ bool do_index = cd.id == MutSup::extra_handle_id || type->isInteger() || type->isDateOrDateTime();
addStreams(cd.id, cd.type, do_index);
dmfile->meta->getColumnStats().emplace(
@@ -142,7 +142,7 @@ void DMFileWriter::write(const Block & block, const BlockProperty & block_proper
stat.not_clean = block_property.not_clean_rows;
stat.bytes = block.bytes(); // This is bytes of pack data in memory.
- auto del_mark_column = tryGetByColumnId(block, TAG_COLUMN_ID).column;
+ auto del_mark_column = tryGetByColumnId(block, MutSup::delmark_col_id).column;
const ColumnVector<UInt8> * del_mark
= !del_mark_column ? nullptr : static_cast<const ColumnVector<UInt8> *>(del_mark_column.get());
@@ -152,9 +152,9 @@ void DMFileWriter::write(const Block & block, const BlockProperty & block_proper
const auto & col = getByColumnId(block, cd.id).column;
writeColumn(cd.id, *cd.type, *col, del_mark);
- if (cd.id == VERSION_COLUMN_ID)
+ if (cd.id == MutSup::version_col_id)
stat.first_version = col->get64(0);
- else if (cd.id == TAG_COLUMN_ID)
+ else if (cd.id == MutSup::delmark_col_id)
stat.first_tag = static_cast<UInt8>(col->get64(0));
}
@@ -204,12 +204,12 @@ void DMFileWriter::writeColumn(
auto & stream = column_streams.at(name);
if (stream->minmaxes)
{
- // For EXTRA_HANDLE_COLUMN_ID, we ignore del_mark when add minmax index.
+ // For MutSup::extra_handle_id, we ignore del_mark when add minmax index.
// Because we need all rows which satisfy a certain range when place delta index no matter whether the row is a delete row.
// For TAG Column, we also ignore del_mark when add minmax index.
stream->minmaxes->addPack(
column,
- (col_id == EXTRA_HANDLE_COLUMN_ID || col_id == TAG_COLUMN_ID) ? nullptr : del_mark);
+ (col_id == MutSup::extra_handle_id || col_id == MutSup::delmark_col_id) ? nullptr : del_mark);
}
/// There could already be enough data to compress into the new block.
diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h
index 1871c9bef29..ce7693e6dea 100644
--- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h
+++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h
@@ -16,6 +16,7 @@
#include <DataStreams/IBlockOutputStream.h>
#include <DataStreams/MarkInCompressedFile.h>
+#include <DataTypes/DataTypeNullable.h>
#include <IO/Buffer/WriteBufferFromOStream.h>
#include <IO/Compression/CompressedWriteBuffer.h>
#include <IO/FileProvider/ChecksumWriteBufferBuilder.h>
@@ -23,7 +24,6 @@
#include <Storages/DeltaMerge/File/DMFile.h>
#include <Storages/DeltaMerge/Index/MinMaxIndex.h>
-
namespace DB::DM
{
diff --git a/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h b/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h
index 396fb5e3941..1e322e62457 100644
--- a/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h
+++ b/dbms/src/Storages/DeltaMerge/Filter/FilterHelper.h
@@ -24,9 +24,10 @@ namespace DB::DM
inline RSOperatorPtr toFilter(RowKeyRange & rowkey_range)
{
Attr handle_attr = {
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- rowkey_range.is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ rowkey_range.is_common_handle ? MutSup::getExtraHandleColumnStringType()
+ : MutSup::getExtraHandleColumnIntType(),
};
if (rowkey_range.is_common_handle)
{
diff --git a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp
index de666f3e622..006dcb8ddf8 100644
--- a/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp
+++ b/dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp
@@ -87,9 +87,11 @@ PushDownExecutorPtr PushDownExecutor::build(
source_columns_of_analyzer.emplace_back(col_name, data_type);
continue;
}
- if (cid == EXTRA_TABLE_ID_COLUMN_ID)
+ if (cid == MutSup::extra_table_id_col_id)
{
- source_columns_of_analyzer.emplace_back(EXTRA_TABLE_ID_COLUMN_NAME, EXTRA_TABLE_ID_COLUMN_TYPE);
+ source_columns_of_analyzer.emplace_back(
+ MutSup::extra_table_id_column_name,
+ MutSup::getExtraTableIdColumnType());
continue;
}
RUNTIME_CHECK_MSG(columns_to_read_map.contains(cid), "ColumnID({}) not found in columns_to_read_map", cid);
@@ -146,7 +148,7 @@ PushDownExecutorPtr PushDownExecutor::build(
for (size_t i = 0; i < table_scan_column_info.size(); ++i)
{
if (table_scan_column_info[i].hasGeneratedColumnFlag()
- || table_scan_column_info[i].id == EXTRA_TABLE_ID_COLUMN_ID)
+ || table_scan_column_info[i].id == MutSup::extra_table_id_col_id)
continue;
auto col = columns_to_read_map.at(table_scan_column_info[i].id);
RUNTIME_CHECK_MSG(
diff --git a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp
index 16d3d493a5b..7a8b35a7c27 100644
--- a/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp
+++ b/dbms/src/Storages/DeltaMerge/Index/MinMaxIndex.cpp
@@ -21,6 +21,7 @@
#include <DataTypes/DataTypeFixedString.h>
#include <DataTypes/DataTypeMyDate.h>
#include <DataTypes/DataTypeMyDateTime.h>
+#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypesNumber.h>
@@ -28,7 +29,6 @@
#include <Storages/DeltaMerge/DeltaMergeHelpers.h>
#include <Storages/DeltaMerge/Index/MinMaxIndex.h>
#include <Storages/DeltaMerge/Index/RoughCheck.h>
-
namespace DB::DM
{
diff --git a/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h b/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h
index fa24b1cca16..1debb7b5a8b 100644
--- a/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h
+++ b/dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h
@@ -176,8 +176,8 @@ class PKSquashingBlockInputStream final : public IBlockInputStream
{
// Sort by handle & version in ascending order.
static SortDescription sort{
- SortColumnDescription{EXTRA_HANDLE_COLUMN_NAME, 1, 0},
- SortColumnDescription{VERSION_COLUMN_NAME, 1, 0}};
+ SortColumnDescription{MutSup::extra_handle_column_name, 1, 0},
+ SortColumnDescription{MutSup::version_column_name, 1, 0}};
if (block.rows() > 1 && !isAlreadySorted(block, sort))
stableSortBlock(block, sort);
}
diff --git a/dbms/src/Storages/DeltaMerge/RowKeyRange.h b/dbms/src/Storages/DeltaMerge/RowKeyRange.h
index 5387439ca86..0fe50ebcfe1 100644
--- a/dbms/src/Storages/DeltaMerge/RowKeyRange.h
+++ b/dbms/src/Storages/DeltaMerge/RowKeyRange.h
@@ -20,6 +20,7 @@
#include <IO/WriteHelpers.h>
#include <Storages/DeltaMerge/DeltaMergeDefines.h>
#include <Storages/DeltaMerge/DeltaMergeHelpers.h>
+#include <Storages/DeltaMerge/Range.h>
#include <Storages/DeltaMerge/dtpb/column_file.pb.h>
#include <Storages/KVStore/Decode/DecodedTiKVKeyValue.h>
#include <Storages/KVStore/MultiRaft/RegionRangeKeys.h>
diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp
index e139e1e8bf6..3a23b522188 100644
--- a/dbms/src/Storages/DeltaMerge/Segment.cpp
+++ b/dbms/src/Storages/DeltaMerge/Segment.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/EventRecorder.h>
#include <Common/Exception.h>
#include <Common/Stopwatch.h>
#include <Common/SyncPoint/SyncPoint.h>
@@ -1009,9 +1010,9 @@ bool Segment::useCleanRead(const SegmentSnapshotPtr & segment_snap, const Column
{
return segment_snap->delta->getRows() == 0 //
&& segment_snap->delta->getDeletes() == 0 //
- && !hasColumn(columns_to_read, EXTRA_HANDLE_COLUMN_ID) //
- && !hasColumn(columns_to_read, VERSION_COLUMN_ID) //
- && !hasColumn(columns_to_read, TAG_COLUMN_ID);
+ && !hasColumn(columns_to_read, MutSup::extra_handle_id) //
+ && !hasColumn(columns_to_read, MutSup::version_col_id) //
+ && !hasColumn(columns_to_read, MutSup::delmark_col_id);
}
BlockInputStreamPtr Segment::getInputStreamModeNormal(
@@ -1148,7 +1149,7 @@ BlockInputStreamPtr Segment::getInputStreamForDataExport(
{
data_stream = std::make_shared<PKSquashingBlockInputStream<false>>(
data_stream,
- EXTRA_HANDLE_COLUMN_ID,
+ MutSup::extra_handle_id,
is_common_handle);
}
data_stream = std::make_shared<DMVersionFilterBlockInputStream<DMVersionFilterMode::COMPACT>>(
@@ -1204,11 +1205,11 @@ BlockInputStreamPtr Segment::getInputStreamModeFast(
for (const auto & c : columns_to_read)
{
- if (c.id == EXTRA_HANDLE_COLUMN_ID)
+ if (c.id == MutSup::extra_handle_id)
{
enable_handle_clean_read = false;
}
- else if (c.id == TAG_COLUMN_ID)
+ else if (c.id == MutSup::delmark_col_id)
{
enable_del_clean_read = false;
}
@@ -1280,7 +1281,7 @@ BlockInputStreamPtr Segment::getInputStreamModeRaw(
for (const auto & c : columns_to_read)
{
- if (c.id != EXTRA_HANDLE_COLUMN_ID)
+ if (c.id != MutSup::extra_handle_id)
new_columns_to_read->push_back(c);
}
@@ -2130,7 +2131,7 @@ std::optional<Segment::SplitInfo> Segment::prepareSplitPhysical( //
my_data = std::make_shared<DMRowKeyFilterBlockInputStream<true>>(my_data, my_ranges, 0);
my_data
- = std::make_shared<PKSquashingBlockInputStream<false>>(my_data, EXTRA_HANDLE_COLUMN_ID, is_common_handle);
+ = std::make_shared<PKSquashingBlockInputStream<false>>(my_data, MutSup::extra_handle_id, is_common_handle);
my_data = std::make_shared<DMVersionFilterBlockInputStream<DMVersionFilterMode::COMPACT>>(
my_data,
*read_info.read_columns,
@@ -2161,7 +2162,7 @@ std::optional<Segment::SplitInfo> Segment::prepareSplitPhysical( //
other_data = std::make_shared<DMRowKeyFilterBlockInputStream<true>>(other_data, other_ranges, 0);
other_data = std::make_shared<PKSquashingBlockInputStream<false>>(
other_data,
- EXTRA_HANDLE_COLUMN_ID,
+ MutSup::extra_handle_id,
is_common_handle);
other_data = std::make_shared<DMVersionFilterBlockInputStream<DMVersionFilterMode::COMPACT>>(
other_data,
@@ -2362,7 +2363,7 @@ StableValueSpacePtr Segment::prepareMerge(
stream = std::make_shared<DMRowKeyFilterBlockInputStream<true>>(stream, rowkey_ranges, 0);
stream = std::make_shared<PKSquashingBlockInputStream<false>>(
stream,
- EXTRA_HANDLE_COLUMN_ID,
+ MutSup::extra_handle_id,
dm_context.is_common_handle);
stream = std::make_shared<DMVersionFilterBlockInputStream<DMVersionFilterMode::COMPACT>>(
stream,
@@ -2703,7 +2704,7 @@ ColumnDefinesPtr Segment::arrangeReadColumns(const ColumnDefine & handle, const
for (const auto & c : columns_to_read)
{
- if (c.id != handle.id && c.id != VERSION_COLUMN_ID && c.id != TAG_COLUMN_ID)
+ if (c.id != handle.id && c.id != MutSup::version_col_id && c.id != MutSup::delmark_col_id)
new_columns_to_read.push_back(c);
}
@@ -3310,9 +3311,9 @@ SkippableBlockInputStreamPtr Segment::getConcatSkippableBlockInputStream(
{
static constexpr bool NeedRowID = false;
// set `is_fast_scan` to true to try to enable clean read
- auto enable_handle_clean_read = !hasColumn(columns_to_read, EXTRA_HANDLE_COLUMN_ID);
+ auto enable_handle_clean_read = !hasColumn(columns_to_read, MutSup::extra_handle_id);
constexpr auto is_fast_scan = true;
- auto enable_del_clean_read = !hasColumn(columns_to_read, TAG_COLUMN_ID);
+ auto enable_del_clean_read = !hasColumn(columns_to_read, MutSup::version_col_id);
SkippableBlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream(
dm_context,
@@ -3366,9 +3367,9 @@ std::tuple<SkippableBlockInputStreamPtr, bool> Segment::getConcatVectorIndexBloc
{
static constexpr bool NeedRowID = false;
// set `is_fast_scan` to true to try to enable clean read
- auto enable_handle_clean_read = !hasColumn(columns_to_read, EXTRA_HANDLE_COLUMN_ID);
+ auto enable_handle_clean_read = !hasColumn(columns_to_read, MutSup::extra_handle_id);
constexpr auto is_fast_scan = true;
- auto enable_del_clean_read = !hasColumn(columns_to_read, TAG_COLUMN_ID);
+ auto enable_del_clean_read = !hasColumn(columns_to_read, MutSup::delmark_col_id);
SkippableBlockInputStreamPtr stable_stream = segment_snap->stable->tryGetInputStreamWithVectorIndex(
dm_context,
diff --git a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp
index a2fada6afe3..39e8fa54b29 100644
--- a/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp
+++ b/dbms/src/Storages/DeltaMerge/convertColumnTypeHelpers.cpp
@@ -18,6 +18,7 @@
#include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypeMyDate.h>
#include <DataTypes/DataTypeMyDateTime.h>
+#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/isSupportedDataTypeCast.h>
@@ -25,7 +26,6 @@
#include <Storages/DeltaMerge/DeltaMergeHelpers.h>
#include <Storages/DeltaMerge/convertColumnTypeHelpers.h>
#include <common/types.h>
-
namespace DB
{
namespace ErrorCodes
diff --git a/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h
index 8ba2a9e29ef..fadeacd785e 100644
--- a/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h
+++ b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h
@@ -170,7 +170,8 @@ class DMTestEnv
columns->emplace_back(getExtraHandleColumnDefine(/*is_common_handle=*/true));
break;
case PkType::PkIsHandleInt64:
- columns->emplace_back(ColumnDefine{PK_ID_PK_IS_HANDLE, PK_NAME_PK_IS_HANDLE, EXTRA_HANDLE_COLUMN_INT_TYPE});
+ columns->emplace_back(
+ ColumnDefine{PK_ID_PK_IS_HANDLE, PK_NAME_PK_IS_HANDLE, MutSup::getExtraHandleColumnIntType()});
break;
case PkType::PkIsHandleInt32:
columns->emplace_back(
@@ -196,18 +197,19 @@ class DMTestEnv
switch (pk_type)
{
case PkType::HiddenTiDBRowID:
- columns.push_back({EXTRA_HANDLE_COLUMN_NAME, EXTRA_HANDLE_COLUMN_INT_TYPE});
+ columns.push_back({MutSup::extra_handle_column_name, MutSup::getExtraHandleColumnIntType()});
break;
case PkType::CommonHandle:
columns.push_back(
{PK_NAME_PK_IS_HANDLE,
- EXTRA_HANDLE_COLUMN_STRING_TYPE}); // For common handle, there must be a user-given primary key.
+ MutSup::
+ getExtraHandleColumnStringType()}); // For common handle, there must be a user-given primary key.
columns.push_back(
- {EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_STRING_TYPE}); // For common handle, a _tidb_rowid is also constructed.
+ {MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnStringType()}); // For common handle, a _tidb_rowid is also constructed.
break;
case PkType::PkIsHandleInt64:
- columns.emplace_back(PK_NAME_PK_IS_HANDLE, EXTRA_HANDLE_COLUMN_INT_TYPE);
+ columns.emplace_back(PK_NAME_PK_IS_HANDLE, MutSup::getExtraHandleColumnIntType());
break;
case PkType::PkIsHandleInt32:
throw Exception("PkIsHandleInt32 is unsupported");
@@ -266,10 +268,10 @@ class DMTestEnv
switch (pk_type)
{
case PkType::HiddenTiDBRowID:
- name = EXTRA_HANDLE_COLUMN_NAME;
+ name = MutSup::extra_handle_column_name;
break;
case PkType::CommonHandle:
- name = EXTRA_HANDLE_COLUMN_NAME;
+ name = MutSup::extra_handle_column_name;
break;
case PkType::PkIsHandleInt64:
name = PK_NAME_PK_IS_HANDLE;
@@ -297,8 +299,8 @@ class DMTestEnv
bool reversed,
UInt64 tso = 2,
const String & pk_name_ = pk_name,
- ColumnID pk_col_id = EXTRA_HANDLE_COLUMN_ID,
- DataTypePtr pk_type = EXTRA_HANDLE_COLUMN_INT_TYPE,
+ ColumnID pk_col_id = MutSup::extra_handle_id,
+ DataTypePtr pk_type = MutSup::getExtraHandleColumnIntType(),
bool is_common_handle = false,
size_t rowkey_column_size = 1,
bool with_internal_columns = true,
@@ -327,15 +329,15 @@ class DMTestEnv
pk_name_,
pk_col_id});
// add extra column if need
- if (pk_col_id != EXTRA_HANDLE_COLUMN_ID)
+ if (pk_col_id != MutSup::extra_handle_id)
{
block.insert(ColumnWithTypeAndName{
DB::tests::makeColumn<Int64>(
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::getExtraHandleColumnIntType(),
createNumbers<Int64>(beg, end, reversed)),
- EXTRA_HANDLE_COLUMN_INT_TYPE,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID});
+ MutSup::getExtraHandleColumnIntType(),
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id});
}
}
if (with_internal_columns)
@@ -343,13 +345,13 @@ class DMTestEnv
// version_col
block.insert(DB::tests::createColumn<UInt64>(
std::vector<UInt64>(num_rows, tso),
- VERSION_COLUMN_NAME,
- VERSION_COLUMN_ID));
+ MutSup::version_column_name,
+ MutSup::version_col_id));
// tag_col
block.insert(DB::tests::createColumn<UInt8>(
std::vector<UInt64>(num_rows, is_deleted),
- TAG_COLUMN_NAME,
- TAG_COLUMN_ID));
+ MutSup::delmark_column_name,
+ MutSup::delmark_col_id));
}
if (with_nullable_uint64)
{
@@ -369,8 +371,8 @@ class DMTestEnv
/*reversed*/ false,
/*tso*/ 2,
pk_name,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnIntType(),
/* is_common_handle */ false,
/* rowkey_column_size */ 1,
/*with_internal_columns*/ true,
@@ -401,9 +403,9 @@ class DMTestEnv
end,
reversed,
tso,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnIntType(),
false,
1,
with_internal_columns);
@@ -413,9 +415,9 @@ class DMTestEnv
end,
reversed,
tso,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
1,
with_internal_columns);
@@ -427,7 +429,7 @@ class DMTestEnv
tso,
PK_NAME_PK_IS_HANDLE,
PK_ID_PK_IS_HANDLE,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::getExtraHandleColumnIntType(),
false,
1,
with_internal_columns);
@@ -459,17 +461,18 @@ class DMTestEnv
Block block;
const size_t num_rows = (ts_end - ts_beg);
// int64 pk_col
- block.insert(DB::tests::createColumn<Int64>(std::vector<Int64>(num_rows, pk), pk_name, EXTRA_HANDLE_COLUMN_ID));
+ block.insert(
+ DB::tests::createColumn<Int64>(std::vector<Int64>(num_rows, pk), pk_name, MutSup::extra_handle_id));
// version_col
block.insert(DB::tests::createColumn<UInt64>(
createNumbers<UInt64>(ts_beg, ts_end, reversed),
- VERSION_COLUMN_NAME,
- VERSION_COLUMN_ID));
+ MutSup::version_column_name,
+ MutSup::version_col_id));
// tag_col
block.insert(DB::tests::createColumn<UInt8>(
std::vector<UInt64>(num_rows, deleted ? 1 : 0),
- TAG_COLUMN_NAME,
- TAG_COLUMN_ID));
+ MutSup::delmark_column_name,
+ MutSup::delmark_col_id));
return block;
}
@@ -490,22 +493,24 @@ class DMTestEnv
if (is_common_handle)
{
Strings values{genMockCommonHandle(pk, rowkey_column_size)};
- block.insert(DB::tests::createColumn<String>(std::move(values), pk_name, EXTRA_HANDLE_COLUMN_ID));
+ block.insert(DB::tests::createColumn<String>(std::move(values), pk_name, MutSup::extra_handle_id));
}
else
{
// int64 pk_col
block.insert(
- DB::tests::createColumn<Int64>(std::vector<Int64>(num_rows, pk), pk_name, EXTRA_HANDLE_COLUMN_ID));
+ DB::tests::createColumn<Int64>(std::vector<Int64>(num_rows, pk), pk_name, MutSup::extra_handle_id));
}
// version_col
block.insert(DB::tests::createColumn<UInt64>(
std::vector<UInt64>(num_rows, tso),
- VERSION_COLUMN_NAME,
- VERSION_COLUMN_ID));
+ MutSup::version_column_name,
+ MutSup::version_col_id));
// tag_col
- block.insert(
- DB::tests::createColumn<UInt8>(std::vector<UInt64>(num_rows, mark), TAG_COLUMN_NAME, TAG_COLUMN_ID));
+ block.insert(DB::tests::createColumn<UInt8>(
+ std::vector<UInt64>(num_rows, mark),
+ MutSup::delmark_column_name,
+ MutSup::delmark_col_id));
// string column
block.insert(DB::tests::createColumn<String>(
Strings{value},
@@ -542,15 +547,18 @@ class DMTestEnv
// int64 pk_col
block.insert(DB::tests::createColumn<Int64>(
createNumbers<Int64>(start_pk, start_pk + rows),
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID));
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id));
// version_col
block.insert(DB::tests::createColumn<UInt64>(
createNumbers<UInt64>(start_ts, start_ts + rows),
- VERSION_COLUMN_NAME,
- VERSION_COLUMN_ID));
+ MutSup::version_column_name,
+ MutSup::version_col_id));
// tag_col
- block.insert(DB::tests::createColumn<UInt8>(std::vector<UInt64>(rows, 0), TAG_COLUMN_NAME, TAG_COLUMN_ID));
+ block.insert(DB::tests::createColumn<UInt8>(
+ std::vector<UInt64>(rows, 0),
+ MutSup::delmark_column_name,
+ MutSup::delmark_col_id));
return block;
}
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp
index 2701ed4f11f..795315d797a 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp
@@ -48,9 +48,9 @@ class DebugBlockInputStream : public BlocksListBlockInputStream
BlockInputStreamPtr genColumnProjInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle)
{
ColumnDefine handle_define(
- TiDBPkColumnID,
+ MutSup::extra_handle_id,
DMTestEnv::pk_name,
- is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE);
+ is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType());
return std::make_shared<DMColumnProjectionBlockInputStream>(
std::make_shared<DebugBlockInputStream>(blocks, is_common_handle),
@@ -63,9 +63,9 @@ BlockInputStreamPtr genDeleteFilterInputStream(
bool is_common_handle)
{
ColumnDefine handle_define(
- TiDBPkColumnID,
+ MutSup::extra_handle_id,
DMTestEnv::pk_name,
- is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE);
+ is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType());
return std::make_shared<DMDeleteFilterBlockInputStream>(
std::make_shared<DebugBlockInputStream>(blocks, is_common_handle),
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp
index 2c563b5b0f8..04fd8e69645 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp
@@ -61,13 +61,13 @@ TEST(PKSquashTest, WithExtraSort)
// Sorted by pk, tso asc
SortDescription sort //
= SortDescription{//
- SortColumnDescription{EXTRA_HANDLE_COLUMN_NAME, 1, 0},
- SortColumnDescription{VERSION_COLUMN_NAME, 1, 0}};
+ SortColumnDescription{MutSup::extra_handle_column_name, 1, 0},
+ SortColumnDescription{MutSup::version_column_name, 1, 0}};
{
auto in = std::make_shared<PKSquashingBlockInputStream</*need_extra_sort*/ true>>(
std::make_shared<BlocksListBlockInputStream>(blocks.begin(), blocks.end()),
- TiDBPkColumnID,
+ MutSup::extra_handle_id,
false);
size_t num_blocks_read = 0;
size_t num_rows_read = 0;
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp
index 6873685a4a6..9d63a7aef04 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_column_file.cpp
@@ -129,14 +129,14 @@ try
while (Block in = column_file_big_reader->readNextBlock())
{
ASSERT_EQ(in.columns(), column_defines->size());
- ASSERT_TRUE(in.has(EXTRA_HANDLE_COLUMN_NAME));
- ASSERT_TRUE(in.has(VERSION_COLUMN_NAME));
- auto & pk_column = in.getByName(EXTRA_HANDLE_COLUMN_NAME).column;
+ ASSERT_TRUE(in.has(MutSup::extra_handle_column_name));
+ ASSERT_TRUE(in.has(MutSup::version_column_name));
+ auto & pk_column = in.getByName(MutSup::extra_handle_column_name).column;
for (size_t i = 0; i < pk_column->size(); i++)
{
ASSERT_EQ(pk_column->getInt(i), num_rows_read + i);
}
- auto & version_column = in.getByName(VERSION_COLUMN_NAME).column;
+ auto & version_column = in.getByName(MutSup::version_column_name).column;
for (size_t i = 0; i < version_column->size(); i++)
{
ASSERT_EQ(version_column->getInt(i), tso_value);
@@ -156,14 +156,14 @@ try
while (Block in = column_file_big_reader2->readNextBlock())
{
ASSERT_EQ(in.columns(), column_defines_pk_and_del->size());
- ASSERT_TRUE(in.has(EXTRA_HANDLE_COLUMN_NAME));
- ASSERT_TRUE(in.has(TAG_COLUMN_NAME));
- auto & pk_column = in.getByName(EXTRA_HANDLE_COLUMN_NAME).column;
+ ASSERT_TRUE(in.has(MutSup::extra_handle_column_name));
+ ASSERT_TRUE(in.has(MutSup::delmark_column_name));
+ auto & pk_column = in.getByName(MutSup::extra_handle_column_name).column;
for (size_t i = 0; i < pk_column->size(); i++)
{
ASSERT_EQ(pk_column->getInt(i), num_rows_read + i);
}
- auto & del_column = in.getByName(TAG_COLUMN_NAME).column;
+ auto & del_column = in.getByName(MutSup::delmark_column_name).column;
for (size_t i = 0; i < del_column->size(); i++)
{
ASSERT_EQ(del_column->getInt(i), 0);
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
index 6cc96bdc63d..56c7cccb7fe 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
@@ -90,9 +90,9 @@ try
{
// check handle column of store
const auto & h = store->getHandle();
- ASSERT_EQ(h.name, EXTRA_HANDLE_COLUMN_NAME);
- ASSERT_EQ(h.id, EXTRA_HANDLE_COLUMN_ID);
- ASSERT_TRUE(h.type->equals(*EXTRA_HANDLE_COLUMN_INT_TYPE));
+ ASSERT_EQ(h.name, MutSup::extra_handle_column_name);
+ ASSERT_EQ(h.id, MutSup::extra_handle_id);
+ ASSERT_TRUE(h.type->equals(*MutSup::getExtraHandleColumnIntType()));
}
{
// check column structure of store
@@ -358,7 +358,7 @@ try
store->getRowKeyColumnSize());
block1 = DeltaMergeStore::addExtraColumnIfNeed(*db_context, store->getHandle(), std::move(block1));
ASSERT_EQ(block1.rows(), nrows);
- ASSERT_TRUE(block1.has(EXTRA_HANDLE_COLUMN_NAME));
+ ASSERT_TRUE(block1.has(MutSup::extra_handle_column_name));
ASSERT_NO_THROW({ block1.checkNumberOfRows(); });
// Make a block that is overlapped with `block1` and it should be squashed by `PKSquashingBlockInputStream`
@@ -375,14 +375,14 @@ try
store->getRowKeyColumnSize());
block2 = DeltaMergeStore::addExtraColumnIfNeed(*db_context, store->getHandle(), std::move(block2));
ASSERT_EQ(block2.rows(), nrows_2);
- ASSERT_TRUE(block2.has(EXTRA_HANDLE_COLUMN_NAME));
+ ASSERT_TRUE(block2.has(MutSup::extra_handle_column_name));
ASSERT_NO_THROW({ block2.checkNumberOfRows(); });
BlockInputStreamPtr stream = std::make_shared<BlocksListBlockInputStream>(BlocksList{block1, block2});
stream = std::make_shared<PKSquashingBlockInputStream<false>>(
stream,
- EXTRA_HANDLE_COLUMN_ID,
+ MutSup::extra_handle_id,
store->isCommonHandle());
ASSERT_INPUTSTREAM_NROWS(stream, nrows + nrows_2);
@@ -562,7 +562,7 @@ try
/* is_fast_scan= */ false,
/* expected_block_size= */ 1024,
/* read_segments */ {},
- /* extra_table_id_index */ InvalidColumnID,
+ /* extra_table_id_index */ MutSup::invalid_col_id,
/* scan_context */ scan_context)[0];
in->readPrefix();
while (in->read()) {};
@@ -590,7 +590,7 @@ try
/* is_fast_scan= */ false,
/* expected_block_size= */ 1024,
/* read_segments */ {},
- /* extra_table_id_index */ InvalidColumnID,
+ /* extra_table_id_index */ MutSup::invalid_col_id,
/* scan_context */ scan_context)[0];
in->readPrefix();
@@ -1307,7 +1307,7 @@ try
BlockInputStreamPtr in = ins[0];
ASSERT_UNORDERED_INPUTSTREAM_COLS_UR(
in,
- Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}),
+ Strings({DMTestEnv::pk_name, MutSup::version_column_name}),
createColumns({
createColumn<Int64>(createNumbers<Int64>(0, 32)),
createColumn<UInt64>(std::vector<UInt64>(32, tso1)),
@@ -1336,7 +1336,7 @@ try
BlockInputStreamPtr in = ins[0];
ASSERT_UNORDERED_INPUTSTREAM_COLS_UR(
in,
- Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}),
+ Strings({DMTestEnv::pk_name, MutSup::version_column_name}),
createColumns({
createColumn<Int64>(createNumbers<Int64>(0, 32)),
createColumn<UInt64>(std::vector<UInt64>(32, tso1)),
@@ -1472,7 +1472,7 @@ try
BlockInputStreamPtr in = ins[0];
ASSERT_UNORDERED_INPUTSTREAM_COLS_UR(
in,
- Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}),
+ Strings({DMTestEnv::pk_name, MutSup::version_column_name}),
createColumns({
createColumn<Int64>(createNumbers<Int64>(0, 32)),
createColumn<UInt64>(std::vector<UInt64>(32, tso1)),
@@ -1501,7 +1501,7 @@ try
BlockInputStreamPtr in = ins[0];
ASSERT_UNORDERED_INPUTSTREAM_COLS_UR(
in,
- Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}),
+ Strings({DMTestEnv::pk_name, MutSup::version_column_name}),
createColumns({
createColumn<Int64>(createNumbers<Int64>(0, 32)),
createColumn<UInt64>(std::vector<UInt64>(32, tso1)),
@@ -1578,7 +1578,8 @@ try
{
break;
}
- const auto & handle = *toColumnVectorDataPtr<Int64>(block.getByName(EXTRA_HANDLE_COLUMN_NAME).column);
+ const auto & handle
+ = *toColumnVectorDataPtr<Int64>(block.getByName(MutSup::extra_handle_column_name).column);
const auto & value = *toColumnVectorDataPtr<UInt64>(block.getByName(value_col_name).column);
for (size_t i = 0; i < block.rows(); i++)
{
@@ -2748,9 +2749,9 @@ try
{
// check handle column of store
const auto & h = store->getHandle();
- ASSERT_EQ(h.name, EXTRA_HANDLE_COLUMN_NAME);
- ASSERT_EQ(h.id, EXTRA_HANDLE_COLUMN_ID);
- ASSERT_TRUE(h.type->equals(*EXTRA_HANDLE_COLUMN_STRING_TYPE));
+ ASSERT_EQ(h.name, MutSup::extra_handle_column_name);
+ ASSERT_EQ(h.id, MutSup::extra_handle_id);
+ ASSERT_TRUE(h.type->equals(*MutSup::getExtraHandleColumnStringType()));
}
{
// check column structure of store
@@ -2946,9 +2947,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
// Add a column of col2:String for test
@@ -3052,9 +3053,9 @@ try
1 * num_write_rows,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
Block block2 = DMTestEnv::prepareSimpleWriteBlock(
@@ -3062,9 +3063,9 @@ try
2 * num_write_rows,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
Block block3 = DMTestEnv::prepareSimpleWriteBlock(
@@ -3072,9 +3073,9 @@ try
3 * num_write_rows,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
store->write(*db_context, db_context->getSettingsRef(), block1);
@@ -3129,9 +3130,9 @@ try
1 * num_write_rows,
false,
tso1,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
Block block2 = DMTestEnv::prepareSimpleWriteBlock(
@@ -3139,9 +3140,9 @@ try
2 * num_write_rows,
false,
tso1,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
Block block3 = DMTestEnv::prepareSimpleWriteBlock(
@@ -3149,9 +3150,9 @@ try
num_write_rows / 2 + num_write_rows,
false,
tso2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
store->write(*db_context, db_context->getSettingsRef(), block1);
@@ -3251,9 +3252,9 @@ try
128,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
rowkey_column_size);
store->write(*db_context, db_context->getSettingsRef(), block);
@@ -3857,8 +3858,8 @@ try
false,
std::numeric_limits<UInt64>::max(), // max version to make sure it's the latest
pk_name,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnIntType(),
false,
1,
true,
@@ -3879,7 +3880,8 @@ try
ColumnDefines real_columns;
for (const auto & col : columns)
{
- if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != TAG_COLUMN_NAME && col.name != VERSION_COLUMN_NAME)
+ if (col.name != MutSup::extra_handle_column_name && col.name != MutSup::delmark_column_name
+ && col.name != MutSup::version_column_name)
{
real_columns.emplace_back(col);
}
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp
index 11adc800029..3d613cfe8f4 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_scan.cpp
@@ -819,8 +819,8 @@ try
false,
3,
DMTestEnv::pk_name,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnIntType(),
false,
1,
true,
@@ -1303,7 +1303,7 @@ try
ColumnDefines real_columns;
for (const auto & col : columns)
{
- if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != TAG_COLUMN_NAME)
+ if (col.name != MutSup::extra_handle_column_name && col.name != MutSup::delmark_column_name)
{
real_columns.emplace_back(col);
}
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp
index 83fe42b5038..0072eb0bd89 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_vector_index.cpp
@@ -85,9 +85,9 @@ class DeltaMergeStoreVectorTest
end,
false,
/*tso= */ 3,
- /*pk_name_=*/EXTRA_HANDLE_COLUMN_NAME,
- /*pk_col_id=*/EXTRA_HANDLE_COLUMN_ID,
- /*pk_type=*/EXTRA_HANDLE_COLUMN_INT_TYPE,
+ /*pk_name_=*/MutSup::extra_handle_column_name,
+ /*pk_col_id=*/MutSup::extra_handle_id,
+ /*pk_type=*/MutSup::getExtraHandleColumnIntType(),
/*is_common_handle=*/false,
/*rowkey_column_size=*/1,
/*with_internal_columns=*/true,
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp
index 3f2919e5ef3..e6ed053c1f7 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp
@@ -2025,9 +2025,9 @@ try
num_rows_write / 2,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
Block block2 = DMTestEnv::prepareSimpleWriteBlock(
@@ -2035,9 +2035,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
auto stream = std::make_shared<DMFileBlockOutputStream>(dbContext(), dm_file, *cols);
@@ -2104,9 +2104,9 @@ try
pk_end,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
stream->write(block, block_property);
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp
index 92db6f9daeb..250a558684d 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_meta_version.cpp
@@ -148,7 +148,7 @@ try
auto dm_file = prepareDMFile(/* file_id= */ 1);
ASSERT_EQ(0, dm_file->metaVersion());
ASSERT_EQ(4, dm_file->meta->getColumnStats().size());
- ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str());
+ ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str());
// Write new metadata
auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{
@@ -158,7 +158,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test";
+ dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test";
ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({}));
iw->finalize();
@@ -173,7 +173,7 @@ try
ASSERT_EQ(0, dm_file->metaVersion());
ASSERT_EQ(4, dm_file->meta->getColumnStats().size());
- ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str());
+ ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str());
// Read out meta version = 1
dm_file = DMFile::restore(
@@ -186,7 +186,7 @@ try
ASSERT_EQ(1, dm_file->metaVersion());
ASSERT_EQ(4, dm_file->meta->getColumnStats().size());
- ASSERT_STREQ("test", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str());
+ ASSERT_STREQ("test", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str());
}
CATCH
@@ -202,7 +202,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file_for_write->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test";
+ dm_file_for_write->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test";
ASSERT_EQ(1, dm_file_for_write->meta->bumpMetaVersion({}));
iw->finalize();
@@ -215,7 +215,7 @@ try
/* meta_version= */ 1);
ASSERT_STREQ(
"test",
- dm_file_for_read_v1->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ dm_file_for_read_v1->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
// Write a new meta with a new version = 2
iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{
@@ -225,14 +225,14 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file_for_write->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test2";
+ dm_file_for_write->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test2";
ASSERT_EQ(2, dm_file_for_write->meta->bumpMetaVersion({}));
iw->finalize();
// Current DMFile instance does not affect
ASSERT_STREQ(
"test",
- dm_file_for_read_v1->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ dm_file_for_read_v1->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
// Read out meta version = 2
auto dm_file_for_read_v2 = DMFile::restore(
@@ -244,7 +244,7 @@ try
/* meta_version= */ 2);
ASSERT_STREQ(
"test2",
- dm_file_for_read_v2->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ dm_file_for_read_v2->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
}
CATCH
@@ -261,7 +261,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test";
+ dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test";
ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({}));
iw->finalize();
@@ -281,7 +281,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file_2->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test_overwrite";
+ dm_file_2->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test_overwrite";
ASSERT_EQ(1, dm_file_2->meta->bumpMetaVersion({}));
ASSERT_NO_THROW({
iw->finalize();
@@ -297,7 +297,7 @@ try
/* meta_version= */ 1);
ASSERT_STREQ(
"test_overwrite",
- dm_file_for_read->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ dm_file_for_read->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
}
CATCH
@@ -307,7 +307,7 @@ try
auto dm_file = prepareDMFile(/* file_id= */ 1);
ASSERT_EQ(0, dm_file->metaVersion());
ASSERT_EQ(4, dm_file->meta->getColumnStats().size());
- ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str());
+ ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str());
// Write new metadata
auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{
@@ -317,7 +317,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test";
+ dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test";
dm_file->meta->bumpMetaVersion({});
iw->finalize();
@@ -432,7 +432,7 @@ try
ASSERT_EQ(0, dm_file->metaVersion());
ASSERT_EQ(4, dm_file->meta->getColumnStats().size());
- ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str());
+ ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str());
// Write new metadata
auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{
@@ -442,7 +442,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test";
+ dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test";
ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({}));
iw->finalize();
@@ -457,12 +457,12 @@ try
/* page_id= */ 0);
auto cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 0);
ASSERT_EQ(0, cn_dmf->metaVersion());
- ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
// Read out meta version = 1
cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 1);
ASSERT_EQ(1, cn_dmf->metaVersion());
- ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
}
CATCH
@@ -480,7 +480,7 @@ try
ASSERT_EQ(0, dm_file->metaVersion());
ASSERT_EQ(4, dm_file->meta->getColumnStats().size());
- ASSERT_STREQ("", dm_file->getColumnStat(::DB::TiDBPkColumnID).additional_data_for_test.c_str());
+ ASSERT_STREQ("", dm_file->getColumnStat(MutSup::extra_handle_id).additional_data_for_test.c_str());
// Write new metadata
auto iw = DMFileV3IncrementWriter::create(DMFileV3IncrementWriter::Options{
@@ -490,7 +490,7 @@ try
.path_pool = path_pool,
.disagg_ctx = db_context->getSharedContextDisagg(),
});
- dm_file->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test = "test";
+ dm_file->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test = "test";
ASSERT_EQ(1, dm_file->meta->bumpMetaVersion({}));
iw->finalize();
@@ -510,7 +510,7 @@ try
/* page_id= */ 0);
auto cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 0);
ASSERT_EQ(0, cn_dmf->metaVersion());
- ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ ASSERT_STREQ("", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
{
auto * file_cache = FileCache::instance();
@@ -520,7 +520,7 @@ try
// Read out meta version = 1
cn_dmf = token->restore(DMFileMeta::ReadMode::all(), 1);
ASSERT_EQ(1, cn_dmf->metaVersion());
- ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ ASSERT_STREQ("test", cn_dmf->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
SCOPE_EXIT({ FileCache::shutdown(); });
}
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp
index 3e0ede20007..dcafc0f22a7 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp
@@ -1129,8 +1129,8 @@ try
false,
3,
"_tidb_rowid",
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnIntType(),
false,
1,
true,
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp
index ea3659931a7..a7c22265717 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp
@@ -136,9 +136,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
// write to segment
@@ -190,9 +190,9 @@ try
num_rows_write + num_rows_write_2,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -235,9 +235,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
// write to segment
@@ -292,9 +292,9 @@ try
num_rows_write + num_rows_write_2,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -340,9 +340,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -410,9 +410,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -488,9 +488,9 @@ try
num_rows_write / 2,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -507,9 +507,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -580,9 +580,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -790,9 +790,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -923,9 +923,9 @@ try
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
@@ -981,9 +981,9 @@ try
num_batches_written * num_rows_per_write + num_rows_per_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
is_common_handle,
rowkey_column_size);
segment->write(dmContext(), std::move(block));
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp
index 919ad3c830e..d25d6f80957 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_simple_pk_test_basic.cpp
@@ -192,8 +192,8 @@ Block SimplePKTestBasic::fillBlock(const FillBlockOptions & options)
false,
version,
DMTestEnv::pk_name,
- EXTRA_HANDLE_COLUMN_ID,
- is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType(),
is_common_handle,
1,
true,
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp
index fcfa1bd3b96..d0ef2c15bf0 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp
@@ -698,7 +698,7 @@ try
runtime_filter_ids,
0,
ctx->getTimezoneInfo());
- Names read_columns = {"col1", EXTRA_TABLE_ID_COLUMN_NAME, "col2"};
+ Names read_columns = {"col1", MutSup::extra_table_id_column_name, "col2"};
BlockInputStreams ins = storage->read(read_columns, query_info, *ctx, stage2, 8192, 1);
ASSERT_EQ(ins.size(), 1);
BlockInputStreamPtr in = ins[0];
@@ -706,7 +706,7 @@ try
in,
Block({
createColumn<Int64>(createNumbers<Int64>(0, num_rows_write), "col1"),
- createConstColumn<Nullable<Int64>>(num_rows_write, table_id, EXTRA_TABLE_ID_COLUMN_NAME),
+ createConstColumn<Nullable<Int64>>(num_rows_write, table_id, MutSup::extra_table_id_column_name),
createColumn<String>(Strings(num_rows_write, "a"), "col2"),
}));
@@ -818,7 +818,7 @@ try
runtime_filter_ids,
0,
ctx->getTimezoneInfo());
- Names read_columns = {"col1", EXTRA_TABLE_ID_COLUMN_NAME, "col2"};
+ Names read_columns = {"col1", MutSup::extra_table_id_column_name, "col2"};
BlockInputStreams ins = storage->read(read_columns, query_info, *ctx, stage2, 8192, 1);
return getInputStreamNRows(ins[0]);
};
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp
index d1a9a17748c..549b3a18b6d 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp
@@ -62,9 +62,9 @@ TEST(RowKeyFilterTest, FilterSortedBlockCommonHandle)
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
1);
auto filtered_block = RowKeyFilter::filterSorted(ranges, std::move(block), 0);
@@ -84,9 +84,9 @@ TEST(RowKeyFilterTest, FilterUnsortedBlockCommonHandle)
num_rows_write,
false,
2,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_STRING_TYPE,
+ MutSup::extra_handle_column_name,
+ MutSup::extra_handle_id,
+ MutSup::getExtraHandleColumnStringType(),
true,
1);
auto filtered_block = RowKeyFilter::filterUnsorted(ranges, std::move(block), 0);
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp
index 0391ff1100b..2d783ce3b1d 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_vector_index.cpp
@@ -485,7 +485,7 @@ try
// does not have index at all.
{
auto ann_query_info = std::make_shared<tipb::ANNQueryInfo>();
- ann_query_info->set_column_id(EXTRA_HANDLE_COLUMN_ID);
+ ann_query_info->set_column_id(MutSup::extra_handle_id);
ann_query_info->set_distance_metric(tipb::VectorDistanceMetric::L2);
ann_query_info->set_top_k(1);
ann_query_info->set_ref_vec_f32(encodeVectorFloat32({1.0, 2.0, 3.8}));
@@ -1159,7 +1159,7 @@ class VectorIndexSegmentTestBase
{
auto options = SegmentTestBasic::SegmentTestOptions{};
if (enable_column_cache_long_term)
- options.pk_col_id = EXTRA_HANDLE_COLUMN_ID;
+ options.pk_col_id = MutSup::extra_handle_id;
SegmentTestBasic::SetUp(options);
}
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp
index 92fd9b04dc5..964482c1702 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task.cpp
@@ -129,7 +129,7 @@ class SegmentReadTaskTest : public SegmentTestBasic
{
blks.push_back(blk);
}
- auto handle_col1 = vstackBlocks(std::move(blks)).getByName(EXTRA_HANDLE_COLUMN_NAME).column;
+ auto handle_col1 = vstackBlocks(std::move(blks)).getByName(MutSup::extra_handle_column_name).column;
auto handle_col2 = getSegmentHandle(task->segment->segmentId(), {task->segment->getRowKeyRange()});
ASSERT_TRUE(sequenceEqual(
toColumnVectorDataPtr<Int64>(handle_col2)->data(),
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp
index 9c686ef750b..8a931ec7b69 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_read_task_pool.cpp
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <DataTypes/DataTypeNullable.h>
#include <Interpreters/Context.h>
#include <Storages/DeltaMerge/ReadThread/SegmentReadTaskScheduler.h>
#include <Storages/DeltaMerge/Segment.h>
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp
index 07854c0f0a0..1fd2affc570 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_replace_stable_data.cpp
@@ -100,8 +100,8 @@ class SegmentReplaceStableData
.disagg_ctx = dm_context->global_context.getSharedContextDisagg(),
});
auto & column_stats = new_dm_file->meta->getColumnStats();
- RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end());
- column_stats[::DB::TiDBPkColumnID].additional_data_for_test = pk_additiona_data;
+ RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end());
+ column_stats[MutSup::extra_handle_id].additional_data_for_test = pk_additiona_data;
new_dm_file->meta->bumpMetaVersion({});
iw->finalize();
@@ -156,9 +156,9 @@ class SegmentReplaceStableData
auto file = files[0];
auto column_stats = file->meta->getColumnStats();
- RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end());
+ RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end());
- auto meta_value = column_stats[::DB::TiDBPkColumnID].additional_data_for_test;
+ auto meta_value = column_stats[MutSup::extra_handle_id].additional_data_for_test;
// Read again using a fresh DMFile restore, to ensure that this value is
// indeed persisted.
@@ -172,8 +172,8 @@ class SegmentReplaceStableData
RUNTIME_CHECK(file2 != nullptr);
column_stats = file2->meta->getColumnStats();
- RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end());
- RUNTIME_CHECK(column_stats[::DB::TiDBPkColumnID].additional_data_for_test == meta_value);
+ RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end());
+ RUNTIME_CHECK(column_stats[MutSup::extra_handle_id].additional_data_for_test == meta_value);
return meta_value;
}
@@ -570,8 +570,8 @@ try
.disagg_ctx = wn_dm_context->global_context.getSharedContextDisagg(),
});
auto & column_stats = new_dm_file->meta->getColumnStats();
- RUNTIME_CHECK(column_stats.find(::DB::TiDBPkColumnID) != column_stats.end());
- column_stats[::DB::TiDBPkColumnID].additional_data_for_test = "tiflash_foo";
+ RUNTIME_CHECK(column_stats.find(MutSup::extra_handle_id) != column_stats.end());
+ column_stats[MutSup::extra_handle_id].additional_data_for_test = "tiflash_foo";
new_dm_file->meta->bumpMetaVersion({});
iw->finalize();
@@ -588,7 +588,7 @@ try
auto cn_files = snapshot->stable->getDMFiles();
ASSERT_EQ(1, cn_files.size());
ASSERT_EQ(0, cn_files[0]->metaVersion());
- ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
}
// Read meta v1 in CN
@@ -600,7 +600,7 @@ try
ASSERT_EQ(1, cn_files[0]->metaVersion());
ASSERT_STREQ(
"tiflash_foo",
- cn_files[0]->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ cn_files[0]->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
}
// Read meta v0 again in CN
@@ -610,7 +610,7 @@ try
auto cn_files = snapshot->stable->getDMFiles();
ASSERT_EQ(1, cn_files.size());
ASSERT_EQ(0, cn_files[0]->metaVersion());
- ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[::DB::TiDBPkColumnID].additional_data_for_test.c_str());
+ ASSERT_STREQ("", cn_files[0]->meta->getColumnStats()[MutSup::extra_handle_id].additional_data_for_test.c_str());
}
}
CATCH
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp
index f2c10e2a677..d8fc3ae12b1 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp
@@ -363,8 +363,8 @@ Block SegmentTestBasic::prepareWriteBlockImpl(Int64 start_key, Int64 end_key, bo
false,
version,
DMTestEnv::pk_name,
- EXTRA_HANDLE_COLUMN_ID,
- options.is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ options.is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType(),
options.is_common_handle,
1,
true,
@@ -381,8 +381,8 @@ Block sortvstackBlocks(std::vector<Block> && blocks)
auto accumulated_block = vstackBlocks(std::move(blocks));
SortDescription sort;
- sort.emplace_back(EXTRA_HANDLE_COLUMN_NAME, 1, 0);
- sort.emplace_back(VERSION_COLUMN_NAME, 1, 0);
+ sort.emplace_back(MutSup::extra_handle_column_name, 1, 0);
+ sort.emplace_back(MutSup::version_column_name, 1, 0);
stableSortBlock(accumulated_block, sort);
return accumulated_block;
@@ -1048,7 +1048,7 @@ ColumnPtr SegmentTestBasic::getSegmentRowId(PageIdU64 segment_id, const RowKeyRa
else
{
auto block = mergeSegmentRowIds(std::move(blks));
- RUNTIME_CHECK(!block.has(EXTRA_HANDLE_COLUMN_NAME));
+ RUNTIME_CHECK(!block.has(MutSup::extra_handle_column_name));
RUNTIME_CHECK(block.segmentRowIdCol() != nullptr);
return block.segmentRowIdCol();
}
@@ -1065,9 +1065,9 @@ ColumnPtr SegmentTestBasic::getSegmentHandle(PageIdU64 segment_id, const RowKeyR
else
{
auto block = vstackBlocks(std::move(blks));
- RUNTIME_CHECK(block.has(EXTRA_HANDLE_COLUMN_NAME));
+ RUNTIME_CHECK(block.has(MutSup::extra_handle_column_name));
RUNTIME_CHECK(block.segmentRowIdCol() == nullptr);
- return block.getByName(EXTRA_HANDLE_COLUMN_NAME).column;
+ return block.getByName(MutSup::extra_handle_column_name).column;
}
}
@@ -1105,47 +1105,51 @@ try
version = 0;
auto block = prepareWriteBlockInSegmentRange(*s1_id, 10);
ASSERT_COLUMN_EQ(
- block.getByName(EXTRA_HANDLE_COLUMN_NAME),
+ block.getByName(MutSup::extra_handle_column_name),
createColumn<Int64>({10, 11, 12, 13, 14, 15, 16, 17, 18, 19}));
- ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn<UInt64>({1, 1, 1, 1, 1, 1, 1, 1, 1, 1}));
+ ASSERT_COLUMN_EQ(
+ block.getByName(MutSup::version_column_name),
+ createColumn<UInt64>({1, 1, 1, 1, 1, 1, 1, 1, 1, 1}));
}
{
// write_rows > segment_rows, start_key not specified
version = 0;
auto block = prepareWriteBlockInSegmentRange(*s1_id, 13);
ASSERT_COLUMN_EQ(
- block.getByName(EXTRA_HANDLE_COLUMN_NAME),
+ block.getByName(MutSup::extra_handle_column_name),
createColumn<Int64>({10, 10, 11, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19}));
ASSERT_COLUMN_EQ(
- block.getByName(VERSION_COLUMN_NAME),
+ block.getByName(MutSup::version_column_name),
createColumn<UInt64>({1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1}));
}
{
// start_key specified, end_key - start_key < write_rows
version = 0;
auto block = prepareWriteBlockInSegmentRange(*s1_id, 2, /* at */ 16);
- ASSERT_COLUMN_EQ(block.getByName(EXTRA_HANDLE_COLUMN_NAME), createColumn<Int64>({16, 17}));
- ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn<UInt64>({1, 1}));
+ ASSERT_COLUMN_EQ(block.getByName(MutSup::extra_handle_column_name), createColumn<Int64>({16, 17}));
+ ASSERT_COLUMN_EQ(block.getByName(MutSup::version_column_name), createColumn<UInt64>({1, 1}));
}
{
version = 0;
auto block = prepareWriteBlockInSegmentRange(*s1_id, 4, /* at */ 16);
- ASSERT_COLUMN_EQ(block.getByName(EXTRA_HANDLE_COLUMN_NAME), createColumn<Int64>({16, 17, 18, 19}));
- ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn<UInt64>({1, 1, 1, 1}));
+ ASSERT_COLUMN_EQ(block.getByName(MutSup::extra_handle_column_name), createColumn<Int64>({16, 17, 18, 19}));
+ ASSERT_COLUMN_EQ(block.getByName(MutSup::version_column_name), createColumn<UInt64>({1, 1, 1, 1}));
}
{
version = 0;
auto block = prepareWriteBlockInSegmentRange(*s1_id, 5, /* at */ 16);
- ASSERT_COLUMN_EQ(block.getByName(EXTRA_HANDLE_COLUMN_NAME), createColumn<Int64>({16, 16, 17, 18, 19}));
- ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn<UInt64>({1, 2, 1, 1, 1}));
+ ASSERT_COLUMN_EQ(block.getByName(MutSup::extra_handle_column_name), createColumn<Int64>({16, 16, 17, 18, 19}));
+ ASSERT_COLUMN_EQ(block.getByName(MutSup::version_column_name), createColumn<UInt64>({1, 2, 1, 1, 1}));
}
{
version = 0;
auto block = prepareWriteBlockInSegmentRange(*s1_id, 10, /* at */ 16);
ASSERT_COLUMN_EQ(
- block.getByName(EXTRA_HANDLE_COLUMN_NAME),
+ block.getByName(MutSup::extra_handle_column_name),
createColumn<Int64>({16, 16, 16, 17, 17, 17, 18, 18, 19, 19}));
- ASSERT_COLUMN_EQ(block.getByName(VERSION_COLUMN_NAME), createColumn<UInt64>({1, 2, 3, 1, 2, 3, 1, 2, 1, 2}));
+ ASSERT_COLUMN_EQ(
+ block.getByName(MutSup::version_column_name),
+ createColumn<UInt64>({1, 2, 3, 1, 2, 3, 1, 2, 1, 2}));
}
{
// write rows < segment rows, start key not specified, should choose a random start.
@@ -1158,7 +1162,7 @@ try
for (size_t i = 0; i < 100; i++)
{
auto block = prepareWriteBlockInSegmentRange(*s1_id, 3);
- if (block.getByName(EXTRA_HANDLE_COLUMN_NAME).column->getInt(0) == 12)
+ if (block.getByName(MutSup::extra_handle_column_name).column->getInt(0) == 12)
start_from_12++;
}
ASSERT_TRUE(start_from_12 > 0); // We should hit at least 1 times in 100 iters.
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp
index 016683bc2df..af787a1c5f8 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_sst_files_stream.cpp
@@ -120,7 +120,7 @@ try
auto blocks = prepareBlocks(-10, 5, 1000);
ASSERT_EQ(blocks.size(), 1);
auto block = blocks[0];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4}));
}
{
@@ -128,27 +128,27 @@ try
ASSERT_EQ(blocks.size(), 5);
{
auto block = blocks[0];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({1, 2, 3}));
}
{
auto block = blocks[1];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({4, 5, 6}));
}
{
auto block = blocks[2];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({7, 8, 9}));
}
{
auto block = blocks[3];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({10, 11, 12}));
}
{
auto block = blocks[4];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({13}));
}
}
@@ -157,17 +157,17 @@ try
ASSERT_EQ(blocks.size(), 3);
{
auto block = blocks[0];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({1}));
}
{
auto block = blocks[1];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({2}));
}
{
auto block = blocks[2];
- auto col = block.getByName(MutableSupport::tidb_pk_column_name);
+ auto col = block.getByName(MutSup::extra_handle_column_name);
ASSERT_COLUMN_EQ(col, createColumn<Int64>({3}));
}
}
diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp
index aee463ff075..6c41bd1e2dd 100644
--- a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp
+++ b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp
@@ -73,9 +73,9 @@ BlockInputStreamPtr getVersionFilterInputStream(
bool is_common_handle)
{
ColumnDefine handle_define(
- TiDBPkColumnID,
+ MutSup::extra_handle_id,
DMTestEnv::pk_name,
- is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE);
+ is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType());
return std::make_shared<DMVersionFilterBlockInputStream<MODE>>(
std::make_shared<DebugBlockInputStream>(blocks, is_common_handle),
columns,
@@ -92,9 +92,9 @@ BlockInputStreamPtr getVersionFilterInputStreamWithRowKeyFilterStream(
const RowKeyRanges & read_ranges)
{
ColumnDefine handle_define(
- TiDBPkColumnID,
+ MutSup::extra_handle_id,
DMTestEnv::pk_name,
- is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE);
+ is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType());
BlockInputStreamPtr rowkey_filter_stream = std::make_shared<DMRowKeyFilterBlockInputStream<is_block_sorted>>(
std::make_shared<DebugBlockInputStream>(blocks, is_common_handle),
diff --git a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp
index 0a14c9df287..6b1aeebde5f 100644
--- a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp
+++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp
@@ -118,9 +118,9 @@ uint64_t DTWorkload::updateBlock(Block & block, uint64_t key)
auto ts = ts_gen->get();
{
auto & cd = (*table_info->columns)[1];
- if (cd.id != VERSION_COLUMN_ID)
+ if (cd.id != MutSup::version_col_id)
{
- LOG_ERROR(log, "Column id not match: {} but {} is required", cd.id, VERSION_COLUMN_ID);
+ LOG_ERROR(log, "Column id not match: {} but {} is required", cd.id, MutSup::version_col_id);
throw std::invalid_argument("Column id not match");
}
diff --git a/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp
index 6a6ee198044..5752f0b33d7 100644
--- a/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp
+++ b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp
@@ -13,6 +13,7 @@
// limitations under the License.
#include <Common/RandomData.h>
+#include <DataTypes/DataTypeDecimal.h>
#include <DataTypes/DataTypeEnum.h>
#include <Storages/DeltaMerge/workload/DataGenerator.h>
#include <Storages/DeltaMerge/workload/KeyGenerator.h>
@@ -73,12 +74,12 @@ class RandomDataGenerator : public DataGenerator
uint64_t ts = ts_gen.get();
{
auto & col_def = (*table_info.columns)[1];
- if (col_def.id != VERSION_COLUMN_ID)
+ if (col_def.id != MutSup::version_col_id)
{
throw std::invalid_argument(fmt::format(
- "(*table_info.columns)[1].id is {} not VERSION_COLUMN_ID {}.",
+ "(*table_info.columns)[1].id is {} not MutSup::version_col_id {}.",
col_def.id,
- VERSION_COLUMN_ID));
+ MutSup::version_col_id));
}
ColumnWithTypeAndName col({}, col_def.type, col_def.name, col_def.id);
IColumn::MutablePtr mut_col = col.type->createColumn();
@@ -91,10 +92,12 @@ class RandomDataGenerator : public DataGenerator
// Generate 'delete mark'
{
auto & col_def = (*table_info.columns)[2];
- if (col_def.id != TAG_COLUMN_ID)
+ if (col_def.id != MutSup::delmark_col_id)
{
- throw std::invalid_argument(
- fmt::format("(*table_info.columns)[2].id is {} not TAG_COLUMN_ID {}.", col_def.id, TAG_COLUMN_ID));
+ throw std::invalid_argument(fmt::format(
+ "(*table_info.columns)[2].id is {} not MutSup::delmark_col_id {}.",
+ col_def.id,
+ MutSup::delmark_col_id));
}
ColumnWithTypeAndName col({}, col_def.type, col_def.name, col_def.id);
IColumn::MutablePtr mut_col = col.type->createColumn();
@@ -113,7 +116,8 @@ class RandomDataGenerator : public DataGenerator
continue;
}
auto & col_def = (*table_info.columns)[i];
- if (col_def.id == table_info.handle.id || col_def.id == VERSION_COLUMN_ID || col_def.id == TAG_COLUMN_ID)
+ if (col_def.id == table_info.handle.id || col_def.id == MutSup::version_col_id
+ || col_def.id == MutSup::delmark_col_id)
{
continue;
}
diff --git a/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp
index 611b69ed71b..e0c6f9b9ea6 100644
--- a/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp
+++ b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp
@@ -13,7 +13,9 @@
// limitations under the License.
#include <Common/nocopyable.h>
+#include <DataTypes/DataTypeDecimal.h>
#include <DataTypes/DataTypeEnum.h>
+#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeString.h>
#include <Storages/DeltaMerge/workload/Options.h>
#include <Storages/DeltaMerge/workload/TableGenerator.h>
@@ -104,7 +106,7 @@ class TablePkType
columns->emplace_back(getExtraHandleColumnDefine(/*is_common_handle=*/true));
break;
case PkType::PkIsHandleInt64:
- columns->emplace_back(ColumnDefine{2, PK_NAME_PK_IS_HANDLE, EXTRA_HANDLE_COLUMN_INT_TYPE});
+ columns->emplace_back(ColumnDefine{2, PK_NAME_PK_IS_HANDLE, MutSup::getExtraHandleColumnIntType()});
break;
case PkType::PkIsHandleInt32:
columns->emplace_back(ColumnDefine{2, PK_NAME_PK_IS_HANDLE, DataTypeFactory::instance().get("Int32")});
diff --git a/dbms/src/Storages/DeltaMerge/workload/Utils.cpp b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp
index 9917001f991..7a45a605acc 100644
--- a/dbms/src/Storages/DeltaMerge/workload/Utils.cpp
+++ b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <Common/MyTime.h>
+#include <DataTypes/DataTypeDecimal.h>
#include <Storages/DeltaMerge/workload/Utils.h>
#include <fmt/chrono.h>
#include <fmt/ranges.h>
@@ -119,4 +121,4 @@ std::string blockToString(const Block & block)
return s;
}
-} // namespace DB::DM::tests
\ No newline at end of file
+} // namespace DB::DM::tests
diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp
index b7e089dca83..0a087c3b0e9 100644
--- a/dbms/src/Storages/ITableDeclaration.cpp
+++ b/dbms/src/Storages/ITableDeclaration.cpp
@@ -84,8 +84,8 @@ Block ITableDeclaration::getSampleBlockForColumns(const Names & column_names) co
for (const auto & name : column_names)
{
- auto col = name == MutableSupport::extra_table_id_column_name
- ? NameAndTypePair(name, MutableSupport::extra_table_id_column_type)
+ auto col = name == MutSup::extra_table_id_column_name
+ ? NameAndTypePair(name, MutSup::getExtraTableIdColumnType())
: getColumn(name);
res.insert({col.type->createColumn(), col.type, name});
}
diff --git a/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp b/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp
index b615e288fd8..fdfdfe84200 100644
--- a/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp
+++ b/dbms/src/Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.cpp
@@ -13,6 +13,7 @@
// limitations under the License.
#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
#include <Storages/KVStore/Decode/DecodingStorageSchemaSnapshot.h>
#include <TiDB/Schema/TiDB.h>
@@ -41,13 +42,13 @@ DecodingStorageSchemaSnapshot::DecodingStorageSchemaSnapshot(
for (size_t i = 0; i < column_defines->size(); i++)
{
auto & cd = (*column_defines)[i];
- if (cd.id != VersionColumnID || with_version_column)
+ if (cd.id != MutSup::version_col_id || with_version_column)
{
col_id_to_block_pos.insert({cd.id, index_in_block++});
}
col_id_to_def_pos.insert({cd.id, i});
- if (cd.id != TiDBPkColumnID && cd.id != VersionColumnID && cd.id != DelMarkColumnID)
+ if (cd.id != MutSup::extra_handle_id && cd.id != MutSup::version_col_id && cd.id != MutSup::delmark_col_id)
{
const auto & columns = table_info_.columns;
column_infos.push_back(columns[column_lut.at(cd.id)]);
@@ -131,11 +132,11 @@ Block createBlockSortByColumnID(DecodingStorageSchemaSnapshotConstPtr schema_sna
{
// col_id == cd.id
// Including some internal columns:
- // - (VersionColumnID, _INTERNAL_VERSION, u64)
- // - (DelMarkColumnID, _INTERNAL_DELMARK, u8)
- // - (TiDBPkColumnID, _tidb_rowid, i64)
+ // - (MutSup::version_col_id, _INTERNAL_VERSION, u64)
+ // - (MutSup::delmark_col_id, _INTERNAL_DELMARK, u8)
+ // - (MutSup::extra_handle_id, _tidb_rowid, i64)
auto & cd = (*(schema_snapshot->column_defines))[def_pos];
- if (!with_version_column && cd.id == VersionColumnID)
+ if (!with_version_column && cd.id == MutSup::version_col_id)
continue;
block.insert({cd.type->createColumn(), cd.type, cd.name, col_id});
}
diff --git a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp
index d4d5c2515d0..e31892dc9b9 100644
--- a/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp
+++ b/dbms/src/Storages/KVStore/Decode/RegionBlockReader.cpp
@@ -201,17 +201,17 @@ bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool
while (raw_delmark_col == nullptr || version_col_resolver.needBuild()
|| extra_handle_column_pos == invalid_column_pos)
{
- if (column_ids_iter->first == DelMarkColumnID)
+ if (column_ids_iter->first == MutSup::delmark_col_id)
{
raw_delmark_col
= static_cast<ColumnUInt8 *>(const_cast<IColumn *>(block.getByPosition(next_column_pos).column.get()));
}
- else if (column_ids_iter->first == VersionColumnID)
+ else if (column_ids_iter->first == MutSup::version_col_id)
{
version_col_resolver.build(
static_cast<ColumnUInt64 *>(const_cast<IColumn *>(block.getByPosition(next_column_pos).column.get())));
}
- else if (column_ids_iter->first == TiDBPkColumnID)
+ else if (column_ids_iter->first == MutSup::extra_handle_id)
{
extra_handle_column_pos = next_column_pos;
}
@@ -299,12 +299,12 @@ bool RegionBlockReader::readImpl(Block & block, const ReadList & data_list, bool
if constexpr (pk_type == TMTPKType::INT64)
static_cast<ColumnInt64 *>(raw_pk_column)->getData().push_back(handle_value);
else if constexpr (pk_type == TMTPKType::UINT64)
- static_cast<ColumnUInt64 *>(raw_pk_column)->getData().push_back(UInt64(handle_value));
+ static_cast<ColumnUInt64 *>(raw_pk_column)->getData().push_back(static_cast<UInt64>(handle_value));
else
{
// The pk_type must be Int32/UInt32 or more narrow type
// so cannot tell its' exact type here, just use `insert(Field)`
- raw_pk_column->insert(Field(handle_value));
+ raw_pk_column->insert(Field{handle_value});
if (unlikely(raw_pk_column->getInt(index) != handle_value))
{
if (!force_decode)
diff --git a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp
index 94aa55ace5d..e0b8ce26862 100644
--- a/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp
+++ b/dbms/src/Storages/KVStore/MultiRaft/PrehandleSnapshot.cpp
@@ -159,7 +159,7 @@ static inline std::tuple<ReadFromStreamResult, PrehandleResult> executeTransform
// Read from SSTs and refine the boundary of blocks output to DTFiles
auto bounded_stream = std::make_shared<DM::BoundedSSTFilesToBlockInputStream>(
sst_stream,
- ::DB::TiDBPkColumnID,
+ MutSup::extra_handle_id,
opts.schema_snap,
split_id);
diff --git a/dbms/src/Storages/KVStore/Types.h b/dbms/src/Storages/KVStore/Types.h
index be11a1b28be..ace9c7365bb 100644
--- a/dbms/src/Storages/KVStore/Types.h
+++ b/dbms/src/Storages/KVStore/Types.h
@@ -57,14 +57,6 @@ enum : IndexID
EmptyIndexID = 0,
};
-// Constants for column id, prevent conflict with TiDB.
-static constexpr ColumnID TiDBPkColumnID = -1;
-static constexpr ColumnID ExtraTableIDColumnID = -3;
-static constexpr ColumnID VersionColumnID = -1024;
-static constexpr ColumnID DelMarkColumnID = -1025;
-static constexpr ColumnID InvalidColumnID = -10000;
-
-
using HandleID = Int64;
using Timestamp = UInt64;
diff --git a/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp b/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp
index 28d2798bb49..8aa9b1d047a 100644
--- a/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp
+++ b/dbms/src/Storages/KVStore/tests/bench_region_block_reader.cpp
@@ -74,7 +74,7 @@ class RegionBlockReaderBenchTest : public benchmark::Fixture
WriteBufferFromOwnString pk_buf;
if (table_info.is_common_handle)
{
- auto & primary_index_info = table_info.getPrimaryIndexInfo();
+ const auto & primary_index_info = table_info.getPrimaryIndexInfo();
for (size_t i = 0; i < primary_index_info.idx_cols.size(); i++)
{
auto idx = column_name_columns_index_map[primary_index_info.idx_cols[i].name];
@@ -135,7 +135,7 @@ BENCHMARK_DEFINE_F(RegionBlockReaderBenchTest, CommonHandle)
auto [table_info, fields] = getNormalTableInfoFields({2, 3, 4}, true);
encodeColumns(table_info, fields, RowEncodeVersion::RowV2, num_rows);
auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info);
- for (auto _ : state)
+ for (auto _ : state) // NOLINT
{
decodeColumns(decoding_schema, true);
}
@@ -146,10 +146,10 @@ BENCHMARK_DEFINE_F(RegionBlockReaderBenchTest, PKIsNotHandle)
(benchmark::State & state)
{
size_t num_rows = state.range(0);
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV2, num_rows);
auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info);
- for (auto _ : state)
+ for (auto _ : state) // NOLINT
{
decodeColumns(decoding_schema, true);
}
@@ -162,7 +162,7 @@ BENCHMARK_DEFINE_F(RegionBlockReaderBenchTest, PKIsHandle)
auto [table_info, fields] = getNormalTableInfoFields({2}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV2, num_rows);
auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info);
- for (auto _ : state)
+ for (auto _ : state) // NOLINT
{
decodeColumns(decoding_schema, true);
}
diff --git a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp
index d76fbe6c078..8902e6305e0 100644
--- a/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp
+++ b/dbms/src/Storages/KVStore/tests/gtest_region_block_reader.cpp
@@ -153,7 +153,7 @@ class RegionBlockReaderTest : public ::testing::Test
{
ASSERT_EQ(column_element.column->size(), rows);
}
- if (column_element.name == EXTRA_HANDLE_COLUMN_NAME)
+ if (column_element.name == MutSup::extra_handle_column_name)
{
if (decoding_schema->is_common_handle)
{
@@ -165,11 +165,11 @@ class RegionBlockReaderTest : public ::testing::Test
ASSERT_FIELD_EQ((*column_element.column)[row], Field(handle_value)) << gen_error_log();
}
}
- else if (column_element.name == VERSION_COLUMN_NAME)
+ else if (column_element.name == MutSup::version_column_name)
{
ASSERT_FIELD_EQ((*column_element.column)[row], Field(version_value)) << gen_error_log();
}
- else if (column_element.name == TAG_COLUMN_NAME)
+ else if (column_element.name == MutSup::delmark_column_name)
{
ASSERT_FIELD_EQ((*column_element.column)[row], Field(NearestFieldType<UInt8>::Type(del_mark_value)))
<< gen_error_log();
@@ -312,7 +312,7 @@ String bytesFromHexString(std::string_view hex_str)
TEST_F(RegionBlockReaderTest, PKIsNotHandle)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
ASSERT_EQ(table_info.is_common_handle, false);
ASSERT_EQ(table_info.pk_is_handle, false);
ASSERT_FALSE(table_info.getColumnInfo(2).hasPriKeyFlag());
@@ -350,27 +350,27 @@ TEST_F(RegionBlockReaderTest, CommonHandle)
TEST_F(RegionBlockReaderTest, MissingColumnRowV2)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV2);
- auto new_table_info = getTableInfoWithMoreColumns({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoWithMoreColumns({MutSup::extra_handle_id}, false);
auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info);
ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, false));
}
TEST_F(RegionBlockReaderTest, MissingColumnRowV1)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV1);
- auto new_table_info = getTableInfoWithMoreColumns({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoWithMoreColumns({MutSup::extra_handle_id}, false);
auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info);
ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, false));
}
TEST_F(RegionBlockReaderTest, ExtraColumnRowV2)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV2);
- auto new_table_info = getTableInfoWithLessColumns({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoWithLessColumns({MutSup::extra_handle_id}, false);
auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info);
ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false));
ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, true));
@@ -378,9 +378,9 @@ TEST_F(RegionBlockReaderTest, ExtraColumnRowV2)
TEST_F(RegionBlockReaderTest, ExtraColumnRowV1)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV1);
- auto new_table_info = getTableInfoWithLessColumns({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoWithLessColumns({MutSup::extra_handle_id}, false);
auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info);
ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false));
ASSERT_TRUE(decodeAndCheckColumns(new_decoding_schema, true));
@@ -388,9 +388,9 @@ TEST_F(RegionBlockReaderTest, ExtraColumnRowV1)
TEST_F(RegionBlockReaderTest, OverflowColumnRowV2)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV2);
- auto new_table_info = getTableInfoWithMoreNarrowIntType({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoWithMoreNarrowIntType({MutSup::extra_handle_id}, false);
auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info);
ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false));
ASSERT_ANY_THROW(decodeAndCheckColumns(new_decoding_schema, true));
@@ -401,9 +401,9 @@ TEST_F(RegionBlockReaderTest, OverflowColumnRowV2)
TEST_F(RegionBlockReaderTest, OverflowColumnRowV1)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV1);
- auto new_table_info = getTableInfoWithMoreNarrowIntType({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoWithMoreNarrowIntType({MutSup::extra_handle_id}, false);
auto new_decoding_schema = getDecodingStorageSchemaSnapshot(new_table_info);
ASSERT_FALSE(decodeAndCheckColumns(new_decoding_schema, false));
ASSERT_ANY_THROW(decodeAndCheckColumns(new_decoding_schema, true));
@@ -415,12 +415,12 @@ TEST_F(RegionBlockReaderTest, OverflowColumnRowV1)
TEST_F(RegionBlockReaderTest, InvalidNULLRowV2)
try
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
ASSERT_FALSE(table_info.getColumnInfo(11).hasNotNullFlag()); // col 11 is nullable
encodeColumns(table_info, fields, RowEncodeVersion::RowV2);
- auto new_table_info = getTableInfoFieldsForInvalidNULLTest({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoFieldsForInvalidNULLTest({MutSup::extra_handle_id}, false);
invalid_null_column_ids.emplace(11);
ASSERT_TRUE(new_table_info.getColumnInfo(11).hasNotNullFlag()); // col 11 is not null
@@ -432,10 +432,10 @@ CATCH
TEST_F(RegionBlockReaderTest, InvalidNULLRowV1)
{
- auto [table_info, fields] = getNormalTableInfoFields({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto [table_info, fields] = getNormalTableInfoFields({MutSup::extra_handle_id}, false);
encodeColumns(table_info, fields, RowEncodeVersion::RowV1);
- auto new_table_info = getTableInfoFieldsForInvalidNULLTest({EXTRA_HANDLE_COLUMN_ID}, false);
+ auto new_table_info = getTableInfoFieldsForInvalidNULLTest({MutSup::extra_handle_id}, false);
invalid_null_column_ids.emplace(11);
ASSERT_TRUE(new_table_info.getColumnInfo(11).hasNotNullFlag()); // col 11 is not null
diff --git a/dbms/src/Storages/KVStore/tests/gtest_spill.cpp b/dbms/src/Storages/KVStore/tests/gtest_spill.cpp
index e4533c43583..3a50e77ae03 100644
--- a/dbms/src/Storages/KVStore/tests/gtest_spill.cpp
+++ b/dbms/src/Storages/KVStore/tests/gtest_spill.cpp
@@ -52,7 +52,7 @@ class KVStoreSpillTest : public KVStoreTestBase
protected:
StorageDeltaMergePtr storage;
- TableID table_id;
+ TableID table_id{0};
};
TEST_F(KVStoreSpillTest, CreateBlock)
@@ -67,8 +67,8 @@ try
{
auto [schema_snapshot, block] = storage->getSchemaSnapshotAndBlockForDecoding(table_lock, true, false);
UNUSED(schema_snapshot);
- EXPECT_NO_THROW(block->getPositionByName(MutableSupport::delmark_column_name));
- EXPECT_THROW(block->getPositionByName(MutableSupport::version_column_name), Exception);
+ EXPECT_NO_THROW(block->getPositionByName(MutSup::delmark_column_name));
+ EXPECT_THROW(block->getPositionByName(MutSup::version_column_name), Exception);
ASSERT_EQ(block->columns(), 3);
}
{
diff --git a/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp b/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp
index 24b84305e81..36a9df765bf 100644
--- a/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp
+++ b/dbms/src/Storages/KVStore/tests/gtest_sync_status.cpp
@@ -154,7 +154,7 @@ TableID createDBAndTable(String db_name, String table_name)
StorageDeltaMergePtr storage_ptr = std::static_pointer_cast<StorageDeltaMerge>(storage);
table_id = storage_ptr->getTableInfo().id;
- EXPECT_EQ(storage->getName(), MutableSupport::delta_tree_storage_name);
+ EXPECT_EQ(storage->getName(), MutSup::delta_tree_storage_name);
EXPECT_EQ(storage->getTableName(), table_name);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
diff --git a/dbms/src/Storages/MutableSupport.cpp b/dbms/src/Storages/MutableSupport.cpp
index d8b91220375..c4376dca872 100644
--- a/dbms/src/Storages/MutableSupport.cpp
+++ b/dbms/src/Storages/MutableSupport.cpp
@@ -12,27 +12,50 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeString.h>
#include <Storages/MutableSupport.h>
namespace DB
{
-const String MutableSupport::mmt_storage_name = "MutableMergeTree";
-const String MutableSupport::txn_storage_name = "TxnMergeTree";
-const String MutableSupport::delta_tree_storage_name = "DeltaMerge";
-
-const String MutableSupport::tidb_pk_column_name = "_tidb_rowid";
-const String MutableSupport::version_column_name = "_INTERNAL_VERSION";
-const String MutableSupport::delmark_column_name = "_INTERNAL_DELMARK";
-const String MutableSupport::extra_table_id_column_name = "_tidb_tid";
-
-const DataTypePtr MutableSupport::tidb_pk_column_int_type = DataTypeFactory::instance().get("Int64");
-const DataTypePtr MutableSupport::tidb_pk_column_string_type
- = DataTypeFactory::instance().get(DataTypeString::getDefaultName());
-const DataTypePtr MutableSupport::version_column_type = DataTypeFactory::instance().get("UInt64");
-const DataTypePtr MutableSupport::delmark_column_type = DataTypeFactory::instance().get("UInt8");
-/// it should not be nullable, but TiDB does not set not null flag for extra_table_id_column_type, so has to align with TiDB
-const DataTypePtr MutableSupport::extra_table_id_column_type = DataTypeFactory::instance().get("Nullable(Int64)");
-;
+const DataTypePtr & MutSup::getExtraHandleColumnIntType()
+{
+ static const auto type = DataTypeFactory::instance().getOrSet("Int64");
+ return type;
+}
+
+const DataTypePtr & MutSup::getExtraHandleColumnStringType()
+{
+ static const auto name_and_types = DataTypeString::getTiDBPkColumnStringNameAndTypes();
+
+ const auto default_name = DataTypeString::getDefaultName();
+ auto itr = std::find_if(name_and_types.begin(), name_and_types.end(), [&default_name](const auto & name_and_type) {
+ return name_and_type.first == default_name;
+ });
+ RUNTIME_CHECK_MSG(
+ itr != name_and_types.end(),
+ "Cannot find '{}' for TiDB primary key column string type",
+ default_name);
+ return itr->second;
+}
+
+const DataTypePtr & MutSup::getVersionColumnType()
+{
+ static const auto type = DataTypeFactory::instance().getOrSet("UInt64");
+ return type;
+}
+
+const DataTypePtr & MutSup::getDelmarkColumnType()
+{
+ static const auto type = DataTypeFactory::instance().getOrSet("UInt8");
+ return type;
+}
+
+const DataTypePtr & MutSup::getExtraTableIdColumnType()
+{
+ /// it should not be nullable, but TiDB does not set not null flag for extra_table_id_column_type, so has to align with TiDB
+ static const auto type = DataTypeFactory::instance().getOrSet("Nullable(Int64)");
+ return type;
+}
} // namespace DB
diff --git a/dbms/src/Storages/MutableSupport.h b/dbms/src/Storages/MutableSupport.h
index 6ac3b262e3b..5a098ccc313 100644
--- a/dbms/src/Storages/MutableSupport.h
+++ b/dbms/src/Storages/MutableSupport.h
@@ -14,84 +14,53 @@
#pragma once
-#include <Common/typeid_cast.h>
-#include <Core/Block.h>
#include <Core/Names.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <IO/WriteHelpers.h>
+#include <DataTypes/IDataType.h>
+#include <Storages/KVStore/Types.h>
#include <ext/singleton.h>
namespace DB
{
-class MutableSupport : public ext::Singleton<MutableSupport>
+class MutSup : public ext::Singleton<MutSup>
{
public:
- MutableSupport()
+ MutSup()
{
mutable_hidden.push_back(version_column_name);
mutable_hidden.push_back(delmark_column_name);
- //mutable_hidden.push_back(tidb_pk_column_name);
-
- all_hidden.insert(all_hidden.end(), mutable_hidden.begin(), mutable_hidden.end());
+ //mutable_hidden.push_back(extra_handle_column_name);
}
const OrderedNameSet & hiddenColumns(const String & table_type_name)
{
- if (mmt_storage_name == table_type_name || txn_storage_name == table_type_name
- || delta_tree_storage_name == table_type_name)
+ if (delta_tree_storage_name == table_type_name)
return mutable_hidden;
return empty;
}
- static const String mmt_storage_name;
- static const String txn_storage_name;
- static const String delta_tree_storage_name;
-
- static const String tidb_pk_column_name;
- static const String version_column_name;
- static const String delmark_column_name;
- static const String extra_table_id_column_name;
-
- static const DataTypePtr tidb_pk_column_int_type;
- static const DataTypePtr tidb_pk_column_string_type;
- static const DataTypePtr version_column_type;
- static const DataTypePtr delmark_column_type;
- static const DataTypePtr extra_table_id_column_type;
-
- /// mark that ColumnId of those columns are defined in dbms/src/Storages/KVStore/Types.h
-
- // TODO: detail doc about these delete marks
- struct DelMark
- {
- static const UInt8 NONE = 0x00;
- static const UInt8 INTERNAL_DEL = 0x01;
- static const UInt8 DEFINITE_DEL = (INTERNAL_DEL << 1);
- static const UInt8 DEL_MASK = (INTERNAL_DEL | DEFINITE_DEL);
- static const UInt8 DATA_MASK = ~DEL_MASK;
-
- static UInt8 getData(UInt8 raw_data) { return raw_data & DATA_MASK; }
-
- static bool isDel(UInt8 raw_data) { return raw_data & DEL_MASK; }
+ inline static const String delta_tree_storage_name = "DeltaMerge";
- static bool isDefiniteDel(UInt8 raw_data) { return raw_data & DEFINITE_DEL; }
+ inline static constexpr ColumnID extra_handle_id = -1;
+ inline static constexpr ColumnID extra_table_id_col_id = -3;
+ inline static constexpr ColumnID version_col_id = -1024;
+ inline static constexpr ColumnID delmark_col_id = -1025;
+ inline static constexpr ColumnID invalid_col_id = -10000;
- static UInt8 genDelMark(bool internal_del, bool definite_del, UInt8 src_data)
- {
- return (internal_del ? INTERNAL_DEL : NONE) | (definite_del ? DEFINITE_DEL : NONE) | getData(src_data);
- }
+ inline static const String extra_handle_column_name = "_tidb_rowid";
+ inline static const String version_column_name = "_INTERNAL_VERSION";
+ inline static const String delmark_column_name = "_INTERNAL_DELMARK";
+ inline static const String extra_table_id_column_name = "_tidb_tid";
- static UInt8 genDelMark(bool internal_del, bool definite_del = false)
- {
- return genDelMark(internal_del, definite_del, 0);
- }
- };
+ ALWAYS_INLINE static const DataTypePtr & getExtraHandleColumnIntType();
+ ALWAYS_INLINE static const DataTypePtr & getExtraHandleColumnStringType();
+ ALWAYS_INLINE static const DataTypePtr & getVersionColumnType();
+ ALWAYS_INLINE static const DataTypePtr & getDelmarkColumnType();
+ ALWAYS_INLINE static const DataTypePtr & getExtraTableIdColumnType();
private:
OrderedNameSet empty;
OrderedNameSet mutable_hidden;
- OrderedNameSet all_hidden;
};
} // namespace DB
diff --git a/dbms/src/Storages/S3/FileCache.cpp b/dbms/src/Storages/S3/FileCache.cpp
index 9ea59004bc5..c47797d86a5 100644
--- a/dbms/src/Storages/S3/FileCache.cpp
+++ b/dbms/src/Storages/S3/FileCache.cpp
@@ -601,11 +601,11 @@ FileType FileCache::getFileTypeOfColData(const std::filesystem::path & p)
auto col_id = std::stol(str_col_id);
switch (col_id)
{
- case EXTRA_HANDLE_COLUMN_ID:
+ case MutSup::extra_handle_id:
return FileType::HandleColData;
- case VERSION_COLUMN_ID:
+ case MutSup::version_col_id:
return FileType::VersionColData;
- case TAG_COLUMN_ID:
+ case MutSup::delmark_col_id:
return FileType::DeleteMarkColData;
default:
return FileType::ColData;
diff --git a/dbms/src/Storages/S3/tests/gtest_filecache.cpp b/dbms/src/Storages/S3/tests/gtest_filecache.cpp
index e88571e79ab..5dca3b79dce 100644
--- a/dbms/src/Storages/S3/tests/gtest_filecache.cpp
+++ b/dbms/src/Storages/S3/tests/gtest_filecache.cpp
@@ -192,8 +192,7 @@ class FileCacheTest : public ::testing::Test
void waitForBgDownload(const FileCache & file_cache)
{
Stopwatch sw;
- UInt64 downloading = 0;
- while ((downloading = file_cache.bg_downloading_count.load(std::memory_order_relaxed)) > 0)
+ while (file_cache.bg_downloading_count.load(std::memory_order_relaxed) > 0)
{
std::this_thread::sleep_for(1000ms);
}
@@ -430,15 +429,19 @@ try
auto handle_fname = fmt::format(
"{}/{}.dat",
s3_fname,
- IDataType::getFileNameForStream(std::to_string(EXTRA_HANDLE_COLUMN_ID), {}));
+ IDataType::getFileNameForStream(std::to_string(MutSup::extra_handle_id), {}));
ASSERT_EQ(FileCache::getFileType(handle_fname), FileType::HandleColData);
auto vec_index_fname = fmt::format("{}/idx_{}.vector", s3_fname, /*index_id*/ 50); // DMFile::vectorIndexFileName
ASSERT_EQ(FileCache::getFileType(vec_index_fname), FileType::VectorIndex);
- auto version_fname
- = fmt::format("{}/{}.dat", s3_fname, IDataType::getFileNameForStream(std::to_string(VERSION_COLUMN_ID), {}));
+ auto version_fname = fmt::format(
+ "{}/{}.dat",
+ s3_fname,
+ IDataType::getFileNameForStream(std::to_string(MutSup::version_col_id), {}));
ASSERT_EQ(FileCache::getFileType(version_fname), FileType::VersionColData);
- auto delmark_fname
- = fmt::format("{}/{}.dat", s3_fname, IDataType::getFileNameForStream(std::to_string(TAG_COLUMN_ID), {}));
+ auto delmark_fname = fmt::format(
+ "{}/{}.dat",
+ s3_fname,
+ IDataType::getFileNameForStream(std::to_string(MutSup::delmark_col_id), {}));
ASSERT_EQ(FileCache::getFileType(delmark_fname), FileType::DeleteMarkColData);
auto unknow_fname0 = fmt::format("{}/123456", s3_fname);
ASSERT_EQ(FileCache::getFileType(unknow_fname0), FileType::Unknow);
diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp
index 685d57071b6..5e9bbd07643 100644
--- a/dbms/src/Storages/StorageDeltaMerge.cpp
+++ b/dbms/src/Storages/StorageDeltaMerge.cpp
@@ -149,8 +149,8 @@ void StorageDeltaMerge::updateTableColumnInfo()
}
else
{
- pks.emplace(EXTRA_HANDLE_COLUMN_NAME);
- pk_column_names.emplace_back(EXTRA_HANDLE_COLUMN_NAME);
+ pks.emplace(MutSup::extra_handle_column_name);
+ pk_column_names.emplace_back(MutSup::extra_handle_column_name);
}
}
else
@@ -188,8 +188,8 @@ void StorageDeltaMerge::updateTableColumnInfo()
col_def.default_value = itr->defaultValueToField();
}
- if (col_def.id != TiDBPkColumnID && col_def.id != VersionColumnID && col_def.id != DelMarkColumnID
- && tidb_table_info.getColumnInfo(col_def.id).hasPriKeyFlag())
+ if (col_def.id != MutSup::extra_handle_id && col_def.id != MutSup::version_col_id
+ && col_def.id != MutSup::delmark_col_id && tidb_table_info.getColumnInfo(col_def.id).hasPriKeyFlag())
{
rowkey_column_defines.push_back(col_def);
}
@@ -225,15 +225,15 @@ void StorageDeltaMerge::updateTableColumnInfo()
table_column_defines.push_back(col_def);
}
- if (!new_columns.materialized.contains(VERSION_COLUMN_NAME))
+ if (!new_columns.materialized.contains(MutSup::version_column_name))
{
- hidden_columns.emplace_back(VERSION_COLUMN_NAME);
- new_columns.materialized.emplace_back(VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE);
+ hidden_columns.emplace_back(MutSup::version_column_name);
+ new_columns.materialized.emplace_back(MutSup::version_column_name, MutSup::getVersionColumnType());
}
- if (!new_columns.materialized.contains(TAG_COLUMN_NAME))
+ if (!new_columns.materialized.contains(MutSup::delmark_column_name))
{
- hidden_columns.emplace_back(TAG_COLUMN_NAME);
- new_columns.materialized.emplace_back(TAG_COLUMN_NAME, TAG_COLUMN_TYPE);
+ hidden_columns.emplace_back(MutSup::delmark_column_name);
+ new_columns.materialized.emplace_back(MutSup::delmark_column_name, MutSup::getDelmarkColumnType());
}
if (pks.size() > 1)
@@ -242,13 +242,15 @@ void StorageDeltaMerge::updateTableColumnInfo()
{
throw Exception("Should not reach here: common handle with pk size > 1", ErrorCodes::LOGICAL_ERROR);
}
- handle_column_define.id = EXTRA_HANDLE_COLUMN_ID;
- handle_column_define.name = EXTRA_HANDLE_COLUMN_NAME;
- handle_column_define.type = EXTRA_HANDLE_COLUMN_INT_TYPE;
- if (!new_columns.materialized.contains(EXTRA_HANDLE_COLUMN_NAME))
+ handle_column_define.id = MutSup::extra_handle_id;
+ handle_column_define.name = MutSup::extra_handle_column_name;
+ handle_column_define.type = MutSup::getExtraHandleColumnIntType();
+ if (!new_columns.materialized.contains(MutSup::extra_handle_column_name))
{
- hidden_columns.emplace_back(EXTRA_HANDLE_COLUMN_NAME);
- new_columns.materialized.emplace_back(EXTRA_HANDLE_COLUMN_NAME, EXTRA_HANDLE_COLUMN_INT_TYPE);
+ hidden_columns.emplace_back(MutSup::extra_handle_column_name);
+ new_columns.materialized.emplace_back(
+ MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnIntType());
}
}
@@ -357,12 +359,12 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const
if (!is_import)
{
// Remove the default columns generated by InterpreterInsertQuery
- if (to_write.has(EXTRA_HANDLE_COLUMN_NAME))
- to_write.erase(EXTRA_HANDLE_COLUMN_NAME);
- if (to_write.has(VERSION_COLUMN_NAME))
- to_write.erase(VERSION_COLUMN_NAME);
- if (to_write.has(TAG_COLUMN_NAME))
- to_write.erase(TAG_COLUMN_NAME);
+ if (to_write.has(MutSup::extra_handle_column_name))
+ to_write.erase(MutSup::extra_handle_column_name);
+ if (to_write.has(MutSup::version_column_name))
+ to_write.erase(MutSup::version_column_name);
+ if (to_write.has(MutSup::delmark_column_name))
+ to_write.erase(MutSup::delmark_column_name);
}
auto & store = getAndMaybeInitStore();
@@ -391,18 +393,18 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const
addColumnToBlock(
to_write,
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE,
+ MutSup::extra_handle_id,
+ MutSup::extra_handle_column_name,
+ is_common_handle ? MutSup::getExtraHandleColumnStringType() : MutSup::getExtraHandleColumnIntType(),
std::move(handle_column));
}
auto block = DeltaMergeStore::addExtraColumnIfNeed(global_context, store->getHandle(), std::move(to_write));
// add version column
- if (!block.has(VERSION_COLUMN_NAME))
+ if (!block.has(MutSup::version_column_name))
{
- auto column = VERSION_COLUMN_TYPE->createColumn();
+ auto column = MutSup::getVersionColumnType()->createColumn();
auto & column_data = typeid_cast<ColumnVector<UInt64> &>(*column).getData();
column_data.resize(rows);
for (size_t i = 0; i < rows; ++i)
@@ -410,13 +412,18 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const
column_data[i] = next_version++;
}
- addColumnToBlock(block, VERSION_COLUMN_ID, VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE, std::move(column));
+ addColumnToBlock(
+ block,
+ MutSup::version_col_id,
+ MutSup::version_column_name,
+ MutSup::getVersionColumnType(),
+ std::move(column));
}
// add tag column (upsert / delete)
- if (!block.has(TAG_COLUMN_NAME))
+ if (!block.has(MutSup::delmark_column_name))
{
- auto column = TAG_COLUMN_TYPE->createColumn();
+ auto column = MutSup::getDelmarkColumnType()->createColumn();
auto & column_data = typeid_cast<ColumnVector<UInt8> &>(*column).getData();
column_data.resize(rows);
UInt8 tag = is_delete ? 1 : 0;
@@ -425,19 +432,24 @@ Block StorageDeltaMerge::buildInsertBlock(bool is_import, bool is_delete, const
column_data[i] = tag;
}
- addColumnToBlock(block, TAG_COLUMN_ID, TAG_COLUMN_NAME, TAG_COLUMN_TYPE, std::move(column));
+ addColumnToBlock(
+ block,
+ MutSup::delmark_col_id,
+ MutSup::delmark_column_name,
+ MutSup::getDelmarkColumnType(),
+ std::move(column));
}
// Set the real column id.
auto header = store->getHeader();
for (auto & col : block)
{
- if (col.name == EXTRA_HANDLE_COLUMN_NAME)
- col.column_id = EXTRA_HANDLE_COLUMN_ID;
- else if (col.name == VERSION_COLUMN_NAME)
- col.column_id = VERSION_COLUMN_ID;
- else if (col.name == TAG_COLUMN_NAME)
- col.column_id = TAG_COLUMN_ID;
+ if (col.name == MutSup::extra_handle_column_name)
+ col.column_id = MutSup::extra_handle_id;
+ else if (col.name == MutSup::version_column_name)
+ col.column_id = MutSup::version_col_id;
+ else if (col.name == MutSup::delmark_column_name)
+ col.column_id = MutSup::delmark_col_id;
else
col.column_id = header->getByName(col.name).column_id;
}
@@ -536,25 +548,25 @@ WriteResult StorageDeltaMerge::write(
{
name = col.name;
cid = col.column_id;
- if (col.name == EXTRA_HANDLE_COLUMN_NAME)
+ if (col.name == MutSup::extra_handle_column_name)
{
- if (col.column_id != EXTRA_HANDLE_COLUMN_ID)
+ if (col.column_id != MutSup::extra_handle_id)
{
ok = false;
break;
}
}
- else if (col.name == VERSION_COLUMN_NAME)
+ else if (col.name == MutSup::version_column_name)
{
- if (col.column_id != VERSION_COLUMN_ID)
+ if (col.column_id != MutSup::version_col_id)
{
ok = false;
break;
}
}
- else if (col.name == TAG_COLUMN_NAME)
+ else if (col.name == MutSup::delmark_column_name)
{
- if (col.column_id != TAG_COLUMN_ID)
+ if (col.column_id != MutSup::delmark_col_id)
{
ok = false;
break;
@@ -655,13 +667,13 @@ void setColumnsToRead(
for (size_t i = 0; i < column_names.size(); i++)
{
ColumnDefine col_define;
- if (column_names[i] == EXTRA_HANDLE_COLUMN_NAME)
+ if (column_names[i] == MutSup::extra_handle_column_name)
col_define = store->getHandle();
- else if (column_names[i] == VERSION_COLUMN_NAME)
+ else if (column_names[i] == MutSup::version_column_name)
col_define = getVersionColumnDefine();
- else if (column_names[i] == TAG_COLUMN_NAME)
+ else if (column_names[i] == MutSup::delmark_column_name)
col_define = getTagColumnDefine();
- else if (column_names[i] == EXTRA_TABLE_ID_COLUMN_NAME)
+ else if (column_names[i] == MutSup::extra_table_id_column_name)
{
extra_table_id_index = i;
continue;
@@ -800,7 +812,7 @@ BlockInputStreams StorageDeltaMerge::read(
// in `Coprocessor.TableScan.columns`, or rough set filter could be
// failed to parsed.
ColumnDefines columns_to_read;
- size_t extra_table_id_index = InvalidColumnID;
+ size_t extra_table_id_index = MutSup::invalid_col_id;
setColumnsToRead(store, columns_to_read, extra_table_id_index, column_names);
const ASTSelectQuery & select_query = typeid_cast<const ASTSelectQuery &>(*query_info.query);
@@ -881,7 +893,7 @@ void StorageDeltaMerge::read(
// in `Coprocessor.TableScan.columns`, or rough set filter could be
// failed to parsed.
ColumnDefines columns_to_read;
- size_t extra_table_id_index = InvalidColumnID;
+ size_t extra_table_id_index = MutSup::invalid_col_id;
setColumnsToRead(store, columns_to_read, extra_table_id_index, column_names);
const ASTSelectQuery & select_query = typeid_cast<const ASTSelectQuery &>(*query_info.query);
@@ -959,7 +971,7 @@ DM::Remote::DisaggPhysicalTableReadSnapshotPtr StorageDeltaMerge::writeNodeBuild
{
auto & store = getAndMaybeInitStore();
ColumnDefines columns_to_read;
- size_t extra_table_id_index = InvalidColumnID;
+ size_t extra_table_id_index = MutSup::invalid_col_id;
setColumnsToRead(store, columns_to_read, extra_table_id_index, column_names);
auto tracing_logger = log->getChild(query_info.req_id);
@@ -1353,9 +1365,9 @@ NamesAndTypes getColumnsFromTableInfo(const TiDB::TableInfo & table_info)
{
// Make primary key as a column, and make the handle column as the primary key.
if (table_info.is_common_handle)
- columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared<DataTypeString>());
+ columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared<DataTypeString>());
else
- columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared<DataTypeInt64>());
+ columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared<DataTypeInt64>());
}
return columns;
@@ -1449,8 +1461,8 @@ ColumnDefines StorageDeltaMerge::getStoreColumnDefines() const
cols.emplace_back(getTagColumnDefine());
for (const auto & col : table_column_info->table_column_defines)
{
- if (col.id != table_column_info->handle_column_define.id && col.id != VERSION_COLUMN_ID
- && col.id != TAG_COLUMN_ID)
+ if (col.id != table_column_info->handle_column_define.id && col.id != MutSup::version_col_id
+ && col.id != MutSup::delmark_col_id)
{
cols.emplace_back(col);
}
@@ -1460,7 +1472,7 @@ ColumnDefines StorageDeltaMerge::getStoreColumnDefines() const
String StorageDeltaMerge::getName() const
{
- return MutableSupport::delta_tree_storage_name;
+ return MutSup::delta_tree_storage_name;
}
void StorageDeltaMerge::rename(
@@ -1605,7 +1617,7 @@ void updateDeltaMergeTableCreateStatement(
"Wrong arguments num: {} in table: {} with engine={}",
args.children.size(),
table_name,
- MutableSupport::delta_tree_storage_name),
+ MutSup::delta_tree_storage_name),
ErrorCodes::BAD_ARGUMENTS);
}
};
diff --git a/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp b/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp
index afd14ff6381..6934d40272b 100644
--- a/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp
+++ b/dbms/src/Storages/System/StorageSystemDTLocalIndexes.cpp
@@ -101,7 +101,7 @@ BlockInputStreams StorageSystemDTLocalIndexes::read(
{
const auto & table_name = it->name();
auto & storage = it->table();
- if (storage->getName() != MutableSupport::delta_tree_storage_name)
+ if (storage->getName() != MutSup::delta_tree_storage_name)
continue;
auto dm_storage = std::dynamic_pointer_cast<StorageDeltaMerge>(storage);
diff --git a/dbms/src/Storages/System/StorageSystemDTSegments.cpp b/dbms/src/Storages/System/StorageSystemDTSegments.cpp
index 0502501a891..d87d38d0cec 100644
--- a/dbms/src/Storages/System/StorageSystemDTSegments.cpp
+++ b/dbms/src/Storages/System/StorageSystemDTSegments.cpp
@@ -101,7 +101,7 @@ BlockInputStreams StorageSystemDTSegments::read(
{
const auto & table_name = it->name();
auto & storage = it->table();
- if (storage->getName() != MutableSupport::delta_tree_storage_name)
+ if (storage->getName() != MutSup::delta_tree_storage_name)
continue;
auto dm_storage = std::dynamic_pointer_cast<StorageDeltaMerge>(storage);
diff --git a/dbms/src/Storages/System/StorageSystemDTTables.cpp b/dbms/src/Storages/System/StorageSystemDTTables.cpp
index 33c8813128d..5143a0290b0 100644
--- a/dbms/src/Storages/System/StorageSystemDTTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemDTTables.cpp
@@ -135,7 +135,7 @@ BlockInputStreams StorageSystemDTTables::read(
{
const auto & table_name = it->name();
auto & storage = it->table();
- if (storage->getName() != MutableSupport::delta_tree_storage_name)
+ if (storage->getName() != MutSup::delta_tree_storage_name)
continue;
auto dm_storage = std::dynamic_pointer_cast<StorageDeltaMerge>(storage);
diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp
index 18984b0c2c2..8d0e40f80ce 100644
--- a/dbms/src/Storages/System/StorageSystemTables.cpp
+++ b/dbms/src/Storages/System/StorageSystemTables.cpp
@@ -225,8 +225,7 @@ BlockInputStreams StorageSystemTables::read(
String tidb_table_name;
TableID table_id = -1;
Timestamp tombstone = 0;
- if (engine_name == MutableSupport::txn_storage_name
- || engine_name == MutableSupport::delta_tree_storage_name)
+ if (engine_name == MutSup::delta_tree_storage_name)
{
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(iterator->table());
if (managed_storage)
@@ -242,10 +241,10 @@ BlockInputStreams StorageSystemTables::read(
res_columns[j++]->insert(tidb_database_name);
res_columns[j++]->insert(tidb_table_name);
- res_columns[j++]->insert(Int64(table_id));
- res_columns[j++]->insert(UInt64(tombstone));
+ res_columns[j++]->insert(static_cast<Int64>(table_id));
+ res_columns[j++]->insert(static_cast<UInt64>(tombstone));
- res_columns[j++]->insert(UInt64(0));
+ res_columns[j++]->insert(static_cast<UInt64>(0));
res_columns[j++]->insert(iterator->table()->getDataPath());
res_columns[j++]->insert(database->getTableMetadataPath(table_name));
@@ -293,7 +292,7 @@ BlockInputStreams StorageSystemTables::read(
res_columns[j++]->insertDefault();
res_columns[j++]->insert(table.first);
res_columns[j++]->insert(table.second->getName());
- res_columns[j++]->insert(UInt64(1));
+ res_columns[j++]->insert(static_cast<UInt64>(1));
res_columns[j++]->insertDefault();
res_columns[j++]->insertDefault();
diff --git a/dbms/src/TestUtils/InputStreamTestUtils.h b/dbms/src/TestUtils/InputStreamTestUtils.h
index 231a7777517..a36258f2392 100644
--- a/dbms/src/TestUtils/InputStreamTestUtils.h
+++ b/dbms/src/TestUtils/InputStreamTestUtils.h
@@ -74,7 +74,7 @@ ::testing::AssertionResult InputStreamVSBlockUnrestrictlyCompareColumns(
// Similar to `InputStreamVSBlockUnrestrictlyCompareColumns` but assume inputstream's blocks are unordered.
// It is only used for normal mode. (The blocks of fast mode have overlap.)
-// This function read all blocks first and sort them by handle column or column at position 0 if EXTRA_HANDLE_COLUMN_NAME not exist.
+// This function read all blocks first and sort them by handle column or column at position 0 if MutSup::extra_handle_column_name not exist.
::testing::AssertionResult UnorderedInputStreamVSBlockUnrestrictlyCompareColumns(
const char * stream_expr,
const char * colnames_expr,
diff --git a/dbms/src/TiDB/Decode/RowCodec.cpp b/dbms/src/TiDB/Decode/RowCodec.cpp
index cda807e87a7..14f0cf7f240 100644
--- a/dbms/src/TiDB/Decode/RowCodec.cpp
+++ b/dbms/src/TiDB/Decode/RowCodec.cpp
@@ -334,7 +334,7 @@ bool appendRowToBlock(
{
const ColumnInfos & column_infos = schema_snapshot->column_infos;
// when pk is handle, we need skip pk column when decoding value
- ColumnID pk_handle_id = InvalidColumnID;
+ ColumnID pk_handle_id = MutSup::invalid_col_id;
if (schema_snapshot->pk_is_handle)
{
pk_handle_id = schema_snapshot->pk_column_ids[0];
diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp
index 2db35dd3fe7..110d1647155 100644
--- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp
+++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp
@@ -1061,11 +1061,11 @@ std::tuple<NamesAndTypes, Strings> parseColumnsFromTableInfo(const TiDB::TableIn
{
// Make primary key as a column, and make the handle column as the primary key.
if (table_info.is_common_handle)
- columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared<DataTypeString>());
+ columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared<DataTypeString>());
else
- columns.emplace_back(MutableSupport::tidb_pk_column_name, std::make_shared<DataTypeInt64>());
+ columns.emplace_back(MutSup::extra_handle_column_name, std::make_shared<DataTypeInt64>());
primary_keys.clear();
- primary_keys.emplace_back(MutableSupport::tidb_pk_column_name);
+ primary_keys.emplace_back(MutSup::extra_handle_column_name);
}
return std::make_tuple(std::move(columns), std::move(primary_keys));
diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp
index 2a5dd792044..5f846e1cd4b 100644
--- a/dbms/src/TiDB/Schema/TiDB.cpp
+++ b/dbms/src/TiDB/Schema/TiDB.cpp
@@ -1183,12 +1183,12 @@ ColumnID TableInfo::getColumnID(const String & name) const
}
}
- if (name == DB::MutableSupport::tidb_pk_column_name)
- return DB::TiDBPkColumnID;
- else if (name == DB::MutableSupport::version_column_name)
- return DB::VersionColumnID;
- else if (name == DB::MutableSupport::delmark_column_name)
- return DB::DelMarkColumnID;
+ if (name == DB::MutSup::extra_handle_column_name)
+ return DB::MutSup::extra_handle_id;
+ else if (name == DB::MutSup::version_column_name)
+ return DB::MutSup::version_col_id;
+ else if (name == DB::MutSup::delmark_column_name)
+ return DB::MutSup::delmark_col_id;
DB::Strings available_columns;
for (const auto & c : columns)
diff --git a/dbms/src/TiDB/tests/RowCodecTestUtils.h b/dbms/src/TiDB/tests/RowCodecTestUtils.h
index fdf88a4d9ea..6b27e51878d 100644
--- a/dbms/src/TiDB/tests/RowCodecTestUtils.h
+++ b/dbms/src/TiDB/tests/RowCodecTestUtils.h
@@ -225,7 +225,7 @@ std::pair<TableInfo, std::vector<Field>> getTableInfoAndFields(
getTableInfoFieldsInternal(column_info_fields, std::forward<Types>(column_value_ids)...);
TableInfo table_info;
std::vector<Field> fields;
- bool pk_is_handle = pk_col_ids.size() == 1 && pk_col_ids[0] != ::DB::TiDBPkColumnID;
+ bool pk_is_handle = pk_col_ids.size() == 1 && pk_col_ids[0] != MutSup::extra_handle_id;
for (auto & column_info_field : column_info_fields)
{
@@ -280,22 +280,22 @@ inline DecodingStorageSchemaSnapshotConstPtr getDecodingStorageSchemaSnapshot(co
if (table_info.is_common_handle)
{
DM::ColumnDefine extra_handle_column{
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_STRING_TYPE};
+ MutSup::extra_handle_id,
+ MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnStringType()};
store_columns.emplace_back(extra_handle_column);
}
else
{
DM::ColumnDefine extra_handle_column{
- EXTRA_HANDLE_COLUMN_ID,
- EXTRA_HANDLE_COLUMN_NAME,
- EXTRA_HANDLE_COLUMN_INT_TYPE};
+ MutSup::extra_handle_id,
+ MutSup::extra_handle_column_name,
+ MutSup::getExtraHandleColumnIntType()};
store_columns.emplace_back(extra_handle_column);
}
- store_columns.emplace_back(VERSION_COLUMN_ID, VERSION_COLUMN_NAME, VERSION_COLUMN_TYPE);
- store_columns.emplace_back(TAG_COLUMN_ID, TAG_COLUMN_NAME, TAG_COLUMN_TYPE);
- ColumnID handle_id = EXTRA_HANDLE_COLUMN_ID;
+ store_columns.emplace_back(MutSup::version_col_id, MutSup::version_column_name, MutSup::getVersionColumnType());
+ store_columns.emplace_back(MutSup::delmark_col_id, MutSup::delmark_column_name, MutSup::getDelmarkColumnType());
+ ColumnID handle_id = MutSup::extra_handle_id;
for (const auto & column_info : table_info.columns)
{
if (table_info.pk_is_handle)
@@ -306,7 +306,7 @@ inline DecodingStorageSchemaSnapshotConstPtr getDecodingStorageSchemaSnapshot(co
store_columns.emplace_back(column_info.id, column_info.name, DB::getDataTypeByColumnInfo(column_info));
}
- if (handle_id != EXTRA_HANDLE_COLUMN_ID)
+ if (handle_id != MutSup::extra_handle_id)
{
auto iter = std::find_if(store_columns.begin(), store_columns.end(), [&](const ColumnDefine & cd) {
return cd.id == handle_id;
@@ -357,7 +357,7 @@ template <bool is_big, typename T>
std::tuple<T, size_t> getValueLengthByRowV2(const T & v)
{
using NearestType = typename NearestFieldType<T>::Type;
- auto [table_info, fields] = getTableInfoAndFields({EXTRA_HANDLE_COLUMN_ID}, false, ColumnIDValue(1, v));
+ auto [table_info, fields] = getTableInfoAndFields({MutSup::extra_handle_id}, false, ColumnIDValue(1, v));
auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info);
WriteBufferFromOwnString ss;
encodeRowV2(table_info, fields, ss);
@@ -372,7 +372,7 @@ template <typename T>
T getValueByRowV1(const T & v)
{
using NearestType = typename NearestFieldType<T>::Type;
- auto [table_info, fields] = getTableInfoAndFields({EXTRA_HANDLE_COLUMN_ID}, false, ColumnIDValue(1, v));
+ auto [table_info, fields] = getTableInfoAndFields({MutSup::extra_handle_id}, false, ColumnIDValue(1, v));
auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info);
WriteBufferFromOwnString ss;
encodeRowV1(table_info, fields, ss);
diff --git a/dbms/src/TiDB/tests/gtest_row_v2.cpp b/dbms/src/TiDB/tests/gtest_row_v2.cpp
index 5341468e8ce..be01a42b04a 100644
--- a/dbms/src/TiDB/tests/gtest_row_v2.cpp
+++ b/dbms/src/TiDB/tests/gtest_row_v2.cpp
@@ -82,7 +82,7 @@ TEST(RowV2Suite, FloatValue)
ASSERT_FLOAT_VALUE(std::numeric_limits<Float64>::max());
}
-#define ASSERT_STRING_VALUE_LENGTH(b, s) ASSERT_EQ(getValueLengthByRowV2<b>(s), std::make_tuple(s, s.length()))
+#define ASSERT_STRING_VALUE_LENGTH(b, s) ASSERT_EQ(getValueLengthByRowV2<b>(s), std::make_tuple(s, (s).length()))
TEST(RowV2Suite, StringValueLength)
{
@@ -101,20 +101,20 @@ TEST(RowV2Suite, DecimalValueLength)
ASSERT_DECIMAL_VALUE(DecimalField(ToDecimal<Float64, Decimal32>(1234.56789, 2), 2));
}
-#define ASSERT_ROW_VALUE(is_big, ...) \
- { \
- auto [table_info, fields] = getTableInfoAndFields({EXTRA_HANDLE_COLUMN_ID}, false, __VA_ARGS__); \
- auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); \
- WriteBufferFromOwnString ss; \
- encodeRowV2(table_info, fields, ss); \
- auto encoded = ss.str(); \
- ASSERT_EQ(is_big, isBig(encoded)); \
- auto block = decodeRowToBlock(encoded, decoding_schema); \
- ASSERT_EQ(fields.size(), block.columns()); \
- for (size_t i = 0; i < fields.size(); i++) \
- { \
- ASSERT_EQ(fields[i], ((*block.getByPosition(i).column)[0])); \
- } \
+#define ASSERT_ROW_VALUE(is_big, ...) \
+ { \
+ auto [table_info, fields] = getTableInfoAndFields({MutSup::extra_handle_id}, false, __VA_ARGS__); \
+ auto decoding_schema = getDecodingStorageSchemaSnapshot(table_info); \
+ WriteBufferFromOwnString ss; \
+ encodeRowV2(table_info, fields, ss); \
+ auto encoded = ss.str(); \
+ ASSERT_EQ(is_big, isBig(encoded)); \
+ auto block = decodeRowToBlock(encoded, decoding_schema); \
+ ASSERT_EQ(fields.size(), block.columns()); \
+ for (size_t i = 0; i < fields.size(); i++) \
+ { \
+ ASSERT_EQ(fields[i], ((*block.getByPosition(i).column)[0])); \
+ } \
}
TEST(RowV2Suite, SmallRow)