diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index dd70119d24510c..4713845a0a3c81 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -1846,7 +1846,7 @@ void parse_json_to_variant_impl(IColumn& column, const char* src, size_t length, auto& [paths, values] = *result; assert(paths.size() == values.size()); size_t old_num_rows = column_variant.rows(); - if (config.enable_flatten_nested) { + if (config.deprecated_enable_flatten_nested) { // here we should check the paths in variant and paths in result, // if two paths which same prefix have different structure, we should throw an exception std::vector check_paths; @@ -2155,7 +2155,9 @@ Status parse_and_materialize_variant_columns(Block& block, const TabletSchema& t std::vector configs(variant_column_pos.size()); for (size_t i = 0; i < variant_column_pos.size(); ++i) { - configs[i].enable_flatten_nested = tablet_schema.variant_flatten_nested(); + // Deprecated legacy flatten-nested switch. Distinct from variant_enable_nested_group. + configs[i].deprecated_enable_flatten_nested = + tablet_schema.deprecated_variant_flatten_nested(); const auto& column = tablet_schema.column(variant_schema_pos[i]); if (!column.is_variant_type()) { return Status::InternalError("column is not variant type, column name: {}", diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp index b2b6d803d79157..6d00beb8d2bdf8 100644 --- a/be/src/storage/tablet/tablet_meta.cpp +++ b/be/src/storage/tablet/tablet_meta.cpp @@ -364,6 +364,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id schema->set_disable_auto_compaction(tablet_schema.disable_auto_compaction); } + // Deprecated legacy flatten-nested switch. Distinct from variant_enable_nested_group. if (tablet_schema.__isset.variant_enable_flatten_nested) { schema->set_enable_variant_flatten_nested(tablet_schema.variant_enable_flatten_nested); } diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index ce95f1ca87019f..432552de37e8a1 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -1289,7 +1289,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _row_store_column_unique_ids.assign(schema.row_store_column_unique_ids().begin(), schema.row_store_column_unique_ids().end()); - _enable_variant_flatten_nested = schema.enable_variant_flatten_nested(); + _deprecated_enable_variant_flatten_nested = schema.enable_variant_flatten_nested(); if (schema.has_is_external_segment_column_meta_used()) { _is_external_segment_column_meta_used = schema.is_external_segment_column_meta_used(); } else { @@ -1370,7 +1370,8 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _row_store_page_size = ori_tablet_schema.row_store_page_size(); _storage_page_size = ori_tablet_schema.storage_page_size(); _storage_dict_page_size = ori_tablet_schema.storage_dict_page_size(); - _enable_variant_flatten_nested = ori_tablet_schema.variant_flatten_nested(); + _deprecated_enable_variant_flatten_nested = + ori_tablet_schema.deprecated_variant_flatten_nested(); // copy from table_schema_param _schema_version = version; @@ -1570,7 +1571,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format); tablet_schema_pb->mutable_row_store_column_unique_ids()->Assign( _row_store_column_unique_ids.begin(), _row_store_column_unique_ids.end()); - tablet_schema_pb->set_enable_variant_flatten_nested(_enable_variant_flatten_nested); + tablet_schema_pb->set_enable_variant_flatten_nested(_deprecated_enable_variant_flatten_nested); tablet_schema_pb->set_is_external_segment_column_meta_used( _is_external_segment_column_meta_used); tablet_schema_pb->set_integer_type_default_use_plain_encoding( @@ -1964,7 +1965,10 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) { if (a._storage_page_size != b._storage_page_size) return false; if (a._storage_dict_page_size != b._storage_dict_page_size) return false; if (a._skip_write_index_on_load != b._skip_write_index_on_load) return false; - if (a._enable_variant_flatten_nested != b._enable_variant_flatten_nested) return false; + if (a._deprecated_enable_variant_flatten_nested != + b._deprecated_enable_variant_flatten_nested) { + return false; + } if (a._is_external_segment_column_meta_used != b._is_external_segment_column_meta_used) return false; if (a._integer_type_default_use_plain_encoding != b._integer_type_default_use_plain_encoding) diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index 90d1c2d6d0a8c6..f5acd96f65a3f4 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -478,10 +478,14 @@ class TabletSchema : public MetadataAdder { _disable_auto_compaction = disable_auto_compaction; } bool disable_auto_compaction() const { return _disable_auto_compaction; } - void set_enable_variant_flatten_nested(bool flatten_nested) { - _enable_variant_flatten_nested = flatten_nested; + // Deprecated legacy switch for flatten-nested variant behavior. + // It is distinct from variant_enable_nested_group. + void set_deprecated_variant_flatten_nested(bool flatten_nested) { + _deprecated_enable_variant_flatten_nested = flatten_nested; + } + bool deprecated_variant_flatten_nested() const { + return _deprecated_enable_variant_flatten_nested; } - bool variant_flatten_nested() const { return _enable_variant_flatten_nested; } void set_enable_single_replica_compaction(bool enable_single_replica_compaction) { _enable_single_replica_compaction = enable_single_replica_compaction; } @@ -821,7 +825,7 @@ class TabletSchema : public MetadataAdder { // Contains column ids of which columns should be encoded into row store. // ATTN: For compability reason empty cids means all columns of tablet schema are encoded to row column std::vector _row_store_column_unique_ids; - bool _enable_variant_flatten_nested = false; + bool _deprecated_enable_variant_flatten_nested = false; std::map _vir_col_idx_to_unique_id; std::map _pruned_columns_data_type; diff --git a/be/src/util/json/json_parser.cpp b/be/src/util/json/json_parser.cpp index 0f6f7de6b349fa..921093105de701 100644 --- a/be/src/util/json/json_parser.cpp +++ b/be/src/util/json/json_parser.cpp @@ -44,9 +44,9 @@ std::optional JSONDataParser::parse(const char* begin, return {}; } ParseContext context; - // enable_flatten_nested controls nested path traversal + // deprecated_enable_flatten_nested controls nested path traversal // NestedGroup expansion is now handled at storage layer - context.enable_flatten_nested = config.enable_flatten_nested; + context.deprecated_enable_flatten_nested = config.deprecated_enable_flatten_nested; context.is_top_array = document.isArray(); traverse(document, context); ParseResult result; @@ -68,8 +68,8 @@ void JSONDataParser::traverse(const Element& element, ParseContext& // handled by VariantNestedBuilder with a max-depth guard. has_nested = false; check_has_nested_object(element); - ctx.has_nested_in_flatten = has_nested && ctx.enable_flatten_nested; - if (has_nested && !ctx.enable_flatten_nested) { + ctx.has_nested_in_flatten = has_nested && ctx.deprecated_enable_flatten_nested; + if (has_nested && !ctx.deprecated_enable_flatten_nested) { // Parse nested arrays to JsonbField JsonbWriter writer; traverseArrayAsJsonb(element.getArray(), writer); @@ -206,7 +206,7 @@ void JSONDataParser::traverseArrayElement(const Element& element, element_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten; element_ctx.is_top_array = ctx.is_top_array; traverse(element, element_ctx); - auto& [_, paths, values, flatten_nested, __, is_top_array] = element_ctx; + auto& [_, paths, values, deprecated_flatten_nested, __, is_top_array] = element_ctx; if (element_ctx.has_nested_in_flatten && is_top_array) { checkAmbiguousStructure(ctx, paths); diff --git a/be/src/util/json/json_parser.h b/be/src/util/json/json_parser.h index 92c2d135e33ff6..4b49259588b623 100644 --- a/be/src/util/json/json_parser.h +++ b/be/src/util/json/json_parser.h @@ -100,7 +100,7 @@ void writeValueAsJsonb(const Element& element, JsonbWriter& writer) { } struct ParseConfig { - bool enable_flatten_nested = false; + bool deprecated_enable_flatten_nested = false; enum class ParseTo { OnlySubcolumns = 0, OnlyDocValueColumn = 1, @@ -127,7 +127,7 @@ class JSONDataParser { PathInDataBuilder builder; std::vector paths; std::vector values; - bool enable_flatten_nested = false; + bool deprecated_enable_flatten_nested = false; bool has_nested_in_flatten = false; bool is_top_array = false; }; diff --git a/be/test/core/jsonb/json_parser_test.cpp b/be/test/core/jsonb/json_parser_test.cpp index 1251619a652b54..f8cfdf8e1626bd 100644 --- a/be/test/core/jsonb/json_parser_test.cpp +++ b/be/test/core/jsonb/json_parser_test.cpp @@ -139,7 +139,7 @@ TEST(JsonParserTest, ParseMultiLevelNestedArray) { EXPECT_EQ(result->paths.size(), 1); // Test complex nested structure - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; std::string json1 = R"({"a":[[1,2],[3],[4,5,6]]})"; // multi level nested array in object result = parser.parse(json1.c_str(), json1.size(), config); @@ -165,7 +165,7 @@ TEST(JsonParserTest, ParseMultiLevelNestedArray) { EXPECT_EQ(result->values[0].get_type(), doris::PrimitiveType::TYPE_JSONB); // test flatten nested - config.enable_flatten_nested = true; + config.deprecated_enable_flatten_nested = true; // TODO: checkAmbiguousStructure is only called when has_nested_in_flatten && is_top_array. // These JSONs are objects (not top-level arrays), so is_top_array=false and the check is skipped. // EXPECT_ANY_THROW(parser.parse(json.c_str(), json.size(), config)); @@ -184,14 +184,14 @@ TEST(JsonParserTest, ParseMultiLevelNestedArray) { TEST(JsonParserTest, ParseNestedAndFlatten) { JSONDataParser parser; ParseConfig config; - config.enable_flatten_nested = true; + config.deprecated_enable_flatten_nested = true; std::string json = R"({"a":[{"b":1},{"b":2}]})"; auto result = parser.parse(json.c_str(), json.size(), config); ASSERT_TRUE(result.has_value()); EXPECT_GT(result->values.size(), 0); - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; std::string json2 = R"({"a":[{"b":1},{"b":2}]})"; result = parser.parse(json2.c_str(), json2.size(), config); ASSERT_TRUE(result.has_value()); @@ -249,7 +249,7 @@ TEST(JsonParserTest, TestIsPrefixFunction) { TEST(JsonParserTest, TestAmbiguousStructureDetection) { JSONDataParser parser; ParseConfig config; - config.enable_flatten_nested = true; + config.deprecated_enable_flatten_nested = true; // TODO: The following 3 cases no longer throw because checkAmbiguousStructure requires // has_nested_in_flatten && is_top_array. "b" contains plain arrays (not nested objects), @@ -276,7 +276,7 @@ TEST(JsonParserTest, TestAmbiguousStructureDetection) { TEST(JsonParserTest, TestNestedArrayHandling) { JSONDataParser parser; ParseConfig config; - config.enable_flatten_nested = true; + config.deprecated_enable_flatten_nested = true; // Test case 1: Simple nested array handling std::string json1 = R"([{"b": 1}, {"c": 2}])"; @@ -296,7 +296,7 @@ TEST(JsonParserTest, TestNestedArrayWithDifferentConfigs) { // Test with flatten_nested = false ParseConfig config1; - config1.enable_flatten_nested = false; + config1.deprecated_enable_flatten_nested = false; std::string json1 = R"([{"b": [1, 2]}, {"b": [3, 4]}])"; auto result1 = parser.parse(json1.c_str(), json1.size(), config1); @@ -306,7 +306,7 @@ TEST(JsonParserTest, TestNestedArrayWithDifferentConfigs) { // Test with flatten_nested = true ParseConfig config2; - config2.enable_flatten_nested = true; + config2.deprecated_enable_flatten_nested = true; // TODO: "b" contains plain arrays (no nested objects), so has_nested=false, // has_nested_in_flatten=false, and checkAmbiguousStructure is not called. @@ -426,7 +426,7 @@ TEST(JsonParserTest, ParseUInt64) { EXPECT_EQ(array_field[0].get(), 18446744073709551615ULL); std::string nested_json = R"({"a": [{"b": 18446744073709551615}]})"; - config.enable_flatten_nested = true; + config.deprecated_enable_flatten_nested = true; result = parser.parse(nested_json.c_str(), nested_json.size(), config); ASSERT_TRUE(result.has_value()); EXPECT_EQ(result->values.size(), 1); @@ -458,7 +458,7 @@ TEST(JsonParserTest, KeyLengthLimitByConfig) { std::string obj_json = "{\"" + key11 + "\": 1}"; EXPECT_ANY_THROW(parser.parse(obj_json.c_str(), obj_json.size(), config)); - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; std::string jsonb_json = "{\"a\": [{\"" + key11 + "\": 1}]}"; EXPECT_ANY_THROW(parser.parse(jsonb_json.c_str(), jsonb_json.size(), config)); } @@ -471,7 +471,7 @@ TEST(JsonParserTest, KeyLengthLimitByConfig) { auto result = parser.parse(obj_json.c_str(), obj_json.size(), config); ASSERT_TRUE(result.has_value()); - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; std::string jsonb_json = "{\"a\": [{\"" + key255 + "\": 1}]}"; result = parser.parse(jsonb_json.c_str(), jsonb_json.size(), config); ASSERT_TRUE(result.has_value()); diff --git a/be/test/exec/common/schema_util_rowset_test.cpp b/be/test/exec/common/schema_util_rowset_test.cpp index a12d2429b8dc90..aa9c2fd13cdc3e 100644 --- a/be/test/exec/common/schema_util_rowset_test.cpp +++ b/be/test/exec/common/schema_util_rowset_test.cpp @@ -143,7 +143,7 @@ static void fill_varaint_column(auto& variant_column, int size, int uid) { auto column_string = assert_cast(column.get()); fill_string_column_with_test_data(column_string, size, uid); ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; variant_util::parse_json_to_variant(*variant_column, *column_string, config); } diff --git a/be/test/exec/common/schema_util_test.cpp b/be/test/exec/common/schema_util_test.cpp index 70b5823c5c945a..1a9b4617c52fb5 100644 --- a/be/test/exec/common/schema_util_test.cpp +++ b/be/test/exec/common/schema_util_test.cpp @@ -1876,7 +1876,7 @@ TEST_F(SchemaUtilTest, parse_and_materialize_variant_columns_ambiguous_paths) { // The variant column is at index 0 std::vector variant_pos = {0}; ParseConfig config; - config.enable_flatten_nested = true; + config.deprecated_enable_flatten_nested = true; // Should throw due to ambiguous paths Status st = variant_util::parse_and_materialize_variant_columns(block, variant_pos, {config}); diff --git a/be/test/storage/segment/variant_column_writer_reader_test.cpp b/be/test/storage/segment/variant_column_writer_reader_test.cpp index d341f2d1ba2359..f690bf74f46f64 100644 --- a/be/test/storage/segment/variant_column_writer_reader_test.cpp +++ b/be/test/storage/segment/variant_column_writer_reader_test.cpp @@ -219,7 +219,7 @@ static void fill_variant_column_with_doc_value_only( VariantUtil::fill_string_column_with_test_data(column_string, num_rows, inserted); ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; config.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; variant_util::parse_json_to_variant(*column_object, *column_string, config); } @@ -1340,7 +1340,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_compact_writer_and_read_doc } ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; config.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; MutableColumnPtr root_variant = @@ -1515,7 +1515,7 @@ TEST_F(VariantColumnWriterReaderTest, test_doc_compact_sparse_write_array_gap) { strings->insert_data(row1.data(), row1.size()); ParseConfig parse_cfg; - parse_cfg.enable_flatten_nested = false; + parse_cfg.deprecated_enable_flatten_nested = false; parse_cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; MutableColumnPtr bucket_variant = @@ -1616,7 +1616,7 @@ TEST_F(VariantColumnWriterReaderTest, test_write_doc_sparse_write_array_gap_and_ strings->insert_data(inserted_json[1].data(), inserted_json[1].size()); ParseConfig parse_cfg; - parse_cfg.enable_flatten_nested = false; + parse_cfg.deprecated_enable_flatten_nested = false; parse_cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; MutableColumnPtr variant_column = @@ -2680,7 +2680,7 @@ TEST_F(VariantColumnWriterReaderTest, test_no_sub_in_sparse_column) { } ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; variant_util::parse_json_to_variant(*column_object, *column_string, config); std::cout << "column_object size: " << assert_cast(column_object.get())->debug_string() << std::endl; @@ -2825,7 +2825,7 @@ TEST_F(VariantColumnWriterReaderTest, test_prefix_in_sub_and_sparse) { } ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; variant_util::parse_json_to_variant(*column_object, *column_string, config); std::cout << "column_object size: " << assert_cast(column_object.get())->debug_string() << std::endl; @@ -3312,7 +3312,7 @@ TEST_F(VariantColumnWriterReaderTest, test_read_with_checksum) { fill_string_column_with_test_data(column_string, size, inserted_jsonstr, path_with_size); ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; variant_util::parse_json_to_variant(*column_object, *column_string, config); }; diff --git a/be/test/storage/segment/variant_util_test.cpp b/be/test/storage/segment/variant_util_test.cpp index daa6d0b2a0e78f..53e9ede840cb73 100644 --- a/be/test/storage/segment/variant_util_test.cpp +++ b/be/test/storage/segment/variant_util_test.cpp @@ -53,7 +53,7 @@ TEST(VariantUtilTest, ParseDocValueToSubcolumns_FillsDefaultsAndValues) { auto json_col = _make_json_column(jsons); ParseConfig cfg; - cfg.enable_flatten_nested = false; + cfg.deprecated_enable_flatten_nested = false; cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; parse_json_to_variant(*variant, *json_col, cfg); @@ -105,7 +105,7 @@ TEST(VariantUtilTest, ParseOnlyDocValueColumn_SerializesMixedTypes) { auto json_col = _make_json_column(jsons); ParseConfig cfg; - cfg.enable_flatten_nested = false; + cfg.deprecated_enable_flatten_nested = false; cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; parse_json_to_variant(*variant, *json_col, cfg); @@ -222,7 +222,7 @@ TEST(VariantUtilTest, ParseVariantColumns_DocModeBinaryToSubcolumns) { auto variant = ColumnVariant::create(0); auto json_col = _make_json_column(jsons); ParseConfig cfg; - cfg.enable_flatten_nested = false; + cfg.deprecated_enable_flatten_nested = false; cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; parse_json_to_variant(*variant, *json_col, cfg); ASSERT_TRUE(variant->is_doc_mode()); @@ -231,7 +231,7 @@ TEST(VariantUtilTest, ParseVariantColumns_DocModeBinaryToSubcolumns) { block.insert({variant->get_ptr(), std::make_shared(0), "v"}); ParseConfig parse_cfg; - parse_cfg.enable_flatten_nested = false; + parse_cfg.deprecated_enable_flatten_nested = false; parse_cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; Status st = parse_and_materialize_variant_columns(block, std::vector {0}, {parse_cfg}); @@ -275,7 +275,7 @@ TEST(VariantUtilTest, ParseVariantColumns_DocModeRejectOnlySubcolumnsConfig) { auto json_col = _make_json_column(jsons); ParseConfig cfg; - cfg.enable_flatten_nested = false; + cfg.deprecated_enable_flatten_nested = false; cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; parse_json_to_variant(*variant, *json_col, cfg); ASSERT_TRUE(variant->is_doc_mode()); @@ -284,7 +284,7 @@ TEST(VariantUtilTest, ParseVariantColumns_DocModeRejectOnlySubcolumnsConfig) { block.insert({variant->get_ptr(), std::make_shared(0), "v"}); ParseConfig parse_cfg; - parse_cfg.enable_flatten_nested = false; + parse_cfg.deprecated_enable_flatten_nested = false; parse_cfg.parse_to = ParseConfig::ParseTo::OnlyDocValueColumn; Status st = parse_and_materialize_variant_columns(block, std::vector {0}, {parse_cfg}); diff --git a/be/test/testutil/variant_util.h b/be/test/testutil/variant_util.h index a15542cce9a97a..9d3cdea9e84fac 100644 --- a/be/test/testutil/variant_util.h +++ b/be/test/testutil/variant_util.h @@ -303,7 +303,7 @@ class VariantUtil { auto column_string = assert_cast(column.get()); auto res = fill_string_column_with_test_data(column_string, size, inserted_jsonstr); ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; variant_util::parse_json_to_variant(*column_object, *column_string, config); return res; } @@ -356,7 +356,7 @@ class VariantUtil { assert(column_string->size() == size); ParseConfig config; // do not treat array with jsonb field - config.enable_flatten_nested = has_nested; + config.deprecated_enable_flatten_nested = has_nested; variant_util::parse_json_to_variant(*variant_column, *column_string, config); } @@ -367,7 +367,7 @@ class VariantUtil { auto column_string = assert_cast(column.get()); auto res = fill_string_column_with_nested_test_data(column_string, size, inserted_jsonstr); ParseConfig config; - config.enable_flatten_nested = false; + config.deprecated_enable_flatten_nested = false; variant_util::parse_json_to_variant(*column_object, *column_string, config); return res; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index b0200c707fcdb5..ba1b8d3c715739 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2618,6 +2618,7 @@ public Boolean disableAutoCompaction() { return false; } + @Deprecated public void setVariantEnableFlattenNested(boolean flattenNested) throws AnalysisException { TableProperty tableProperty = getOrCreatTableProperty(); tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED, @@ -2625,6 +2626,7 @@ public void setVariantEnableFlattenNested(boolean flattenNested) throws Analysis tableProperty.buildVariantEnableFlattenNested(); } + @Deprecated public Boolean variantEnableFlattenNested() { if (tableProperty != null) { return tableProperty.variantEnableFlattenNested(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index c260fbb99c32c0..7ada8286d9878d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -327,11 +327,22 @@ public boolean disableAutoCompaction() { } public TableProperty buildVariantEnableFlattenNested() { + migrateDeprecatedVariantEnableFlattenNestedProperty(); variantEnableFlattenNested = Boolean.parseBoolean( properties.getOrDefault(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED, "false")); return this; } + private void migrateDeprecatedVariantEnableFlattenNestedProperty() { + if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED) + && properties.containsKey(PropertyAnalyzer.LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED)) { + properties.put(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED, + properties.remove(PropertyAnalyzer.LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED)); + return; + } + properties.remove(PropertyAnalyzer.LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED); + } + public boolean variantEnableFlattenNested() { return variantEnableFlattenNested; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index c6a8f0df8db20e..76e447b6a1090d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -157,7 +157,14 @@ public class PropertyAnalyzer { public static final String PROPERTIES_DISABLE_AUTO_COMPACTION = "disable_auto_compaction"; - public static final String PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED = "variant_enable_flatten_nested"; + // Legacy persisted switch for flatten-nested variant behavior before it was deprecated. + @Deprecated + public static final String LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED = "variant_enable_flatten_nested"; + + // Deprecated legacy switch for flatten-nested variant behavior. + // It is distinct from variant_enable_nested_group. + @Deprecated + public static final String PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED = "deprecated_variant_enable_flatten_nested"; public static final String PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION = "enable_single_replica_compaction"; @@ -835,6 +842,7 @@ public static Boolean analyzeDisableAutoCompaction(Map propertie + " must be `true` or `false`"); } + @Deprecated public static Boolean analyzeVariantFlattenNested(Map properties) throws AnalysisException { if (properties == null || properties.isEmpty()) { return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index d353520ae63d6c..cbbc3dd1797242 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2492,7 +2492,8 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th boolean variantEnableFlattenNested = false; try { variantEnableFlattenNested = PropertyAnalyzer.analyzeVariantFlattenNested(properties); - // only if session variable: enable_variant_flatten_nested = true and + // Deprecated legacy flatten-nested switches. + // Only if session variable: enable_variant_flatten_nested = true and // table property: variant_enable_flatten_nested = true // we can enable variant flatten nested otherwise throw error if (ctx != null && ctx.getSessionVariable().getEnableVariantFlattenNested() diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 67e45b7d146886..8dd29586e77c41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -808,8 +808,10 @@ public class SessionVariable implements Serializable, Writable { public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant"; - // enable variant flatten nested as session variable, default is false, - // which means do not flatten nested when create table + // Deprecated legacy switch for flatten-nested variant behavior. + // It is distinct from variant_enable_nested_group. + // Default is false, which means do not flatten nested when create table. + @Deprecated public static final String ENABLE_VARIANT_FLATTEN_NESTED = "enable_variant_flatten_nested"; public static final String ENABLE_VARIANT_SCHEMA_AUTO_CAST = "enable_variant_schema_auto_cast"; @@ -1723,6 +1725,7 @@ public static boolean getTopNLazyMaterializationUsingIndex() { @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT, needForward = true) private boolean disableInvertedIndexV1ForVaraint = true; + @Deprecated @VariableMgr.VarAttr(name = ENABLE_VARIANT_FLATTEN_NESTED, needForward = true) private boolean enableVariantFlattenNested = false; @@ -6221,10 +6224,12 @@ public boolean getDisableInvertedIndexV1ForVaraint() { return disableInvertedIndexV1ForVaraint; } + @Deprecated public void setEnableVariantFlattenNested(boolean enableVariantFlattenNested) { this.enableVariantFlattenNested = enableVariantFlattenNested; } + @Deprecated public boolean getEnableVariantFlattenNested() { return enableVariantFlattenNested; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index 42f104371e818e..881c79865d7268 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -25,6 +25,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.FastByteArrayOutputStream; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.UnitTestUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -139,6 +140,21 @@ public void testResetPropertiesForRestore() { Assert.assertEquals((short) 3, olapTable.getDefaultReplicaAllocation().getTotalReplicaNum()); } + @Test + public void testBuildVariantEnableFlattenNestedWithLegacyPropertyKey() throws IOException { + Map properties = Maps.newHashMap(); + properties.put(PropertyAnalyzer.LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED, "true"); + + TableProperty tableProperty = new TableProperty(properties); + tableProperty.gsonPostProcess(); + + Assert.assertTrue(tableProperty.variantEnableFlattenNested()); + Assert.assertEquals("true", + tableProperty.getProperties().get(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED)); + Assert.assertFalse( + tableProperty.getProperties().containsKey(PropertyAnalyzer.LEGACY_PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED)); + } + @Test public void testGetPartitionRowCount() { OlapTable olapTable = new OlapTable(); diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out index 9f6e334cc9abdf..497f300612039f 100644 --- a/regression-test/data/query_p0/system/test_table_properties.out +++ b/regression-test/data/query_p0/system/test_table_properties.out @@ -12,6 +12,7 @@ internal test_table_properties_db duplicate_table compaction_policy size_based internal test_table_properties_db duplicate_table compression ZSTD internal test_table_properties_db duplicate_table data_sort.col_num 3 internal test_table_properties_db duplicate_table data_sort.sort_type LEXICAL +internal test_table_properties_db duplicate_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db duplicate_table disable_auto_compaction false internal test_table_properties_db duplicate_table enable_mow_light_delete false internal test_table_properties_db duplicate_table enable_single_replica_compaction false @@ -38,7 +39,6 @@ internal test_table_properties_db duplicate_table time_series_compaction_file_co internal test_table_properties_db duplicate_table time_series_compaction_goal_size_mbytes 1024 internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1 internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600 -internal test_table_properties_db duplicate_table variant_enable_flatten_nested false internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5 internal test_table_properties_db listtable _auto_bucket false internal test_table_properties_db listtable binlog.enable false @@ -49,6 +49,7 @@ internal test_table_properties_db listtable compaction_policy size_based internal test_table_properties_db listtable compression ZSTD internal test_table_properties_db listtable data_sort.col_num 6 internal test_table_properties_db listtable data_sort.sort_type LEXICAL +internal test_table_properties_db listtable deprecated_variant_enable_flatten_nested false internal test_table_properties_db listtable disable_auto_compaction false internal test_table_properties_db listtable enable_mow_light_delete false internal test_table_properties_db listtable enable_single_replica_compaction false @@ -75,7 +76,6 @@ internal test_table_properties_db listtable time_series_compaction_file_count_th internal test_table_properties_db listtable time_series_compaction_goal_size_mbytes 1024 internal test_table_properties_db listtable time_series_compaction_level_threshold 1 internal test_table_properties_db listtable time_series_compaction_time_threshold_seconds 3600 -internal test_table_properties_db listtable variant_enable_flatten_nested false internal test_table_properties_db listtable vertical_compaction_num_columns_per_group 5 internal test_table_properties_db unique_table _auto_bucket false internal test_table_properties_db unique_table binlog.enable false @@ -86,6 +86,7 @@ internal test_table_properties_db unique_table compaction_policy size_based internal test_table_properties_db unique_table compression ZSTD internal test_table_properties_db unique_table data_sort.col_num 2 internal test_table_properties_db unique_table data_sort.sort_type LEXICAL +internal test_table_properties_db unique_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db unique_table disable_auto_compaction false internal test_table_properties_db unique_table enable_mow_light_delete false internal test_table_properties_db unique_table enable_single_replica_compaction false @@ -112,7 +113,6 @@ internal test_table_properties_db unique_table time_series_compaction_file_count internal test_table_properties_db unique_table time_series_compaction_goal_size_mbytes 1024 internal test_table_properties_db unique_table time_series_compaction_level_threshold 1 internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600 -internal test_table_properties_db unique_table variant_enable_flatten_nested false internal test_table_properties_db unique_table vertical_compaction_num_columns_per_group 5 -- !select_check_3 -- @@ -125,6 +125,7 @@ internal test_table_properties_db duplicate_table compaction_policy size_based internal test_table_properties_db duplicate_table compression ZSTD internal test_table_properties_db duplicate_table data_sort.col_num 3 internal test_table_properties_db duplicate_table data_sort.sort_type LEXICAL +internal test_table_properties_db duplicate_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db duplicate_table disable_auto_compaction false internal test_table_properties_db duplicate_table enable_mow_light_delete false internal test_table_properties_db duplicate_table enable_single_replica_compaction false @@ -151,7 +152,6 @@ internal test_table_properties_db duplicate_table time_series_compaction_file_co internal test_table_properties_db duplicate_table time_series_compaction_goal_size_mbytes 1024 internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1 internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600 -internal test_table_properties_db duplicate_table variant_enable_flatten_nested false internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5 internal test_table_properties_db unique_table _auto_bucket false internal test_table_properties_db unique_table binlog.enable false @@ -162,6 +162,7 @@ internal test_table_properties_db unique_table compaction_policy size_based internal test_table_properties_db unique_table compression ZSTD internal test_table_properties_db unique_table data_sort.col_num 2 internal test_table_properties_db unique_table data_sort.sort_type LEXICAL +internal test_table_properties_db unique_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db unique_table disable_auto_compaction false internal test_table_properties_db unique_table enable_mow_light_delete false internal test_table_properties_db unique_table enable_single_replica_compaction false @@ -188,7 +189,6 @@ internal test_table_properties_db unique_table time_series_compaction_file_count internal test_table_properties_db unique_table time_series_compaction_goal_size_mbytes 1024 internal test_table_properties_db unique_table time_series_compaction_level_threshold 1 internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600 -internal test_table_properties_db unique_table variant_enable_flatten_nested false internal test_table_properties_db unique_table vertical_compaction_num_columns_per_group 5 -- !select_check_4 -- @@ -203,6 +203,7 @@ internal test_table_properties_db duplicate_table compaction_policy size_based internal test_table_properties_db duplicate_table compression ZSTD internal test_table_properties_db duplicate_table data_sort.col_num 3 internal test_table_properties_db duplicate_table data_sort.sort_type LEXICAL +internal test_table_properties_db duplicate_table deprecated_variant_enable_flatten_nested false internal test_table_properties_db duplicate_table disable_auto_compaction false internal test_table_properties_db duplicate_table enable_mow_light_delete false internal test_table_properties_db duplicate_table enable_single_replica_compaction false @@ -229,7 +230,6 @@ internal test_table_properties_db duplicate_table time_series_compaction_file_co internal test_table_properties_db duplicate_table time_series_compaction_goal_size_mbytes 1024 internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1 internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600 -internal test_table_properties_db duplicate_table variant_enable_flatten_nested false internal test_table_properties_db duplicate_table vertical_compaction_num_columns_per_group 5 -- !select_check_6 -- diff --git a/regression-test/suites/variant_doc_mode_p2/load.groovy b/regression-test/suites/variant_doc_mode_p2/load.groovy index 6dc40484b2afe4..e9a926f380e12d 100644 --- a/regression-test/suites/variant_doc_mode_p2/load.groovy +++ b/regression-test/suites/variant_doc_mode_p2/load.groovy @@ -92,7 +92,7 @@ suite("test_doc_value_p2", "nonConcurrent,p2"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "false", "inverted_index_storage_format"= "v2"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "false", "inverted_index_storage_format"= "v2"); """ // 2015 load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") @@ -154,7 +154,7 @@ suite("test_doc_value_p2", "nonConcurrent,p2"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "false", "bloom_filter_columns" = "v"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "false", "bloom_filter_columns" = "v"); """ sql """insert into github_events2 select * from github_events order by k""" sql """select v['payload']['commits'] from github_events order by k ;""" diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy index 5fca8a07016aec..0bf08c0633ced4 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy @@ -85,7 +85,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "true", "inverted_index_storage_format"= "v2"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "true", "inverted_index_storage_format"= "v2"); """ // 2015 load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") @@ -147,7 +147,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true", "bloom_filter_columns" = "v"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "true", "bloom_filter_columns" = "v"); """ sql """insert into github_events2 select * from github_events order by k""" sql """select v['payload']['commits'] from github_events order by k ;""" diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy index f53ed7d1b425db..cfe29e2e682189 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy @@ -83,7 +83,7 @@ suite("regression_test_variant_github_events_upgrade_p2", "nonConcurrent,p2"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "true", "bloom_filter_columns" = "v", "variant_enable_flatten_nested" = "true", "inverted_index_storage_format"= "v2", "enable_unique_key_merge_on_write" = "false"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "bloom_filter_columns" = "v", "deprecated_variant_enable_flatten_nested" = "true", "inverted_index_storage_format"= "v2", "enable_unique_key_merge_on_write" = "false"); """ // 2015 load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy index e01aa8fd52dc90..7d01312953ddd7 100644 --- a/regression-test/suites/variant_p0/delete_update.groovy +++ b/regression-test/suites/variant_p0/delete_update.groovy @@ -29,7 +29,7 @@ suite("regression_test_variant_delete_and_update", "variant_type"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 3 - properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "false", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "false", "deprecated_variant_enable_flatten_nested" = "true"); """ // test mor table diff --git a/regression-test/suites/variant_p0/doc_mode/delete_update.groovy b/regression-test/suites/variant_p0/doc_mode/delete_update.groovy index 992dd0b114ac6a..340ea9a7edf769 100644 --- a/regression-test/suites/variant_p0/doc_mode/delete_update.groovy +++ b/regression-test/suites/variant_p0/doc_mode/delete_update.groovy @@ -30,7 +30,7 @@ suite("variant_delete_and_update_doc_value", "p0"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 3 - properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "false", "variant_enable_flatten_nested" = "false"); + properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "false", "deprecated_variant_enable_flatten_nested" = "false"); """ // test mor table diff --git a/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy b/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy index 1e5dd1fcaf639c..f4bde164b4367a 100644 --- a/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy +++ b/regression-test/suites/variant_p0/doc_mode/predefine/load.groovy @@ -60,7 +60,7 @@ suite("test_variant_predefine_doc_value", "nonConcurrent"){ `v1` variant NULL, INDEX idx_var_sub(`v1`) USING INVERTED PROPERTIES("parser" = "english") ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 2 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "variant_enable_flatten_nested" = "true"); + PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "deprecated_variant_enable_flatten_nested" = "true"); """ sql """insert into test_predefine1 values(1, '{"predefine_col1" : 1024}')""" sql """insert into test_predefine1 values(2, '{"predefine_col2" : 1.11111}')""" @@ -219,7 +219,7 @@ suite("test_variant_predefine_doc_value", "nonConcurrent"){ `id` bigint NOT NULL, `v` variant<'nested.a':string> NULL) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "variant_enable_flatten_nested" = "false", "disable_auto_compaction" = "true");""" + PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "deprecated_variant_enable_flatten_nested" = "false", "disable_auto_compaction" = "true");""" // test alter nested no effect at present sql "truncate table test_predefine3" diff --git a/regression-test/suites/variant_p0/doc_mode/schema_change/test_double_write_when_schema_change.groovy b/regression-test/suites/variant_p0/doc_mode/schema_change/test_double_write_when_schema_change.groovy index a063e13b7431db..4619f0aae8e425 100644 --- a/regression-test/suites/variant_p0/doc_mode/schema_change/test_double_write_when_schema_change.groovy +++ b/regression-test/suites/variant_p0/doc_mode/schema_change/test_double_write_when_schema_change.groovy @@ -68,7 +68,7 @@ suite("double_write_schema_change_doc_value", "nonConcurrent") { ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 2 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "false"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "false"); """ set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296") diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy index ea685a86041e86..f3786f608a62c5 100644 --- a/regression-test/suites/variant_p0/nested.groovy +++ b/regression-test/suites/variant_p0/nested.groovy @@ -36,7 +36,7 @@ suite("regression_test_variant_nested", "p0"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "true"); """ sql """ insert into var_nested values (1, '{"xx" : 10}'); @@ -118,7 +118,7 @@ parallel_pipeline_task_num=7,profile_level=1,enable_pipeline_engine=true,enable_ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "deprecated_variant_enable_flatten_nested" = "true"); """ sql """insert into var_nested2 select * from var_nested order by k limit 1024""" qt_sql """select /*+SET_VAR(batch_size=4064,broker_load_batch_size=16352,disable_streaming_preaggregations=true,enable_distinct_streaming_aggregation=true,parallel_pipeline_task_num=1,profile_level=1,enable_parallel_scan=true,parallel_scan_max_scanners_count=48,parallel_scan_min_rows_per_scanner=16384,enable_fold_constant_by_be=true,enable_rewrite_element_at_to_slot=true,runtime_filter_type=12,enable_parallel_result_sink=false,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=false,enable_local_exchange=false,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_common_expr_pushdown_for_inverted_index=true,spill_min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,max_fetch_remote_schema_tablet_count=512,enable_spill=false,enable_force_spill=false,data_queue_max_blocks=1,spill_streaming_agg_mem_limit=268435456,spill_aggregation_partition_count=5) */ * from var_nested2 order by k limit 10;""" @@ -140,7 +140,7 @@ where phone_numbers['type'] = 'GSM' OR phone_numbers['type'] = 'HOME' and phone_ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "deprecated_variant_enable_flatten_nested" = "true"); """ sql "insert into var_nested_array_agg select * from var_nested" // 1. array_contains @@ -157,7 +157,7 @@ where phone_numbers['type'] = 'GSM' OR phone_numbers['type'] = 'HOME' and phone_ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "true", "deprecated_variant_enable_flatten_nested" = "true"); """ sql "insert into var_nested_explode_variant_with_abnomal select * from var_nested" // 1. v['nested']['x'] is null root diff --git a/regression-test/suites/variant_p0/nested/load.groovy b/regression-test/suites/variant_p0/nested/load.groovy index f1c5b6121d0f39..ee193fab61bf9f 100644 --- a/regression-test/suites/variant_p0/nested/load.groovy +++ b/regression-test/suites/variant_p0/nested/load.groovy @@ -36,7 +36,7 @@ // ) // DUPLICATE KEY(`k`) // DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case -// properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "true"); +// properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "true"); // """ // exception "If you want to enable variant flatten nested, please set session variable" // } @@ -51,7 +51,7 @@ // ) // DUPLICATE KEY(`k`) // DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case -// properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "true"); +// properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "true"); // """ // sql """ insert into ${table_name} values (1, '{"nested": [{"a": 1, "c": 1.1}, {"b": "1"}]}'); """ // @@ -108,7 +108,7 @@ // ) // DUPLICATE KEY(`k`) // DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case -// properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "true"); +// properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "true"); // """ // // insert a array of object for a, b, c first then insert structure conflict in one row // // insert structure conflict in one row @@ -153,7 +153,7 @@ // // // drop table // sql """ drop table ${table_name_1} """ -// sql """ create table ${table_name_1} (k bigint, v variant) duplicate key(k) distributed by hash(k) buckets 1 properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "true") """ +// sql """ create table ${table_name_1} (k bigint, v variant) duplicate key(k) distributed by hash(k) buckets 1 properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "true") """ // // insert scalar data first then insert structure conflict in one row // sql """ // insert into ${table_name_1} values (1, '{"nested": {"a": 2.5, "b": "123.1"}}'); diff --git a/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy b/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy index f8edc713c2af4c..9c75f20dcb5e46 100644 --- a/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy +++ b/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy @@ -39,7 +39,7 @@ suite("nested_in_top_array", "p0"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case - properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "true"); """ sql """ insert into ${table_name} values (1, '[{"a": 1, "c": 1.1}, {"b": "1"}]'); """ diff --git a/regression-test/suites/variant_p0/nested2.groovy b/regression-test/suites/variant_p0/nested2.groovy index 05b6be2027655b..e1b9d492887a28 100644 --- a/regression-test/suites/variant_p0/nested2.groovy +++ b/regression-test/suites/variant_p0/nested2.groovy @@ -37,7 +37,7 @@ suite("variant_nested_type_conflict", "p0"){ // ) // DUPLICATE KEY(`k`) // DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case - // properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + // properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "true"); // """ // def sql_select_batch = { // qt_sql_0 """select * from ${table_name} order by k""" diff --git a/regression-test/suites/variant_p0/predefine/delete_update.groovy b/regression-test/suites/variant_p0/predefine/delete_update.groovy index 8f62d8cdf9e810..e74482875afa9a 100644 --- a/regression-test/suites/variant_p0/predefine/delete_update.groovy +++ b/regression-test/suites/variant_p0/predefine/delete_update.groovy @@ -30,7 +30,7 @@ suite("regression_test_variant_predefine_delete_and_update", "variant_type"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 3 - properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "false", "variant_enable_flatten_nested" = "false", "disable_auto_compaction" = "true"); + properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "false", "deprecated_variant_enable_flatten_nested" = "false", "disable_auto_compaction" = "true"); """ // test mor table diff --git a/regression-test/suites/variant_p0/predefine/load.groovy b/regression-test/suites/variant_p0/predefine/load.groovy index 099e07cdbef5df..a2fb8b9ea4b7fc 100644 --- a/regression-test/suites/variant_p0/predefine/load.groovy +++ b/regression-test/suites/variant_p0/predefine/load.groovy @@ -63,7 +63,7 @@ suite("regression_test_variant_predefine_schema", "p0"){ `v1` variant NULL, INDEX idx_var_sub(`v1`) USING INVERTED PROPERTIES("parser" = "english") ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 2 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "variant_enable_flatten_nested" = "true"); + PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "deprecated_variant_enable_flatten_nested" = "true"); """ sql """insert into test_predefine1 values(1, '{"predefine_col1" : 1024}')""" sql """insert into test_predefine1 values(2, '{"predefine_col2" : 1.11111}')""" @@ -222,7 +222,7 @@ suite("regression_test_variant_predefine_schema", "p0"){ `id` bigint NOT NULL, `v` variant<'nested.a':string, properties("variant_enable_doc_mode" = "false")> NULL) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "variant_enable_flatten_nested" = "false", "disable_auto_compaction" = "true");""" + PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "deprecated_variant_enable_flatten_nested" = "false", "disable_auto_compaction" = "true");""" // test alter nested no effect at present sql "truncate table test_predefine3" diff --git a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy index 725bf1b5f00ba5..3c1932390de89b 100644 --- a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy +++ b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy @@ -67,7 +67,7 @@ suite("double_write_schema_change_with_variant", "nonConcurrent") { ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 2 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false", "deprecated_variant_enable_flatten_nested" = "true"); """ set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296") diff --git a/regression-test/suites/variant_p0/test_types_in_variant.groovy b/regression-test/suites/variant_p0/test_types_in_variant.groovy index b258035db122d9..373d90dba6190b 100644 --- a/regression-test/suites/variant_p0/test_types_in_variant.groovy +++ b/regression-test/suites/variant_p0/test_types_in_variant.groovy @@ -81,7 +81,7 @@ suite("regression_test_variant_types", "var_view") { ) engine = olap duplicate key (id) distributed by hash(id) buckets 1 - properties ("replication_num" = "1", "variant_enable_flatten_nested" = "true") + properties ("replication_num" = "1", "deprecated_variant_enable_flatten_nested" = "true") """ sql """ set enable_variant_flatten_nested = false """ diff --git a/regression-test/suites/variant_p1/doc_snapshot/load.groovy b/regression-test/suites/variant_p1/doc_snapshot/load.groovy index cca5fae70bcf34..8db8e0b9731869 100644 --- a/regression-test/suites/variant_p1/doc_snapshot/load.groovy +++ b/regression-test/suites/variant_p1/doc_snapshot/load.groovy @@ -76,7 +76,7 @@ suite("predefine_type_multi_index_doc_value", "p1"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "false"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "false"); """ // 2015 diff --git a/regression-test/suites/variant_p1/predefine/load.groovy b/regression-test/suites/variant_p1/predefine/load.groovy index ffc710454b43d2..2809f5ed280a75 100644 --- a/regression-test/suites/variant_p1/predefine/load.groovy +++ b/regression-test/suites/variant_p1/predefine/load.groovy @@ -77,7 +77,7 @@ suite("test_predefine_type_multi_index", "p1"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "true", "variant_enable_flatten_nested" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "deprecated_variant_enable_flatten_nested" = "true"); """ // 2015