Skip to content

Commit 12d88d9

Browse files
vikramahuja1001vikramahuja1001
authored
HIVE-29093 : ALTER TABLE ADD PARTITION fails if the DEFAULT_PARTITION_NAME property is a substring of partition value to be added (#5977)
1 parent 392e732 commit 12d88d9

File tree

6 files changed

+154
-19
lines changed

6 files changed

+154
-19
lines changed

‎ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -56,30 +56,42 @@ private PartitionUtils() {
5656
}
5757

5858
/**
59-
* Certain partition values are are used by hive. e.g. the default partition in dynamic partitioning and the
59+
* Certain partition values are used by hive. e.g. the default partition in dynamic partitioning and the
6060
* intermediate partition values used in the archiving process. Naturally, prohibit the user from creating partitions
6161
* with these reserved values. The check that this function is more restrictive than the actual limitation, but it's
6262
* simpler. Should be okay since the reserved names are fairly long and uncommon.
6363
*/
64-
public static void validatePartitions(HiveConf conf, Map<String, String> partitionSpec) throws SemanticException {
65-
Set<String> reservedPartitionValues = new HashSet<>();
64+
public static void validatePartitions(HiveConf conf, Map<String, String> partitionSpec) {
6665
// Partition can't have this name
67-
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.DEFAULT_PARTITION_NAME));
68-
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME));
66+
Set<String> reservedPartitionValues =
67+
new HashSet<String>() {{
68+
add(HiveConf.getVar(conf, ConfVars.DEFAULT_PARTITION_NAME));
69+
add(HiveConf.getVar(conf, ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME));
70+
}};
71+
6972
// Partition value can't end in this suffix
70-
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL));
71-
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED));
72-
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED));
73-
74-
for (Entry<String, String> e : partitionSpec.entrySet()) {
75-
for (String s : reservedPartitionValues) {
76-
String value = e.getValue();
77-
if (value != null && value.contains(s)) {
78-
throw new SemanticException(ErrorMsg.RESERVED_PART_VAL.getMsg(
79-
"(User value: " + e.getValue() + " Reserved substring: " + s + ")"));
80-
}
73+
Set<String> reservedPartitionSuffixes =
74+
new HashSet<String>() {{
75+
add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL));
76+
add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED));
77+
add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED));
78+
}};
79+
80+
partitionSpec.forEach((key, value) -> {
81+
if (value == null) {
82+
return;
8183
}
82-
}
84+
reservedPartitionValues.stream().filter(value::equals).findAny()
85+
.ifPresent(s -> {
86+
throw new RuntimeException(ErrorMsg.RESERVED_PART_VAL.getMsg(
87+
"(User value: " + value + " Reserved string: " + s + ")"));
88+
});
89+
reservedPartitionSuffixes.stream().filter(value::endsWith).findAny()
90+
.ifPresent(s -> {
91+
throw new RuntimeException(ErrorMsg.RESERVED_PART_VAL.getMsg(
92+
"(User value: " + value + " Partition value cannot end with Reserved substring: " + s + ")"));
93+
});
94+
});
8395
}
8496

8597
public static ExprNodeGenericFuncDesc makeBinaryPredicate(String fn, ExprNodeDesc left, ExprNodeDesc right)

‎ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,50 @@ public void testHiveCloseCurrent() throws Throwable {
422422
assertTrue(hive1 != hive2);
423423
}
424424

425+
@Test
426+
public void testValidatePartitions() {
427+
Map<String, String> partitionSpec = new HashMap<>();
428+
429+
partitionSpec.put("a", HiveConf.getVar(hiveConf, ConfVars.DEFAULT_PARTITION_NAME));
430+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
431+
432+
partitionSpec.clear();
433+
partitionSpec.put("a", HiveConf.getVar(hiveConf, ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME));
434+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
435+
436+
partitionSpec.clear();
437+
partitionSpec.put("a", "random" + HiveConf.getVar(hiveConf, ConfVars.DEFAULT_PARTITION_NAME) + "partition");
438+
PartitionUtils.validatePartitions(hiveConf, partitionSpec);
439+
440+
partitionSpec.clear();
441+
partitionSpec.put("a", "random" + HiveConf.getVar(hiveConf, ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME) + "partition");
442+
PartitionUtils.validatePartitions(hiveConf, partitionSpec);
443+
444+
partitionSpec.clear();
445+
partitionSpec.put("a", HiveConf.getVar(hiveConf, ConfVars.METASTORE_INT_ORIGINAL));
446+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
447+
448+
partitionSpec.clear();
449+
partitionSpec.put("a", HiveConf.getVar(hiveConf, ConfVars.METASTORE_INT_ARCHIVED));
450+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
451+
452+
partitionSpec.clear();
453+
partitionSpec.put("a", HiveConf.getVar(hiveConf, ConfVars.METASTORE_INT_EXTRACTED));
454+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
455+
456+
partitionSpec.clear();
457+
partitionSpec.put("a", "random_part" + HiveConf.getVar(hiveConf, ConfVars.METASTORE_INT_ORIGINAL));
458+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
459+
460+
partitionSpec.clear();
461+
partitionSpec.put("a", "random_part" + HiveConf.getVar(hiveConf, ConfVars.METASTORE_INT_ARCHIVED));
462+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
463+
464+
partitionSpec.clear();
465+
partitionSpec.put("a", "random_part" + HiveConf.getVar(hiveConf, ConfVars.METASTORE_INT_EXTRACTED));
466+
Assert.assertThrows(RuntimeException.class, () -> PartitionUtils.validatePartitions(hiveConf, partitionSpec));
467+
}
468+
425469
@Test
426470
public void testGetAndDropTables() throws Throwable {
427471
try {

‎ql/src/test/queries/clientpositive/alter_table_add_partition.q‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,20 @@ insert into mp partition (b=1) values (1);
1111

1212
desc formatted mp;
1313
desc formatted mp partition (b=1);
14+
15+
drop table test_part;
16+
create table test_part(a int) partitioned by (b string);
17+
18+
alter table test_part add partition(b='one');
19+
20+
set hive.exec.default.partition.name=random;
21+
22+
alter table test_part add partition(b='random_access_memory');
23+
24+
show partitions test_part;
25+
26+
alter table test_part add partition(b='partition_random');
27+
28+
show partitions test_part;
29+
30+
drop table test_part;

‎ql/src/test/results/clientnegative/archive5.q.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
FAILED: SemanticException [Error 10111]: Partition value contains a reserved substring (User value: 14_INTERMEDIATE_ORIGINAL Reserved substring: _INTERMEDIATE_ORIGINAL)
1+
FAILED: RuntimeException [Error 10111]: Partition value contains a reserved substring (User value: 14_INTERMEDIATE_ORIGINAL Partition value cannot end with Reserved substring: _INTERMEDIATE_ORIGINAL)

‎ql/src/test/results/clientnegative/default_partition_name.q.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ POSTHOOK: query: create table default_partition_name (key int, value string) par
66
POSTHOOK: type: CREATETABLE
77
POSTHOOK: Output: database:default
88
POSTHOOK: Output: default@default_partition_name
9-
FAILED: SemanticException [Error 10111]: Partition value contains a reserved substring (User value: __HIVE_DEFAULT_PARTITION__ Reserved substring: __HIVE_DEFAULT_PARTITION__)
9+
FAILED: RuntimeException [Error 10111]: Partition value contains a reserved substring (User value: __HIVE_DEFAULT_PARTITION__ Reserved string: __HIVE_DEFAULT_PARTITION__)

‎ql/src/test/results/clientpositive/llap/alter_table_add_partition.q.out‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,3 +211,65 @@ Bucket Columns: []
211211
Sort Columns: []
212212
Storage Desc Params:
213213
serialization.format 1
214+
PREHOOK: query: drop table test_part
215+
PREHOOK: type: DROPTABLE
216+
PREHOOK: Output: database:default
217+
POSTHOOK: query: drop table test_part
218+
POSTHOOK: type: DROPTABLE
219+
POSTHOOK: Output: database:default
220+
PREHOOK: query: create table test_part(a int) partitioned by (b string)
221+
PREHOOK: type: CREATETABLE
222+
PREHOOK: Output: database:default
223+
PREHOOK: Output: default@test_part
224+
POSTHOOK: query: create table test_part(a int) partitioned by (b string)
225+
POSTHOOK: type: CREATETABLE
226+
POSTHOOK: Output: database:default
227+
POSTHOOK: Output: default@test_part
228+
PREHOOK: query: alter table test_part add partition(b='one')
229+
PREHOOK: type: ALTERTABLE_ADDPARTS
230+
PREHOOK: Output: default@test_part
231+
POSTHOOK: query: alter table test_part add partition(b='one')
232+
POSTHOOK: type: ALTERTABLE_ADDPARTS
233+
POSTHOOK: Output: default@test_part
234+
POSTHOOK: Output: default@test_part@b=one
235+
PREHOOK: query: alter table test_part add partition(b='random_access_memory')
236+
PREHOOK: type: ALTERTABLE_ADDPARTS
237+
PREHOOK: Output: default@test_part
238+
POSTHOOK: query: alter table test_part add partition(b='random_access_memory')
239+
POSTHOOK: type: ALTERTABLE_ADDPARTS
240+
POSTHOOK: Output: default@test_part
241+
POSTHOOK: Output: default@test_part@b=random_access_memory
242+
PREHOOK: query: show partitions test_part
243+
PREHOOK: type: SHOWPARTITIONS
244+
PREHOOK: Input: default@test_part
245+
POSTHOOK: query: show partitions test_part
246+
POSTHOOK: type: SHOWPARTITIONS
247+
POSTHOOK: Input: default@test_part
248+
b=one
249+
b=random_access_memory
250+
PREHOOK: query: alter table test_part add partition(b='partition_random')
251+
PREHOOK: type: ALTERTABLE_ADDPARTS
252+
PREHOOK: Output: default@test_part
253+
POSTHOOK: query: alter table test_part add partition(b='partition_random')
254+
POSTHOOK: type: ALTERTABLE_ADDPARTS
255+
POSTHOOK: Output: default@test_part
256+
POSTHOOK: Output: default@test_part@b=partition_random
257+
PREHOOK: query: show partitions test_part
258+
PREHOOK: type: SHOWPARTITIONS
259+
PREHOOK: Input: default@test_part
260+
POSTHOOK: query: show partitions test_part
261+
POSTHOOK: type: SHOWPARTITIONS
262+
POSTHOOK: Input: default@test_part
263+
b=one
264+
b=partition_random
265+
b=random_access_memory
266+
PREHOOK: query: drop table test_part
267+
PREHOOK: type: DROPTABLE
268+
PREHOOK: Input: default@test_part
269+
PREHOOK: Output: database:default
270+
PREHOOK: Output: default@test_part
271+
POSTHOOK: query: drop table test_part
272+
POSTHOOK: type: DROPTABLE
273+
POSTHOOK: Input: default@test_part
274+
POSTHOOK: Output: database:default
275+
POSTHOOK: Output: default@test_part

0 commit comments

Comments
 (0)