diff --git a/ql/src/test/results/clientpositive/llap/show_partitions2.q.out b/ql/src/test/results/clientpositive/llap/show_partitions2.q.out index 1ed9eaa56b75..3b6184a61640 100644 --- a/ql/src/test/results/clientpositive/llap/show_partitions2.q.out +++ b/ql/src/test/results/clientpositive/llap/show_partitions2.q.out @@ -211,8 +211,8 @@ PREHOOK: Input: default@mpart1 POSTHOOK: query: SHOW PARTITIONS mpart1 PARTITION(ds = '1980-11-10') LIMIT 3 POSTHOOK: type: SHOWPARTITIONS POSTHOOK: Input: default@mpart1 -ds=1980-11-10/hr=12/rs=EU ds=1980-11-10/hr=10/rs=AS +ds=1980-11-10/hr=12/rs=EU ds=1980-11-10/hr=15/rs=EU PREHOOK: query: SHOW PARTITIONS mpart1 where ds = '__HIVE_DEFAULT_PARTITION__' AND hr = '__HIVE_DEFAULT_PARTITION__' PREHOOK: type: SHOWPARTITIONS diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 965ff0edc169..f22c89de8c7f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -28,7 +28,6 @@ import java.net.InetAddress; import java.nio.ByteBuffer; import java.sql.Connection; -import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Statement; import java.time.Instant; @@ -3812,7 +3811,7 @@ protected Integer getJdoResult(GetHelper ctx) * you want results for. E.g., if resultsCol is partitionName, the Collection * has types of String, and if resultsCol is null, the types are MPartition. */ - private Collection getPartitionPsQueryResults(String catName, String dbName, + private Collection getPartitionPsQueryResults(String catName, String dbName, String tableName, List part_vals, int max_parts, String resultsCol) throws MetaException, NoSuchObjectException { @@ -3832,6 +3831,7 @@ private Collection getPartitionPsQueryResults(String catName, String dbN String filter = getJDOFilterStrForPartitionVals(table, part_vals, params); try (QueryWrapper query = new QueryWrapper(pm.newQuery(MPartition.class))) { query.setFilter(filter); + query.setOrdering("partitionName ascending"); query.declareParameters(makeParameterDeclarationString(params)); if (max_parts >= 0) { // User specified a row limit, set it on the Query @@ -3840,9 +3840,7 @@ private Collection getPartitionPsQueryResults(String catName, String dbN if (resultsCol != null && !resultsCol.isEmpty()) { query.setResult(resultsCol); } - - Collection result = (Collection) query.executeWithMap(params); - + Collection result = (Collection) query.executeWithMap(params); return Collections.unmodifiableCollection(new ArrayList<>(result)); } } @@ -3913,11 +3911,11 @@ protected List getSqlResult(GetHelper> ctx) throws Me protected List getJdoResult(GetHelper> ctx) throws MetaException, NoSuchObjectException { List result = new ArrayList<>(); - Collection parts = getPartitionPsQueryResults(catName, dbName, tblName, + Collection parts = getPartitionPsQueryResults(catName, dbName, tblName, args.getPart_vals(), args.getMax(), null); boolean isAcidTable = TxnUtils.isAcidTable(ctx.getTable()); - for (Object o : parts) { - Partition part = convertToPart(catName, dbName, tblName, (MPartition) o, isAcidTable, args); + for (MPartition o : parts) { + Partition part = convertToPart(catName, dbName, tblName, o, isAcidTable, args); result.add(part); } return result; @@ -9504,6 +9502,9 @@ public Map updateTableColumnStatistics(ColumnStatistics colStats // So let's not use them anywhere unless absolutely necessary. String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); + lockForUpdate("TBLS", "TBL_ID", Optional.of("\"TBL_ID\" = " + mTable.getId())); + // Get the newest version of mTable + pm.refresh(mTable); Table table = convertToTable(mTable); List colNames = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { @@ -9592,23 +9593,25 @@ public Map updatePartitionColumnStatistics(Table table, MTable m List statsObjs = colStats.getStatsObj(); ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); - Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), getMPartition( - catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals, mTable), TxnUtils.isAcidTable(table)); + MPartition mPartition = getMPartition( + catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals, mTable); + Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), + mPartition, TxnUtils.isAcidTable(table)); List colNames = new ArrayList<>(); for(ColumnStatisticsObj statsObj : statsObjs) { colNames.add(statsObj.getColName()); } - Map oldStats = getPartitionColStats(table, statsDesc - .getPartName(), colNames, colStats.getEngine()); - - MPartition mPartition = getMPartition( - catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals, mTable); if (partition == null) { throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); } + Map oldStats = getPartitionColStats(table, statsDesc + .getPartName(), colNames, colStats.getEngine()); + lockForUpdate("PARTITIONS", "PART_ID", Optional.of("\"PART_ID\" = " + mPartition.getId())); + pm.refresh(mPartition); + for (ColumnStatisticsObj statsObj : statsObjs) { MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj, colStats.getEngine()); @@ -11435,36 +11438,34 @@ public List getAllWriteEventInfo(long txnId, String dbName, Stri return writeEventInfoList; } - private void prepareQuotes() throws SQLException { - String s = dbType.getPrepareTxnStmt(); - if (s != null) { - assert pm.currentTransaction().isActive(); - JDOConnection jdoConn = pm.getDataStoreConnection(); - try (Statement statement = ((Connection) jdoConn.getNativeConnection()).createStatement()) { - statement.execute(s); - } finally { - jdoConn.close(); - } - } - } - - private void lockNotificationSequenceForUpdate() throws MetaException { + private void lockForUpdate(String tableName, String column, Optional rowFilter) + throws MetaException { if (sqlGenerator.getDbProduct().isDERBY() && directSql != null) { // Derby doesn't allow FOR UPDATE to lock the row being selected (See https://db.apache // .org/derby/docs/10.1/ref/rrefsqlj31783.html) . So lock the whole table. Since there's // only one row in the table, this shouldn't cause any performance degradation. new RetryingExecutor(conf, () -> { - directSql.lockDbTable("NOTIFICATION_SEQUENCE"); + directSql.lockDbTable(tableName); }).run(); } else { - String selectQuery = "select \"NEXT_EVENT_ID\" from \"NOTIFICATION_SEQUENCE\""; + String selectQuery = "select \"" + column + "\" from \"" + tableName + "\"" + + rowFilter.map(f -> " where " + f).orElse(""); String lockingQuery = sqlGenerator.addForUpdateClause(selectQuery); new RetryingExecutor(conf, () -> { - prepareQuotes(); - try (QueryWrapper query = new QueryWrapper(pm.newQuery("javax.jdo.query.SQL", lockingQuery))) { - query.setUnique(true); - // only need to execute it to get db Lock - query.execute(); + String txnStmt = dbType.getPrepareTxnStmt(); + List statements = new ArrayList<>(); + if (txnStmt != null) { + statements.add(txnStmt); + } + statements.add(lockingQuery); + assert pm.currentTransaction().isActive(); + JDOConnection jdoConn = pm.getDataStoreConnection(); + try (Statement statement = ((Connection) jdoConn.getNativeConnection()).createStatement()) { + for (String s : statements) { + statement.execute(s); + } + } finally { + jdoConn.close(); } }).run(); } @@ -11532,7 +11533,7 @@ public void addNotificationEvent(NotificationEvent entry) throws MetaException { try { pm.flush(); openTransaction(); - lockNotificationSequenceForUpdate(); + lockForUpdate("NOTIFICATION_SEQUENCE", "NEXT_EVENT_ID", Optional.empty()); query = pm.newQuery(MNotificationNextId.class); Collection ids = (Collection) query.execute(); MNotificationNextId mNotificationNextId = null; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java index 267c9e8e5acd..f4750649505c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MPartition.java @@ -22,7 +22,7 @@ import java.util.Map; public class MPartition { - + private long id; private String partitionName; // partitionname ==> (key=value/)*(key=value) private MTable table; private List values; @@ -31,7 +31,6 @@ public class MPartition { private MStorageDescriptor sd; private Map parameters; private long writeId; - public MPartition() {} /** @@ -159,4 +158,8 @@ public long getWriteId() { public void setWriteId(long writeId) { this.writeId = writeId; } + + public long getId() { + return id; + } } diff --git a/standalone-metastore/metastore-server/src/main/resources/package.jdo b/standalone-metastore/metastore-server/src/main/resources/package.jdo index af5209ff07e2..8e29e0a35228 100644 --- a/standalone-metastore/metastore-server/src/main/resources/package.jdo +++ b/standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -506,15 +506,15 @@ - + - - - + + +