From 5d326237b118590eaebf41a813f873738246adb5 Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Fri, 11 Oct 2024 11:08:21 +0800 Subject: [PATCH] fixed --- .../paimon/table/system/BranchesTable.java | 79 +++---------------- .../apache/paimon/utils/BranchManager.java | 65 +++++++++++---- 2 files changed, 60 insertions(+), 84 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java index c5c85549c5733..059148b3f6e04 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java @@ -18,7 +18,6 @@ package org.apache.paimon.table.system; -import org.apache.paimon.Snapshot; import org.apache.paimon.branch.Branch; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; @@ -29,8 +28,6 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.schema.SchemaManager; -import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.ReadonlyTable; @@ -46,9 +43,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; import org.apache.paimon.utils.BranchManager; -import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.IteratorRecordReader; -import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ProjectedRow; import org.apache.paimon.utils.SerializationUtils; @@ -64,14 +59,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.SortedMap; -import java.util.stream.Collectors; import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; -import static org.apache.paimon.utils.BranchManager.BRANCH_PREFIX; -import static org.apache.paimon.utils.BranchManager.branchPath; -import static org.apache.paimon.utils.FileUtils.listVersionedDirectories; -import static org.apache.paimon.utils.Preconditions.checkArgument; /** A {@link Table} for showing branches of table. */ public class BranchesTable implements ReadonlyTable { @@ -230,73 +219,25 @@ public RecordReader createReader(Split split) { private List branches(FileStoreTable table) throws IOException { BranchManager branchManager = table.branchManager(); - SchemaManager schemaManager = new SchemaManager(fileIO, table.location()); - - Map branchesAsMap = branchManager.branchObjectsAsMap(); - - List> paths = - listVersionedDirectories(fileIO, branchManager.branchDirectory(), BRANCH_PREFIX) - .map(status -> Pair.of(status.getPath(), status.getModificationTime())) - .collect(Collectors.toList()); + List branches = branchManager.branchObjects(); List result = new ArrayList<>(); - for (Pair path : paths) { - String branchName = path.getLeft().getName().substring(BRANCH_PREFIX.length()); - String basedTag = null; - Long basedSnapshotId = null; - long creationTime = path.getRight(); - - Optional tableSchema = - schemaManager.copyWithBranch(branchName).latest(); - if (tableSchema.isPresent()) { - FileStoreTable branchTable = - FileStoreTableFactory.create( - fileIO, new Path(branchPath(table.location(), branchName))); - SortedMap> snapshotTags = - branchTable.tagManager().tags(); - Long earliestSnapshotId = branchTable.snapshotManager().earliestSnapshotId(); - if (snapshotTags.isEmpty()) { - // create based on snapshotId - basedSnapshotId = earliestSnapshotId; - } else { - Snapshot snapshot = snapshotTags.firstKey(); - if (Objects.equals(earliestSnapshotId, snapshot.id())) { - // create based on tag - List tags = snapshotTags.get(snapshot); - checkArgument(tags.size() == 1); - basedTag = tags.get(0); - basedSnapshotId = snapshot.id(); - } else { - // create based on snapshotId - basedSnapshotId = earliestSnapshotId; - } - } - } - Branch currentBranch = null; - if (branchesAsMap.containsKey(branchName)) { - currentBranch = branchesAsMap.get(branchName); - } - + for (Branch branch : branches) { result.add( GenericRow.of( - BinaryString.fromString(branchName), - BinaryString.fromString(basedTag), - basedSnapshotId, - (currentBranch == null - || currentBranch.getBranchCreateTime() == null) - ? Timestamp.fromLocalDateTime( - DateTimeUtils.toLocalDateTime(creationTime)) - : Timestamp.fromLocalDateTime( - currentBranch.getBranchCreateTime()), - (currentBranch == null - || currentBranch.getBranchTimeRetained() == null) + BinaryString.fromString(branch.getBranchName()), + BinaryString.fromString(branch.getFromTagName()), + branch.getFromSnapshotId(), + branch.getBranchCreateTime() != null + ? Timestamp.fromLocalDateTime(branch.getBranchCreateTime()) + : null, + (branch.getBranchTimeRetained() == null) ? null - : Optional.ofNullable(currentBranch.getBranchTimeRetained()) + : Optional.ofNullable(branch.getBranchTimeRetained()) .map(Object::toString) .map(BinaryString::fromString) .orElse(null))); } - return result; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java index 699f974c7d55a..d6c0ca617d8ab 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java @@ -24,6 +24,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.tag.Tag; import org.slf4j.Logger; @@ -34,9 +36,10 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.SortedMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -260,28 +263,23 @@ public List branches() { } } - public Map branchObjectsAsMap() { - List branches = branchObjects(); - if (branches.isEmpty()) { - return Collections.emptyMap(); - } - return branchObjects().stream() - .collect(Collectors.toMap(branch -> branch.getBranchName(), branch -> branch)); - } - /** Get all {@link Tag}s. */ public List branchObjects() { try { - List paths = + List> paths = listVersionedDirectories(fileIO, branchDirectory(), BRANCH_PREFIX) - .map(status -> status.getPath()) + .map(status -> Pair.of(status.getPath(), status.getModificationTime())) .collect(Collectors.toList()); List branches = new ArrayList<>(); - for (Path path : paths) { - String branchName = path.getName().substring(BRANCH_PREFIX.length()); + + for (Pair path : paths) { + String branchName = path.getLeft().getName().substring(BRANCH_PREFIX.length()); Branch branch = Branch.safelyFromPath(fileIO, branchMetadataPath(branchName)); if (branch != null) { branches.add(branch); + } else { + // Compatible with older versions + branches.add(extractBranchInfo(path, branchName)); } } return branches; @@ -290,6 +288,43 @@ public List branchObjects() { } } + private Branch extractBranchInfo(Pair path, String branchName) { + String fromTagName = null; + Long fromSnapshotId = null; + long creationTime = path.getRight(); + + Optional tableSchema = schemaManager.copyWithBranch(branchName).latest(); + if (tableSchema.isPresent()) { + FileStoreTable branchTable = + FileStoreTableFactory.create( + fileIO, new Path(branchPath(tablePath, branchName))); + SortedMap> snapshotTags = branchTable.tagManager().tags(); + Long earliestSnapshotId = branchTable.snapshotManager().earliestSnapshotId(); + if (snapshotTags.isEmpty()) { + // create based on snapshotId + fromSnapshotId = earliestSnapshotId; + } else { + Snapshot snapshot = snapshotTags.firstKey(); + if (Objects.equals(earliestSnapshotId, snapshot.id())) { + // create based on tag + List tags = snapshotTags.get(snapshot); + checkArgument(tags.size() == 1); + fromTagName = tags.get(0); + fromSnapshotId = snapshot.id(); + } else { + // create based on snapshotId + fromSnapshotId = earliestSnapshotId; + } + } + } + return new Branch( + branchName, + fromSnapshotId, + fromTagName, + DateTimeUtils.toLocalDateTime(creationTime), + null); + } + private void validateBranch(String branchName) { checkArgument( !isMainBranch(branchName),