diff --git a/build.sbt b/build.sbt index a54837543d9..e78dfa7454b 100644 --- a/build.sbt +++ b/build.sbt @@ -156,11 +156,6 @@ lazy val commonSettings = Seq( unidocSourceFilePatterns := Nil, ) -// enforce java code style -def javafmtCheckSettings() = Seq( - (Compile / compile) := ((Compile / compile) dependsOn (Compile / javafmtCheckAll)).value -) - /** * Note: we cannot access sparkVersion.value here, since that can only be used within a task or * setting macro. diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaHistoryManager.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaHistoryManager.java index 099e56d2b6a..58d55fea11e 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaHistoryManager.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaHistoryManager.java @@ -16,7 +16,6 @@ package io.delta.kernel.internal; import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; -import static io.delta.kernel.internal.fs.Path.getName; import io.delta.kernel.engine.Engine; import io.delta.kernel.exceptions.TableNotFoundException; @@ -93,10 +92,10 @@ public static long getEarliestRecreatableCommit(Engine engine, Path logPath) try (CloseableIterator files = listFrom(engine, logPath, 0) .filter( - fs -> - FileNames.isCommitFile(getName(fs.getPath())) - || FileNames.isCheckpointFile(getName(fs.getPath())))) { - + fs -> { + final Path path = new Path(fs.getPath()); + return FileNames.isCommitFile(path) || FileNames.isCheckpointFile(path); + })) { if (!files.hasNext()) { // listFrom already throws an error if the directory is truly empty, thus this must // be because no files are checkpoint or delta files @@ -114,8 +113,8 @@ public static long getEarliestRecreatableCommit(Engine engine, Path logPath) // remember it and return it once we detect that we've seen a smaller or equal delta // version. while (files.hasNext()) { - String nextFilePath = files.next().getPath(); - if (FileNames.isCommitFile(getName(nextFilePath))) { + final Path nextFilePath = new Path(files.next().getPath()); + if (FileNames.isCommitFile(nextFilePath)) { long version = FileNames.deltaVersion(nextFilePath); if (version == 0L) { return version; @@ -200,7 +199,7 @@ private static List getCommits(Engine engine, Path logPath, long start) throws TableNotFoundException { CloseableIterator commits = listFrom(engine, logPath, start) - .filter(fs -> FileNames.isCommitFile(getName(fs.getPath()))) + .filter(fs -> FileNames.isCommitFile(new Path(fs.getPath()))) .map(fs -> new Commit(FileNames.deltaVersion(fs.getPath()), fs.getModificationTime())); return monotonizeCommitTimestamps(commits); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointInstance.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointInstance.java index d189bf2021c..30efc84c839 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointInstance.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/CheckpointInstance.java @@ -51,11 +51,11 @@ public boolean usesSidecars() { public final Optional filePath; // Guaranteed to be present for V2 checkpoints. - public CheckpointInstance(String path) { + public CheckpointInstance(Path path) { Preconditions.checkArgument( FileNames.isCheckpointFile(path), "not a valid checkpoint file name"); - String[] pathParts = getPathName(path).split("\\."); + String[] pathParts = path.getName().split("\\."); if (pathParts.length == 3 && pathParts[2].equals("parquet")) { // Classic checkpoint 00000000000000000010.checkpoint.parquet @@ -75,9 +75,9 @@ public CheckpointInstance(String path) { this.version = Long.parseLong(pathParts[0]); this.numParts = Optional.empty(); this.format = CheckpointFormat.V2; - this.filePath = Optional.of(new Path(path)); + this.filePath = Optional.of(path); } else { - throw new RuntimeException("Unrecognized checkpoint path format: " + getPathName(path)); + throw new RuntimeException("Unrecognized checkpoint path format: " + path.getName()); } } @@ -193,9 +193,4 @@ public int hashCode() { // the filepath (which is empty) when hashing. return Objects.hash(version, numParts, format, filePath); } - - private String getPathName(String path) { - int slash = path.lastIndexOf("/"); - return path.substring(slash + 1); - } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/Checkpointer.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/Checkpointer.java index 4f09b706a59..c19bbdf82cf 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/Checkpointer.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/Checkpointer.java @@ -111,13 +111,13 @@ public static Optional findLastCompleteCheckpointBefore( List checkpoints = new ArrayList<>(); while (deltaLogFileIter.hasNext()) { FileStatus fileStatus = deltaLogFileIter.next(); - String fileName = new Path(fileStatus.getPath()).getName(); + Path filePath = new Path(fileStatus.getPath()); long currentFileVersion; - if (FileNames.isCommitFile(fileName)) { - currentFileVersion = FileNames.deltaVersion(fileName); - } else if (FileNames.isCheckpointFile(fileName)) { - currentFileVersion = FileNames.checkpointVersion(fileName); + if (FileNames.isCommitFile(filePath)) { + currentFileVersion = FileNames.deltaVersion(filePath); + } else if (FileNames.isCheckpointFile(filePath)) { + currentFileVersion = FileNames.checkpointVersion(filePath); } else { // allow all other types of files. currentFileVersion = currentVersion; @@ -133,7 +133,7 @@ public static Optional findLastCompleteCheckpointBefore( break; } if (validCheckpointFile(fileStatus)) { - checkpoints.add(new CheckpointInstance(fileStatus.getPath())); + checkpoints.add(new CheckpointInstance(filePath)); } numberOfFilesSearched++; } @@ -162,8 +162,7 @@ public static Optional findLastCompleteCheckpointBefore( } private static boolean validCheckpointFile(FileStatus fileStatus) { - return FileNames.isCheckpointFile(new Path(fileStatus.getPath()).getName()) - && fileStatus.getSize() > 0; + return FileNames.isCheckpointFile(new Path(fileStatus.getPath())) && fileStatus.getSize() > 0; } /** The path to the file that holds metadata about the most recent checkpoint. */ diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/fs/Path.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/fs/Path.java index ee70bdd47cd..eab603e21f3 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/fs/Path.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/fs/Path.java @@ -529,8 +529,4 @@ public void validateObject() throws InvalidObjectException { throw new InvalidObjectException("No URI in deserialized Path"); } } - - public static String getName(String pathString) { - return new Path(pathString).getName(); - } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActionsIterator.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActionsIterator.java index bfb51ff5402..c1113e9d847 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActionsIterator.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ActionsIterator.java @@ -263,7 +263,6 @@ public ColumnarBatch extractSidecarsFromBatch( // Sidecars will exist in schema. Extract sidecar files, then remove sidecar files from // batch output. - List outputFiles = new ArrayList<>(); int sidecarIndex = columnarBatch.getSchema().fieldNames().indexOf(LogReplay.SIDECAR_FIELD_NAME); ColumnVector sidecarVector = columnarBatch.getColumnVector(sidecarIndex); for (int i = 0; i < columnarBatch.getSize(); i++) { diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java index d4d352abd23..135c0457e11 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/ConflictChecker.java @@ -30,6 +30,7 @@ import io.delta.kernel.internal.*; import io.delta.kernel.internal.actions.CommitInfo; import io.delta.kernel.internal.actions.SetTransaction; +import io.delta.kernel.internal.fs.Path; import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.FileStatus; @@ -238,7 +239,7 @@ private List getWinningCommitFiles(Engine engine) { List winningCommitFiles = new ArrayList<>(); while (files.hasNext()) { FileStatus file = files.next(); - if (FileNames.isCommitFile(file.getPath())) { + if (FileNames.isCommitFile(new Path(file.getPath()))) { winningCommitFiles.add(file); } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/DeltaLogFile.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/DeltaLogFile.java index 5e0d77e63c9..7d22276f1b3 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/DeltaLogFile.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/replay/DeltaLogFile.java @@ -34,21 +34,21 @@ public enum LogType { } public static DeltaLogFile forCommitOrCheckpoint(FileStatus file) { - String fileName = new Path(file.getPath()).getName(); - LogType logType = null; - long version = -1; - if (FileNames.isCommitFile(fileName)) { + Path filePath = new Path(file.getPath()); + LogType logType; + long version; + if (FileNames.isCommitFile(filePath)) { logType = LogType.COMMIT; - version = FileNames.deltaVersion(fileName); - } else if (FileNames.isClassicCheckpointFile(fileName)) { + version = FileNames.deltaVersion(filePath); + } else if (FileNames.isClassicCheckpointFile(filePath)) { logType = LogType.CHECKPOINT_CLASSIC; - version = FileNames.checkpointVersion(fileName); - } else if (FileNames.isMulitPartCheckpointFile(fileName)) { + version = FileNames.checkpointVersion(filePath); + } else if (FileNames.isMultiPartCheckpointFile(filePath)) { logType = LogType.MULTIPART_CHECKPOINT; - version = FileNames.checkpointVersion(fileName); - } else if (FileNames.isV2CheckpointFile(fileName)) { + version = FileNames.checkpointVersion(filePath); + } else if (FileNames.isV2CheckpointFile(filePath)) { logType = LogType.V2_CHECKPOINT_MANIFEST; - version = FileNames.checkpointVersion(fileName); + version = FileNames.checkpointVersion(filePath); } else { throw new IllegalArgumentException( "File is not a commit or checkpoint file: " + file.getPath()); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/SnapshotManager.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/SnapshotManager.java index a8c22bc8b5d..0262c349010 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/SnapshotManager.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/snapshot/SnapshotManager.java @@ -19,7 +19,6 @@ import static io.delta.kernel.internal.DeltaErrors.wrapEngineExceptionThrowsIO; import static io.delta.kernel.internal.TableFeatures.validateWriteSupportedTable; import static io.delta.kernel.internal.checkpoints.Checkpointer.findLastCompleteCheckpointBefore; -import static io.delta.kernel.internal.fs.Path.getName; import static io.delta.kernel.internal.replay.LogReplayUtils.assertLogFilesBelongToTable; import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.lang.String.format; @@ -260,11 +259,11 @@ private CloseableIterator listFrom(Engine engine, long startVersion) * file (e.g., 000000000.json), or checkpoint file. (e.g., * 000000001.checkpoint.00001.00003.parquet) * - * @param fileName Name of the file (not the full path) + * @param filePath Full path of the file * @return Boolean Whether the file is delta log files */ - private boolean isDeltaCommitOrCheckpointFile(String fileName) { - return FileNames.isCheckpointFile(fileName) || FileNames.isCommitFile(fileName); + private boolean isDeltaCommitOrCheckpointFile(Path filePath) { + return FileNames.isCheckpointFile(filePath) || FileNames.isCommitFile(filePath); } /** @@ -350,16 +349,16 @@ protected final Optional> listDeltaAndCheckpointFiles( while (fileStatusesIter.hasNext()) { final FileStatus fileStatus = fileStatusesIter.next(); - final String fileName = getName(fileStatus.getPath()); + final Path filePath = new Path(fileStatus.getPath()); // Pick up all checkpoint and delta files - if (!isDeltaCommitOrCheckpointFile(fileName)) { + if (!isDeltaCommitOrCheckpointFile(filePath)) { continue; } // Checkpoint files of 0 size are invalid but may be ignored silently when read, // hence we drop them so that we never pick up such checkpoints. - if (FileNames.isCheckpointFile(fileName) && fileStatus.getSize() == 0) { + if (FileNames.isCheckpointFile(filePath) && fileStatus.getSize() == 0) { continue; } // Take files until the version we want to load @@ -385,7 +384,7 @@ protected final Optional> listDeltaAndCheckpointFiles( // files and so maxDeltaVersionSeen should be equal to fileVersion. // But we are being defensive here and taking max of all the // fileVersions seen. - if (FileNames.isCommitFile(fileName)) { + if (FileNames.isCommitFile(filePath)) { maxDeltaVersionSeen.set( Math.max( maxDeltaVersionSeen.get(), FileNames.deltaVersion(fileStatus.getPath()))); @@ -424,7 +423,7 @@ private List getUnbackfilledCommits( .map( commitCoordinatorClientHandler -> { logger.info( - "Getting un-backfilled commits from commit coordinator for " + "table: {}", + "Getting un-backfilled commits from commit coordinator for table: {}", tablePath); return commitCoordinatorClientHandler .getCommits(startVersion, versionToLoad.orElse(null)) @@ -661,8 +660,7 @@ protected Optional getLogSegmentAtOrBeforeVersion( Tuple2, List> checkpointsAndDeltas = ListUtils.partition( - newFiles, - fileStatus -> FileNames.isCheckpointFile(new Path(fileStatus.getPath()).getName())); + newFiles, fileStatus -> FileNames.isCheckpointFile(new Path(fileStatus.getPath()))); final List checkpoints = checkpointsAndDeltas._1; final List deltas = checkpointsAndDeltas._2; @@ -682,7 +680,7 @@ protected Optional getLogSegmentAtOrBeforeVersion( final List checkpointFiles = checkpoints.stream() - .map(f -> new CheckpointInstance(f.getPath())) + .map(f -> new CheckpointInstance(new Path(f.getPath()))) .collect(Collectors.toList()); logDebug(() -> format("checkpointFiles: %s", Arrays.toString(checkpointFiles.toArray()))); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/FileNames.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/FileNames.java index 93c554c9e8f..f87e8c89d68 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/FileNames.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/FileNames.java @@ -54,10 +54,9 @@ public static long deltaVersion(Path path) { return Long.parseLong(path.getName().split("\\.")[0]); } - public static long deltaVersion(String path) { - final int slashIdx = path.lastIndexOf(Path.SEPARATOR); - final String name = path.substring(slashIdx + 1); - return Long.parseLong(name.split("\\.")[0]); + public static long deltaVersion(String pathString) { + final Path path = new Path(pathString); + return deltaVersion(path); } /** Returns the version for the given checkpoint path. */ @@ -65,10 +64,9 @@ public static long checkpointVersion(Path path) { return Long.parseLong(path.getName().split("\\.")[0]); } - public static long checkpointVersion(String path) { - final int slashIdx = path.lastIndexOf(Path.SEPARATOR); - final String name = path.substring(slashIdx + 1); - return Long.parseLong(name.split("\\.")[0]); + public static long checkpointVersion(String pathString) { + final Path path = new Path(pathString); + return checkpointVersion(path); } public static String sidecarFile(Path path, String sidecar) { @@ -128,26 +126,25 @@ public static List checkpointFileWithParts(Path path, long version, int nu return output; } - public static boolean isCheckpointFile(String fileName) { - return CHECKPOINT_FILE_PATTERN.matcher(new Path(fileName).getName()).matches(); + public static boolean isCheckpointFile(Path filePath) { + return CHECKPOINT_FILE_PATTERN.matcher(filePath.getName()).matches(); } - public static boolean isClassicCheckpointFile(String fileName) { - return CLASSIC_CHECKPOINT_FILE_PATTERN.matcher(fileName).matches(); + public static boolean isClassicCheckpointFile(Path filePath) { + return CLASSIC_CHECKPOINT_FILE_PATTERN.matcher(filePath.getName()).matches(); } - public static boolean isMulitPartCheckpointFile(String fileName) { - return MULTI_PART_CHECKPOINT_FILE_PATTERN.matcher(fileName).matches(); + public static boolean isMultiPartCheckpointFile(Path filePath) { + return MULTI_PART_CHECKPOINT_FILE_PATTERN.matcher(filePath.getName()).matches(); } - public static boolean isV2CheckpointFile(String fileName) { - return V2_CHECKPOINT_FILE_PATTERN.matcher(fileName).matches(); + public static boolean isV2CheckpointFile(Path filePath) { + return V2_CHECKPOINT_FILE_PATTERN.matcher(filePath.getName()).matches(); } - public static boolean isCommitFile(String fileName) { - String filename = new Path(fileName).getName(); - return DELTA_FILE_PATTERN.matcher(filename).matches() - || UUID_DELTA_FILE_REGEX.matcher(filename).matches(); + public static boolean isCommitFile(Path filePath) { + return DELTA_FILE_PATTERN.matcher(filePath.getName()).matches() + || UUID_DELTA_FILE_REGEX.matcher(filePath.getName()).matches(); } /** @@ -157,9 +154,9 @@ public static boolean isCommitFile(String fileName) { * upgrade. */ public static long getFileVersion(Path path) { - if (isCheckpointFile(path.getName())) { + if (isCheckpointFile(path)) { return checkpointVersion(path); - } else if (isCommitFile(path.getName())) { + } else if (isCommitFile(path)) { return deltaVersion(path); // } else if (isChecksumFile(path)) { // checksumVersion(path); diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/CheckpointInstanceSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/CheckpointInstanceSuite.scala index 435eeede49e..ba9d6686b6d 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/CheckpointInstanceSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/CheckpointInstanceSuite.scala @@ -30,12 +30,12 @@ class CheckpointInstanceSuite extends AnyFunSuite { test("checkpoint instance comparisons") { val ci1_single_1 = new CheckpointInstance(1, Optional.empty()) val ci1_withparts_2 = new CheckpointInstance(1, Optional.of(2)) - val ci1_v2_1 = new CheckpointInstance("01.checkpoint.abc.parquet" ) + val ci1_v2_1 = new CheckpointInstance(new Path("01.checkpoint.abc.parquet" )) val ci2_single_1 = new CheckpointInstance(2, Optional.empty()) val ci2_withparts_4 = new CheckpointInstance(2, Optional.of(4)) - val ci2_v2_1 = new CheckpointInstance("02.checkpoint.abc.parquet" ) - val ci2_v2_2 = new CheckpointInstance("02.checkpoint.def.parquet") + val ci2_v2_1 = new CheckpointInstance(new Path("02.checkpoint.abc.parquet" )) + val ci2_v2_2 = new CheckpointInstance(new Path("02.checkpoint.def.parquet")) val ci3_single_1 = new CheckpointInstance(3, Optional.empty()) val ci3_withparts_2 = new CheckpointInstance(3, Optional.of(2)) @@ -64,11 +64,11 @@ class CheckpointInstanceSuite extends AnyFunSuite { } test("checkpoint instance equality") { - val single = new CheckpointInstance("01.checkpoint.parquet") - val multipartPart1 = new CheckpointInstance("01.checkpoint.01.02.parquet") - val multipartPart2 = new CheckpointInstance("01.checkpoint.02.02.parquet") - val v2Checkpoint1 = new CheckpointInstance("01.checkpoint.abc-def.parquet") - val v2Checkpoint2 = new CheckpointInstance("01.checkpoint.ghi-klm.parquet") + val single = new CheckpointInstance(new Path("01.checkpoint.parquet")) + val multipartPart1 = new CheckpointInstance(new Path("01.checkpoint.01.02.parquet")) + val multipartPart2 = new CheckpointInstance(new Path("01.checkpoint.02.02.parquet")) + val v2Checkpoint1 = new CheckpointInstance(new Path("01.checkpoint.abc-def.parquet")) + val v2Checkpoint2 = new CheckpointInstance(new Path("01.checkpoint.ghi-klm.parquet")) // Single checkpoint is not equal to any other checkpoints at the same version. Seq(multipartPart1, multipartPart2, v2Checkpoint1, v2Checkpoint2).foreach { ci => @@ -94,7 +94,7 @@ class CheckpointInstanceSuite extends AnyFunSuite { test("checkpoint instance instantiation") { // classic checkpoint val classicCheckpoint = new CheckpointInstance( - new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.parquet").toString) + new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.parquet")) assert(classicCheckpoint.version == 10) assert(!classicCheckpoint.numParts.isPresent()) assert(classicCheckpoint.format == CheckpointInstance.CheckpointFormat.CLASSIC) @@ -102,8 +102,9 @@ class CheckpointInstanceSuite extends AnyFunSuite { // multi-part checkpoint val multipartCheckpoint = new CheckpointInstance( - new Path(FAKE_DELTA_LOG_PATH, - "00000000000000000010.checkpoint.0000000002.0000000003.parquet").toString) + new Path( + FAKE_DELTA_LOG_PATH, + "00000000000000000010.checkpoint.0000000002.0000000003.parquet")) assert(multipartCheckpoint.version == 10) assert(multipartCheckpoint.numParts.isPresent() && multipartCheckpoint.numParts.get() == 3) assert(multipartCheckpoint.format == CheckpointInstance.CheckpointFormat.MULTI_PART) @@ -111,8 +112,7 @@ class CheckpointInstanceSuite extends AnyFunSuite { // V2 checkpoint val v2Checkpoint = new CheckpointInstance( - new Path(FAKE_DELTA_LOG_PATH, - "00000000000000000010.checkpoint.abcda-bacbac.parquet").toString) + new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.abcda-bacbac.parquet")) assert(v2Checkpoint.version == 10) assert(!v2Checkpoint.numParts.isPresent()) assert(v2Checkpoint.format == CheckpointInstance.CheckpointFormat.V2) @@ -121,13 +121,11 @@ class CheckpointInstanceSuite extends AnyFunSuite { // invalid checkpoints intercept[RuntimeException] { new CheckpointInstance( - new Path(FAKE_DELTA_LOG_PATH, - "00000000000000000010.checkpoint.000000.a.parquet").toString) + new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.checkpoint.000000.a.parquet")) } intercept[RuntimeException] { new CheckpointInstance( - new Path(FAKE_DELTA_LOG_PATH, - "00000000000000000010.parquet").toString) + new Path(FAKE_DELTA_LOG_PATH, "00000000000000000010.parquet")) } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/FileNamesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/FileNamesSuite.scala index 0dbad9bbdb2..34ea471dc8e 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/FileNamesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/FileNamesSuite.scala @@ -24,12 +24,12 @@ import org.scalatest.funsuite.AnyFunSuite class FileNamesSuite extends AnyFunSuite { test("isCheckpointFile") { - assert(FileNames.isCheckpointFile("/a/123.checkpoint.parquet")) - assert(FileNames.isCheckpointFile("/a/123.checkpoint.0000000001.0000000087.parquet")) - assert(FileNames.isCheckpointFile("/a/000000010.checkpoint.80a083e8-7026.json")) - assert(FileNames.isCheckpointFile("/a/000000010.checkpoint.80a083e8-7026.parquet")) - assert(!FileNames.isCheckpointFile("/a/123.json")) - assert(!FileNames.isCommitFile("/a/123.checkpoint.3.json")) + assert(FileNames.isCheckpointFile(new Path("/a/123.checkpoint.parquet"))) + assert(FileNames.isCheckpointFile(new Path("/a/123.checkpoint.0000000001.0000000087.parquet"))) + assert(FileNames.isCheckpointFile(new Path("/a/000000010.checkpoint.80a083e8-7026.json"))) + assert(FileNames.isCheckpointFile(new Path("/a/000000010.checkpoint.80a083e8-7026.parquet"))) + assert(!FileNames.isCheckpointFile(new Path("/a/123.json"))) + assert(!FileNames.isCommitFile(new Path("/a/123.checkpoint.3.json"))) } test("checkpointVersion") { diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala index aa38cda8409..9e42c0a4d76 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/SnapshotManagerSuite.scala @@ -212,7 +212,7 @@ class SnapshotManagerSuite extends AnyFunSuite with MockFileSystemClientUtils { FileNames.checkpointVersion(topLevelFile.getPath) == v } if (matchingCheckpoints.nonEmpty) { - matchingCheckpoints.maxBy(f => new CheckpointInstance(f._1.getPath)) match { + matchingCheckpoints.maxBy(f => new CheckpointInstance(new Path(f._1.getPath))) match { case (c, sidecars) => (Seq(c), sidecars) } } else { diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CheckpointV2ReadSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CheckpointV2ReadSuite.scala index 9edcd0d901f..f0d0152994a 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CheckpointV2ReadSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CheckpointV2ReadSuite.scala @@ -23,6 +23,7 @@ import io.delta.kernel.defaults.engine.DefaultEngine import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestRow, TestUtils} import io.delta.kernel.expressions.Literal import io.delta.kernel.internal.checkpoints.CheckpointInstance +import io.delta.kernel.internal.fs.{Path => DeltaPath} import io.delta.kernel.internal.{InternalScanFileUtils, SnapshotImpl} import io.delta.tables.DeltaTable import org.apache.hadoop.conf.Configuration @@ -92,7 +93,8 @@ class CheckpointV2ReadSuite extends AnyFunSuite with TestUtils with ExpressionTe f => FileNames.checkpointVersion(new Path(f.getPath))) .contains(expectedV2CkptToRead)) assert(snapshotImpl.getLogSegment.checkpoints.asScala.map( - f => new CheckpointInstance(f.getPath).format == CheckpointInstance.CheckpointFormat.V2) + f => new CheckpointInstance(new DeltaPath(f.getPath)).format + == CheckpointInstance.CheckpointFormat.V2) .contains(expectV2CheckpointFormat)) diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala index c3fe8acaca1..dfb94699409 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/LogReplayEngineMetricsSuite.scala @@ -386,7 +386,7 @@ trait FileReadMetrics { self: Object => private def updateVersionsRead(fileStatus: FileStatus): Unit = { val path = new Path(fileStatus.getPath) - if (FileNames.isCommitFile(path.getName) || FileNames.isCheckpointFile(path.getName)) { + if (FileNames.isCommitFile(path) || FileNames.isCheckpointFile(path)) { val version = FileNames.getFileVersion(path) // We may split json/parquet reads, so don't record the same file multiple times diff --git a/project/Checkstyle.scala b/project/Checkstyle.scala index ee6906522c9..66ca6c682bf 100644 --- a/project/Checkstyle.scala +++ b/project/Checkstyle.scala @@ -14,10 +14,11 @@ * limitations under the License. */ -import com.etsy.sbt.checkstyle.CheckstylePlugin.autoImport._ -import org.scalastyle.sbt.ScalastylePlugin.autoImport._ -import sbt._ -import sbt.Keys._ +import com.etsy.sbt.checkstyle.CheckstylePlugin.autoImport.* +import com.lightbend.sbt.JavaFormatterPlugin.autoImport.javafmtCheckAll +import org.scalastyle.sbt.ScalastylePlugin.autoImport.* +import sbt.* +import sbt.Keys.* object Checkstyle { @@ -100,4 +101,8 @@ object Checkstyle { } } + // Enforce java code style + lazy val javafmtCheckSettings = Seq( + (Compile / compile) := ((Compile / compile) dependsOn (Compile / javafmtCheckAll)).value + ) }