From d5d1df7f0521be55e7d44b18a45817b7509db770 Mon Sep 17 00:00:00 2001 From: bheroder Date: Fri, 8 Sep 2023 15:02:54 -0700 Subject: [PATCH] add uri to filesystem get --- .../scala/com/linkedin/feathr/offline/util/HdfsUtils.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala index 3ea971822..8a84b9363 100644 --- a/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala +++ b/feathr-impl/src/main/scala/com/linkedin/feathr/offline/util/HdfsUtils.scala @@ -6,6 +6,7 @@ import org.apache.hadoop.fs.{FileSystem, LocatedFileStatus, Path, PathFilter, Re import org.apache.log4j.{Logger, PatternLayout, WriterAppender} import java.io.{FileSystem => _, _} +import java.net.URI import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit import java.time.{LocalDateTime, ZoneId, ZoneOffset} @@ -451,7 +452,7 @@ object HdfsUtils { * @return true if exists else false. */ def exists(path: String, conf: Configuration = conf): Boolean = { - FileSystem.get(conf).exists(new Path(path)) + FileSystem.get(URI.create(path), conf).exists(new Path(path)) } /** @@ -489,7 +490,7 @@ object HdfsUtils { * @return true if nonEmpty */ def nonEmpty(inputPath: String, conf: Configuration = conf): Boolean = { - val fs = FileSystem.get(conf) + val fs = FileSystem.get(URI.create(inputPath), conf) val path = new Path(inputPath) if (!exists(inputPath)) return false if (fs.getFileStatus(path).isDirectory) {