From 87f0e2dc576a4770840862d87bd7e3351a89b8a2 Mon Sep 17 00:00:00 2001 From: WilliamZhu Date: Sat, 9 Sep 2023 12:00:43 +0800 Subject: [PATCH] get hadoop configuration from spark session --- .../src/main/java/tech/mlsql/datalake/DataLake.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamingpro-core/src/main/java/tech/mlsql/datalake/DataLake.scala b/streamingpro-core/src/main/java/tech/mlsql/datalake/DataLake.scala index 349d3c0c0..d9111e3b8 100644 --- a/streamingpro-core/src/main/java/tech/mlsql/datalake/DataLake.scala +++ b/streamingpro-core/src/main/java/tech/mlsql/datalake/DataLake.scala @@ -6,6 +6,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.WowTableIdentifier import org.apache.spark.sql.mlsql.session.MLSQLException import tech.mlsql.common.utils.path.PathFun +import tech.mlsql.tool.HDFSOperatorV2 /** * 2019-08-31 WilliamZhu(allwefantasy@gmail.com) @@ -55,7 +56,7 @@ class DataLake(sparkSession: SparkSession) { } private def listPath(path: Path) = { - val fs = FileSystem.get(new Configuration()) + val fs = FileSystem.get(HDFSOperatorV2.hadoopConfiguration) fs.listStatus(path) } }