Merge branch 'fix/add_files' into HEAD

Author: SirOibaf

java/beam/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.9.0-RC9</version>
+    <version>3.9.0-RC10</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/flink/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.9.0-RC9</version>
+    <version>3.9.0-RC10</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/hsfs/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.9.0-RC9</version>
+    <version>3.9.0-RC10</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/pom.xml

@@ -7,7 +7,7 @@
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-parent</artifactId>
     <packaging>pom</packaging>
-    <version>3.9.0-RC9</version>
+    <version>3.9.0-RC10</version>
     <modules>
         <module>hsfs</module>
         <module>spark</module>

java/spark/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hsfs-parent</artifactId>
         <groupId>com.logicalclocks</groupId>
-        <version>3.9.0-RC9</version>
+        <version>3.9.0-RC10</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

python/hsfs/engine/spark.py

@@ -20,7 +20,6 @@
 import json
 import os
 import re
-import shutil
 import warnings
 from datetime import date, datetime, timezone
 from typing import Any, List, Optional, TypeVar, Union
@@ -994,22 +993,16 @@ def add_file(self, file):
 
         # for external clients, download the file
         if isinstance(client.get_instance(), client.external.Client):
-            tmp_file = os.path.join(SparkFiles.getRootDirectory(), file_name)
+            tmp_file = "f/tmp/{file_name)"
             print("Reading key file from storage connector.")
             response = self._dataset_api.read_content(file, util.get_dataset_type(file))
 
             with open(tmp_file, "wb") as f:
                 f.write(response.content)
-        else:
-            self._spark_context.addFile(file)
 
-            # The file is not added to the driver current working directory
-            # We should add it manually by copying from the download location
-            # The file will be added to the executors current working directory
-            # before the next task is executed
-            shutil.copy(SparkFiles.get(file_name), file_name)
+        self._spark_context.addFile(file)
 
-        return file_name
+        return SparkFiles.get(file_name)
 
     def profile(
         self,

python/hsfs/version.py

@@ -14,4 +14,4 @@
 #   limitations under the License.
 #
 
-__version__ = "3.9.0rc9"
+__version__ = "3.9.0rc10"

utils/java/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-utils</artifactId>
-    <version>3.9.0-RC9</version>
+    <version>3.9.0-RC10</version>
 
     <properties>
         <hops.version>3.2.0.0-SNAPSHOT</hops.version>