Compound File SDK Support (#48)

* wip: compound file helpers

* wip: compound file upload

* fix: fix chunk size (#49)

* fix: better handler null url error cases

* feat: add signer param to dataset object (#50)

* Add compound file upload example

Co-authored-by: Vitalii Bulyzhyn <vitalybulyzhin@gmail.com>

Author: PJEstrada

sdk/diffgram/file/compound_file.py

@@ -0,0 +1,190 @@
+from diffgram.core.core import Project
+from typing import List
+from diffgram.file.file import File
+from diffgram.job.job import Job
+
+
+class CompoundChildFile:
+    path: str
+    url: str
+    blob_path: str
+    file_name: str
+    bucket_name: str
+    child_file_type: str
+    connection_id: int
+    media_type: str
+    job: Job
+    job_id: int
+    directory_id: int
+    instance_list: list
+    frame_packet_map: dict
+    assume_new_instances_machine_made: bool
+    convert_names_to_label_files: bool
+    video_split_duration: int
+
+    def __init__(self,
+                 child_file_type: str,
+                 path: str = None,
+                 url: str = None,
+                 blob_path: str = None,
+                 file_name: str = None,
+                 bucket_name: str = None,
+                 connection_id: int = None,
+                 media_type: str = None,
+                 job: Job = None,
+                 job_id: int = None,
+                 directory_id: int = None,
+                 instance_list: list = None,
+                 frame_packet_map: dict = None,
+                 assume_new_instances_machine_made: bool = None,
+                 convert_names_to_label_files: bool = None,
+                 video_split_duration: int = None):
+        self.child_file_type = child_file_type
+        self.path = path
+        self.url = url
+        self.blob_path = blob_path
+        self.file_name = file_name
+        self.bucket_name = bucket_name
+        self.connection_id = connection_id
+        self.media_type = media_type
+        self.job = job
+        self.job_id = job_id
+        self.directory_id = directory_id
+        self.instance_list = instance_list
+        self.frame_packet_map = frame_packet_map
+        self.assume_new_instances_machine_made = assume_new_instances_machine_made
+        self.convert_names_to_label_files = convert_names_to_label_files
+        self.video_split_duration = video_split_duration
+
+
+class CompoundFile:
+    project: Project
+    parent_file_data: dict
+    child_files_to_upload: List[CompoundChildFile]
+
+    def __init__(self, project: Project, name: str, directory_id: int):
+        self.project = project
+        self.name = name
+        self.directory_id = directory_id
+        self.child_files_to_upload = []
+
+    def remove_compound_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
+        self.child_files_to_upload.remove(child_file)
+        return self.child_files_to_upload
+
+    def __create_compound_parent_file(self):
+        url = f'/api/v1/project/{self.project.project_string_id}/file/new-compound'
+        data = {
+            'name': self.name,
+            'directory_id': self.directory_id
+        }
+        response = self.project.session.post(url = self.project.host + url,
+                                             json = data)
+        self.project.handle_errors(response)
+        data = response.json()
+        self.parent_file_data = data.get('file')
+        print('self,', self.parent_file_data)
+        return data.get('file')
+
+    def __create_child_file(self, child_file: CompoundChildFile):
+        if child_file.child_file_type == 'from_local':
+            return self.project.file.from_local(
+                path = child_file.path,
+                directory_id = self.directory_id,
+                instance_list = child_file.instance_list,
+                frame_packet_map = child_file.frame_packet_map,
+                assume_new_instances_machine_made = child_file.assume_new_instances_machine_made,
+                convert_names_to_label_files = child_file.convert_names_to_label_files,
+                parent_file_id = self.parent_file_data.get('id')
+            )
+        elif child_file.child_file_type == 'from_url':
+            return self.project.file.from_url(
+                url = child_file.path,
+                media_type = child_file.media_type,
+                job_id = child_file.job_id,
+                job = child_file.job,
+                video_split_duration = child_file.video_split_duration,
+                instance_list = child_file.instance_list,
+                frame_packet_map = child_file.frame_packet_map,
+                parent_file_id = self.parent_file_data.get('id')
+            )
+        elif child_file.child_file_type == 'from_blob_path':
+            return self.project.file.from_blob_path(
+                blob_path = child_file.blob_path,
+                bucket_name = child_file.bucket_name,
+                connection_id = child_file.connection_id,
+                media_type = child_file.media_type,
+                instance_list = child_file.instance_list,
+                file_name = child_file.file_name,
+                frame_packet_map = child_file.frame_packet_map,
+                parent_file_id = self.parent_file_data.get('id')
+            )
+
+    def add_child_from_local(self,
+                             path: str,
+                             instance_list: list = None,
+                             frame_packet_map: dict = None,
+                             assume_new_instances_machine_made: bool = True,
+                             convert_names_to_label_files: bool = True):
+        new_child_file = CompoundChildFile(
+            child_file_type = "from_local",
+            path = path,
+            directory_id = self.directory_id,
+            instance_list = instance_list,
+            frame_packet_map = frame_packet_map,
+            assume_new_instances_machine_made = assume_new_instances_machine_made,
+            convert_names_to_label_files = convert_names_to_label_files
+        )
+        self.child_files_to_upload.append(new_child_file)
+        return new_child_file
+
+    def add_child_file_from_url(self,
+                                url: str,
+                                media_type: str = "image",
+                                job: Job = None,
+                                job_id: int = None,
+                                video_split_duration: int = None,
+                                instance_list: list = None,
+                                frame_packet_map: dict = None):
+        new_child_file = CompoundChildFile(
+            child_file_type = "from_url",
+            url = url,
+            media_type = media_type,
+            job = job,
+            directory_id = self.directory_id,
+            job_id = job_id,
+            video_split_duration = video_split_duration,
+            instance_list = instance_list,
+            frame_packet_map = frame_packet_map,
+        )
+        self.child_files_to_upload.append(new_child_file)
+        return new_child_file
+
+    def add_child_from_blob_path(self,
+                                 blob_path: str,
+                                 bucket_name: str,
+                                 connection_id: int,
+                                 media_type: str = 'image',
+                                 instance_list: list = None,
+                                 file_name: str = None,
+                                 frame_packet_map: dict = None
+                                 ):
+        new_child_file = CompoundChildFile(
+            child_file_type = "from_blob_path",
+            blob_path = blob_path,
+            bucket_name = bucket_name,
+            connection_id = connection_id,
+            directory_id = self.directory_id,
+            media_type = media_type,
+            instance_list = instance_list,
+            file_name = file_name,
+            frame_packet_map = frame_packet_map,
+        )
+        self.child_files_to_upload.append(new_child_file)
+        return new_child_file
+
+    def upload(self):
+        parent_file_data: dict = self.__create_compound_parent_file()
+        for child_file in self.child_files_to_upload:
+            self.__create_child_file(child_file)
+        return parent_file_data

sdk/diffgram/file/file_constructor.py

@@ -43,7 +43,8 @@ def from_local(
             instance_list: list = None,
             frame_packet_map: dict = None,
             assume_new_instances_machine_made: bool = True,
-            convert_names_to_label_files: bool = True
+            convert_names_to_label_files: bool = True,
+            parent_file_id: int = None
     ):
         """
         Create a Project file from local path
@@ -75,9 +76,8 @@ def from_local(
                 assume_new_instances_machine_made = assume_new_instances_machine_made,
                 convert_names_to_label_files = convert_names_to_label_files
             )
-
+        json_payload['parent_file_id'] = parent_file_id
         files['json'] = (None, json.dumps(json_payload), 'application/json')
-
         endpoint = "/api/walrus/v1/project/" + self.client.project_string_id \
                    + "/input/from_local"
 
@@ -106,7 +106,8 @@ def __build_packet_payload(self,
                                directory_id: int = None,
                                bucket_name: str = None,
                                file_name: str = None,
-                               blob_path: str = None):
+                               blob_path: str = None,
+                               parent_file_id: int = None):
         packet = {'media': {}}
         packet['media']['url'] = url
         packet['media']['type'] = media_type
@@ -115,6 +116,7 @@ def __build_packet_payload(self,
         packet['frame_packet_map'] = frame_packet_map
         packet['type'] = type
         packet['connection_id'] = connection_id
+        packet['parent_file_id'] = parent_file_id
         packet['directory_id'] = directory_id
         packet['original_filename'] = file_name
         packet['bucket_name'] = bucket_name
@@ -137,7 +139,8 @@ def from_blob_path(self,
                        media_type: str = 'image',
                        instance_list: list = None,
                        file_name: str = None,
-                       frame_packet_map: dict = None):
+                       frame_packet_map: dict = None,
+                       parent_file_id: int = None):
         """
           Bind a blob path in the given connection ID into Diffgram
         :param blob_path:
@@ -163,7 +166,8 @@ def from_blob_path(self,
             connection_id = connection_id,
             file_name = name,
             directory_id = directory_id,
-            type = "from_blob_path"
+            type = "from_blob_path",
+            parent_file_id=parent_file_id
         )
         self.from_packet(packet = packet)
         return True
@@ -176,7 +180,8 @@ def from_url(
             job_id: int = None,
             video_split_duration: int = None,
             instance_list: list = None,  # for Images
-            frame_packet_map: dict = None  # for Video
+            frame_packet_map: dict = None,  # for Video
+            parent_file_id: int = None
     ):
         """
 
@@ -208,7 +213,8 @@ def from_url(
             job_id = job_id,
             video_split_duration = video_split_duration,
             instance_list = instance_list,
-            frame_packet_map = frame_packet_map
+            frame_packet_map = frame_packet_map,
+            parent_file_id = parent_file_id
         )
         self.from_packet(packet = packet)
 

sdk/samples/import_compound_file.py

@@ -0,0 +1,17 @@
+from diffgram import Project
+from diffgram.file.compound_file import CompoundFile
+
+project = Project(host = "https://diffgram.com",
+		  project_string_id = "replace_with_project_string",
+		  client_id = "replace_with_client_id",
+		  client_secret = "replace_with_client_secret")
+
+parent = CompoundFile(
+    project=project, 
+    name='myFirstCompoundFile', 
+    directory_id=project.default_directory.id
+)
+
+parent.add_child_from_local(path='path/to/your_file.jpg')
+parent.add_child_from_local(path='path/to/your_second_file.jpg')
+parent.upload()