implement upload feature

README.md

@@ -10,7 +10,7 @@ pip install git+https://github.com/moises-ai/maestro-worker-python.git
 
 To install a version (recommended):
 ```
-pip install git+https://github.com/moises-ai/maestro-worker-python.git@1.0.18
+pip install git+https://github.com/moises-ai/maestro-worker-python.git@1.0.19
 ```
 
 ## Maestro init
@@ -66,3 +66,10 @@ docker-compose build
 ```bash
 docker-compose run --service-ports worker
 ```
+
+### Developing this package
+Run in development mode:
+
+```bash
+python3 setup.py develop
+```

maestro_worker_python/data/requirements.txt

@@ -1 +1 @@
-git+https://github.com/moises-ai/maestro-worker-python.git@1.0.18
+git+https://github.com/moises-ai/maestro-worker-python.git@1.0.19

setup.py

@@ -4,8 +4,9 @@
 
 setup(
     name="maestro_worker_python",
-    py_modules=["maestro_worker_python", "maestro_worker_python.worker_example"],
-    version="1.0.18",
+    py_modules=["maestro_worker_python",
+                "maestro_worker_python.worker_example"],
+    version="1.0.19",
     description="Utility to run workers on Moises/Maestro",
     readme="README.md",
     python_requires=">=3.8",
@@ -14,18 +15,21 @@
     license="MIT",
     packages=find_packages(exclude=["tests*"]),
     package_data={
-      'maestro_worker_python': ['data/*.*', 'data/models/*.*', 'data/Dockerfile'],
-   },
+        'maestro_worker_python': ['data/*.*', 'data/models/*.*', 'data/Dockerfile'],
+    },
     install_requires=[
         str(r)
         for r in pkg_resources.parse_requirements(
             open(os.path.join(os.path.dirname(__file__), "requirements.txt"))
         )
     ],
-    entry_points = {
-        'console_scripts': ['maestro-server=maestro_worker_python.server:main', 'maestro-cli=maestro_worker_python.cli:main', 'maestro-init=maestro_worker_python.init:main'],
-
+    entry_points={
+        'console_scripts': ['maestro-server=maestro_worker_python.server:main',
+                            'maestro-cli=maestro_worker_python.cli:main',
+                            'maestro-init=maestro_worker_python.init:main'
+                            # 'maestro-validate=maestro_worker_python.validate:main'
+                            ],
     },
     include_package_data=True,
     extras_require={'dev': ['pytest']},
-)
+)

tests/upload_files.py

@@ -0,0 +1,44 @@
+
+import threading
+from dataclasses import dataclass
+from typing import List
+import requests
+import logging
+logging.basicConfig(level=logging.INFO)
+
+
+@dataclass
+class UploadFile:
+    file_path: str
+    file_type: str
+    signed_url: str
+
+
+def upload_files(upload_files: List[UploadFile]):
+    logging.info(f"Uploading {len(upload_files)} files")
+    threads_upload = []
+    did_raise_exception = threading.Event()
+    for file_ in upload_files:
+        t = threading.Thread(target=_upload, args=(
+            file_, did_raise_exception,))
+        threads_upload.append(t)
+        t.start()
+
+    for t in threads_upload:
+        t.join()
+
+    if did_raise_exception.is_set():
+        raise Exception("Error during file Upload")
+
+def _upload(upload_file: UploadFile, did_raise_exception):
+    logging.info(f"Uploading:{upload_file.file_path}")
+    try:
+        with open(upload_file.file_path, 'rb') as data:
+            response = requests.put(upload_file.signed_url, data=data, headers={
+                "Content-Type": upload_file.file_type})
+            response.raise_for_status()
+            logging.info(f"Uploaded {upload_file.file_path}")
+    except Exception as e:
+        did_raise_exception.set()
+        logging.exception(e)
+        raise e