From c88b59894a4fe5b52589f18cb6735d7916ac1354 Mon Sep 17 00:00:00 2001
From: Geraldo Ramos <geraldorneto@gmail.com>
Date: Thu, 27 Oct 2022 20:10:17 +0000
Subject: [PATCH] implement upload feature

---
 README.md                                   |  9 ++++-
 maestro_worker_python/data/requirements.txt |  2 +-
 setup.py                                    | 20 ++++++----
 tests/upload_files.py                       | 44 +++++++++++++++++++++
 4 files changed, 65 insertions(+), 10 deletions(-)
 create mode 100644 tests/upload_files.py

diff --git a/README.md b/README.md
index dd50145..302b0ff 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ pip install git+https://github.com/moises-ai/maestro-worker-python.git
 
 To install a version (recommended):
 ```
-pip install git+https://github.com/moises-ai/maestro-worker-python.git@1.0.18
+pip install git+https://github.com/moises-ai/maestro-worker-python.git@1.0.19
 ```
 
 ## Maestro init
@@ -66,3 +66,10 @@ docker-compose build
 ```bash
 docker-compose run --service-ports worker
 ```
+
+### Developing this package
+Run in development mode:
+
+```bash
+python3 setup.py develop
+```
\ No newline at end of file
diff --git a/maestro_worker_python/data/requirements.txt b/maestro_worker_python/data/requirements.txt
index 4db1c44..9441cbf 100644
--- a/maestro_worker_python/data/requirements.txt
+++ b/maestro_worker_python/data/requirements.txt
@@ -1 +1 @@
-git+https://github.com/moises-ai/maestro-worker-python.git@1.0.18
\ No newline at end of file
+git+https://github.com/moises-ai/maestro-worker-python.git@1.0.19
\ No newline at end of file
diff --git a/setup.py b/setup.py
index dea8a78..48211ac 100644
--- a/setup.py
+++ b/setup.py
@@ -4,8 +4,9 @@
 
 setup(
     name="maestro_worker_python",
-    py_modules=["maestro_worker_python", "maestro_worker_python.worker_example"],
-    version="1.0.18",
+    py_modules=["maestro_worker_python",
+                "maestro_worker_python.worker_example"],
+    version="1.0.19",
     description="Utility to run workers on Moises/Maestro",
     readme="README.md",
     python_requires=">=3.8",
@@ -14,18 +15,21 @@
     license="MIT",
     packages=find_packages(exclude=["tests*"]),
     package_data={
-      'maestro_worker_python': ['data/*.*', 'data/models/*.*', 'data/Dockerfile'],
-   },
+        'maestro_worker_python': ['data/*.*', 'data/models/*.*', 'data/Dockerfile'],
+    },
     install_requires=[
         str(r)
         for r in pkg_resources.parse_requirements(
             open(os.path.join(os.path.dirname(__file__), "requirements.txt"))
         )
     ],
-    entry_points = {
-        'console_scripts': ['maestro-server=maestro_worker_python.server:main', 'maestro-cli=maestro_worker_python.cli:main', 'maestro-init=maestro_worker_python.init:main'],
-        
+    entry_points={
+        'console_scripts': ['maestro-server=maestro_worker_python.server:main',
+                            'maestro-cli=maestro_worker_python.cli:main',
+                            'maestro-init=maestro_worker_python.init:main'
+                            # 'maestro-validate=maestro_worker_python.validate:main'
+                            ],
     },
     include_package_data=True,
     extras_require={'dev': ['pytest']},
-)
\ No newline at end of file
+)
diff --git a/tests/upload_files.py b/tests/upload_files.py
new file mode 100644
index 0000000..2f738a1
--- /dev/null
+++ b/tests/upload_files.py
@@ -0,0 +1,44 @@
+
+import threading
+from dataclasses import dataclass
+from typing import List
+import requests
+import logging
+logging.basicConfig(level=logging.INFO)
+
+
+@dataclass
+class UploadFile:
+    file_path: str
+    file_type: str
+    signed_url: str
+
+
+def upload_files(upload_files: List[UploadFile]):
+    logging.info(f"Uploading {len(upload_files)} files")
+    threads_upload = []
+    did_raise_exception = threading.Event()
+    for file_ in upload_files:
+        t = threading.Thread(target=_upload, args=(
+            file_, did_raise_exception,))
+        threads_upload.append(t)
+        t.start()
+    
+    for t in threads_upload:
+        t.join()
+
+    if did_raise_exception.is_set():
+        raise Exception("Error during file Upload")
+
+def _upload(upload_file: UploadFile, did_raise_exception):
+    logging.info(f"Uploading:{upload_file.file_path}")
+    try:
+        with open(upload_file.file_path, 'rb') as data:
+            response = requests.put(upload_file.signed_url, data=data, headers={
+                "Content-Type": upload_file.file_type})
+            response.raise_for_status()
+            logging.info(f"Uploaded {upload_file.file_path}")
+    except Exception as e:
+        did_raise_exception.set()
+        logging.exception(e)
+        raise e
\ No newline at end of file