From 77d3985a4f67b5938fd40f4c514853114e191261 Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Fri, 26 Apr 2024 15:38:50 +0200 Subject: [PATCH 01/24] fix: make description optional --- deepaas/cmd/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index e59971eb..c4978360 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -110,7 +110,7 @@ def _fields_to_dict(fields_in): param["default"] = val.missing # infer 'help' - val_help = val.metadata["description"] + val_help = val.metadata.get("description", "") # argparse hates % sign: if "%" in val_help: # replace single occurancies of '%' with '%%' @@ -118,7 +118,7 @@ def _fields_to_dict(fields_in): val_help = re.sub(r"(? Date: Mon, 6 May 2024 15:59:17 +0200 Subject: [PATCH 02/24] feat: allow multiple input files --- deepaas/cmd/cli.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index c4978360..6ee6e151 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -60,6 +60,7 @@ fields.URL: str, fields.Url: str, fields.UUID: str, + fields.Field: str, } @@ -159,6 +160,18 @@ def _get_model_name(model_name=None): sys.exit(1) +def _get_file_args(fields_in): + """Function to retrieve a list of file-type fields + :param fields_in: mashmallow fields + :return: list + """ + file_fields = [] + for k, v in fields_in.items(): + if type(v) is fields.Field: + file_fields.append(k) + return file_fields + + # Get the model name model_name = CONF.model_name @@ -174,6 +187,10 @@ def _get_model_name(model_name=None): predict_args = _fields_to_dict(model_obj.get_predict_args()) train_args = _fields_to_dict(model_obj.get_train_args()) +# Find which of the arguments are going to be files +file_args = {} +file_args['predict'] = _get_file_args(model_obj.get_predict_args()) +file_args['train'] = _get_file_args(model_obj.get_train_args()) # Function to add later these arguments to CONF via SubCommandOpt def _add_methods(subparsers): @@ -291,29 +308,31 @@ def main(): if CONF.deepaas_with_multiprocessing: mp.set_start_method("spawn", force=True) - # TODO(multi-file): change to many files ('for' itteration) - if CONF.methods.__contains__("files"): - if CONF.methods.files: + # Create file wrapper for file args (if provided) + for farg in file_args[CONF.methods.name]: + if getattr(CONF.methods, farg, None): + fpath = conf_vars[farg] + # create tmp file as later it supposed # to be deleted by the application temp = tempfile.NamedTemporaryFile() temp.close() # copy original file into tmp file - with open(conf_vars["files"], "rb") as f: + with open(fpath, "rb") as f: with open(temp.name, "wb") as f_tmp: for line in f: f_tmp.write(line) # create file object - file_type = mimetypes.MimeTypes().guess_type(conf_vars["files"])[0] + file_type = mimetypes.MimeTypes().guess_type(fpath)[0] file_obj = v2_wrapper.UploadedFile( name="data", filename=temp.name, content_type=file_type, - original_filename=conf_vars["files"], + original_filename=fpath, ) - # re-write 'files' parameter in conf_vars - conf_vars["files"] = file_obj + # re-write parameter in conf_vars + conf_vars[farg] = file_obj # debug of input parameters LOG.debug("[DEBUG provided options, conf_vars]: {}".format(conf_vars)) From 28f2127ceeec17d81beeca434290eb44c9b2db67 Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Tue, 14 May 2024 12:07:08 +0200 Subject: [PATCH 03/24] refactor: remove unneeded check We are already mapping these fields --- deepaas/cmd/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 6ee6e151..c3a1b4a5 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -95,9 +95,6 @@ def _fields_to_dict(fields_in): if val_type in FIELD_TYPE_CONVERTERS: param["type"] = FIELD_TYPE_CONVERTERS[val_type] - if key == "files" or key == "urls": - param["type"] = str - # infer "required" try: val_req = val.required From 4fa01aaad7a559e9e18fb5706f789d012fd2da73 Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Tue, 14 May 2024 12:08:34 +0200 Subject: [PATCH 04/24] feat: allow linebreaks in help messages --- deepaas/cmd/cli.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index c3a1b4a5..76eff0c7 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -17,6 +17,7 @@ # import asyncio +import argparse import deepaas import json import mimetypes @@ -116,7 +117,11 @@ def _fields_to_dict(fields_in): val_help = re.sub(r"(? Date: Tue, 14 May 2024 13:55:58 +0200 Subject: [PATCH 05/24] feat: add required info to help message --- deepaas/cmd/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 76eff0c7..cf9bc6aa 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -122,6 +122,9 @@ def _fields_to_dict(fields_in): if val_type is fields.Field: val_help += "\nType: FILEPATH" + if val_req: + val_help += "\n*Required*" + param["help"] = val_help dict_out[key] = param From c4b9a57fa4bb1964686bbb958e4b7bc42c85ea57 Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Tue, 14 May 2024 14:11:28 +0200 Subject: [PATCH 06/24] feat: add types to help message --- deepaas/cmd/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index cf9bc6aa..1a95b5ed 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -119,12 +119,14 @@ def _fields_to_dict(fields_in): if "enum" in val.metadata.keys(): val_help += f"\nChoices: {val.metadata['enum']}" + val_help += f"\nType: {param['type'].__name__}" if val_type is fields.Field: - val_help += "\nType: FILEPATH" + val_help += " (filepath)" if val_req: val_help += "\n*Required*" + val_help = val_help.lstrip('\n') # remove escape when no description found param["help"] = val_help dict_out[key] = param From f52101d52690dfc7235336f740bea4aea514890f Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Thu, 16 May 2024 13:41:42 +0200 Subject: [PATCH 07/24] feat: properly support lists and dicts in argparse --- deepaas/cmd/cli.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 1a95b5ed..4428f80f 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -18,6 +18,7 @@ # import asyncio import argparse +import ast import deepaas import json import mimetypes @@ -44,17 +45,21 @@ # Not all types are covered! If not listed, the type is 'str' # see https://marshmallow.readthedocs.io/en/stable/marshmallow.fields.html + +# Passing lists or dicts with argparse is not straight forward, so we use pass them as +# string and parse them with `ast.literal_eval` +# ref: https://stackoverflow.com/questions/7625786/type-dict-in-argparse-add-argument FIELD_TYPE_CONVERTERS = { fields.Bool: bool, fields.Boolean: bool, fields.Date: str, fields.DateTime: str, - fields.Dict: dict, + fields.Dict: ast.literal_eval, fields.Email: str, fields.Float: float, fields.Int: int, fields.Integer: int, - fields.List: list, + fields.List: ast.literal_eval, fields.Str: str, fields.String: str, fields.Time: str, @@ -119,7 +124,13 @@ def _fields_to_dict(fields_in): if "enum" in val.metadata.keys(): val_help += f"\nChoices: {val.metadata['enum']}" - val_help += f"\nType: {param['type'].__name__}" + if val_type is fields.List: + val_help += '\nType: list, enclosed as string: "[...]"' + elif val_type is fields.Dict: + val_help += '\nType: dict, enclosed as string: "{...}"' + else: + val_help += f"\nType: {param['type'].__name__}" + if val_type is fields.Field: val_help += " (filepath)" From 216f71a366a5e3a83d6c30f4841c20d7ae27479e Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Thu, 16 May 2024 14:00:16 +0200 Subject: [PATCH 08/24] style: organize `_fields_to_dict` code --- deepaas/cmd/cli.py | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 4428f80f..bddfb70b 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -94,35 +94,18 @@ def _fields_to_dict(fields_in): # initialise param with no "default", type "str" (!), empty "help" param = {"default": None, "type": str, "help": ""} - # infer "type" - # see FIELD_TYPE_CONVERTERS for converting - # mashmallow field types to python types - val_type = type(val) - if val_type in FIELD_TYPE_CONVERTERS: - param["type"] = FIELD_TYPE_CONVERTERS[val_type] - - # infer "required" - try: - val_req = val.required - except Exception: - val_req = False - param["required"] = val_req - - # infer 'default' - # if the field is not required, there must be default value - if not val_req: - param["default"] = val.missing - - # infer 'help' + # Initialize help string val_help = val.metadata.get("description", "") - # argparse hates % sign: if "%" in val_help: + # argparse hates % sign: # replace single occurancies of '%' with '%%' # since "%%"" is accepted by argparse val_help = re.sub(r"(? Date: Fri, 17 May 2024 11:08:15 +0200 Subject: [PATCH 09/24] feat: support bools --- deepaas/cmd/cli.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index bddfb70b..20b3b7ef 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -49,9 +49,17 @@ # Passing lists or dicts with argparse is not straight forward, so we use pass them as # string and parse them with `ast.literal_eval` # ref: https://stackoverflow.com/questions/7625786/type-dict-in-argparse-add-argument +# We follow a similar approach with bools +# ref: https://stackoverflow.com/a/59579733/18471590 +# ref: https://stackoverflow.com/questions/715417/converting-from-a-string-to-boolean-in-python/18472142#18472142 + +def str2bool(v): + return v.lower() in ("yes", "true", "t", "1") + + FIELD_TYPE_CONVERTERS = { - fields.Bool: bool, - fields.Boolean: bool, + fields.Bool: str2bool, + fields.Boolean: str2bool, fields.Date: str, fields.DateTime: str, fields.Dict: ast.literal_eval, @@ -111,6 +119,8 @@ def _fields_to_dict(fields_in): val_help += '\nType: list, enclosed as string: "[...]"' elif val_type is fields.Dict: val_help += '\nType: dict, enclosed as string: "{...}"' + elif val_type in [fields.Bool, fields.Boolean]: + val_help += '\nType: bool' else: val_help += f"\nType: {param['type'].__name__}" if val_type is fields.Field: From 13e61053e76bf3fc93f645b5bcd9a5ec765512a8 Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Fri, 17 May 2024 11:53:15 +0200 Subject: [PATCH 10/24] style: fix flake8 --- deepaas/cmd/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 20b3b7ef..82fbb68c 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -51,10 +51,11 @@ # ref: https://stackoverflow.com/questions/7625786/type-dict-in-argparse-add-argument # We follow a similar approach with bools # ref: https://stackoverflow.com/a/59579733/18471590 -# ref: https://stackoverflow.com/questions/715417/converting-from-a-string-to-boolean-in-python/18472142#18472142 +# ref: https://stackoverflow.com/questions/715417/converting-from-a-string-to-boolean-in-python/18472142#18472142 # noqa + def str2bool(v): - return v.lower() in ("yes", "true", "t", "1") + return v.lower() in ("yes", "true", "t", "1") FIELD_TYPE_CONVERTERS = { @@ -138,7 +139,7 @@ def _fields_to_dict(fields_in): if "enum" in val.metadata.keys(): val_help += f"\nChoices: {val.metadata['enum']}" - val_help = val_help.lstrip('\n') # remove escape when no description found + val_help = val_help.lstrip('\n') # remove escape when no description found param["help"] = val_help dict_out[key] = param @@ -210,6 +211,7 @@ def _get_file_args(fields_in): file_args['predict'] = _get_file_args(model_obj.get_predict_args()) file_args['train'] = _get_file_args(model_obj.get_train_args()) + # Function to add later these arguments to CONF via SubCommandOpt def _add_methods(subparsers): """Function to add argparse subparsers via SubCommandOpt (see below) From 53a4c322987bcd2c8e5041253dfbd3cc81236d78 Mon Sep 17 00:00:00 2001 From: Ignacio Heredia Date: Fri, 17 May 2024 11:59:20 +0200 Subject: [PATCH 11/24] style: fix black --- deepaas/cmd/cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 82fbb68c..e2e517fb 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -121,7 +121,7 @@ def _fields_to_dict(fields_in): elif val_type is fields.Dict: val_help += '\nType: dict, enclosed as string: "{...}"' elif val_type in [fields.Bool, fields.Boolean]: - val_help += '\nType: bool' + val_help += "\nType: bool" else: val_help += f"\nType: {param['type'].__name__}" if val_type is fields.Field: @@ -139,7 +139,7 @@ def _fields_to_dict(fields_in): if "enum" in val.metadata.keys(): val_help += f"\nChoices: {val.metadata['enum']}" - val_help = val_help.lstrip('\n') # remove escape when no description found + val_help = val_help.lstrip("\n") # remove escape when no description found param["help"] = val_help dict_out[key] = param @@ -208,8 +208,8 @@ def _get_file_args(fields_in): # Find which of the arguments are going to be files file_args = {} -file_args['predict'] = _get_file_args(model_obj.get_predict_args()) -file_args['train'] = _get_file_args(model_obj.get_train_args()) +file_args["predict"] = _get_file_args(model_obj.get_predict_args()) +file_args["train"] = _get_file_args(model_obj.get_train_args()) # Function to add later these arguments to CONF via SubCommandOpt @@ -217,6 +217,7 @@ def _add_methods(subparsers): """Function to add argparse subparsers via SubCommandOpt (see below) for DEEPaaS methods get_metadata, warm, predict, train """ + # Use RawTextHelpFormatter to allow for line breaks in argparse help messages. def help_formatter(prog): return argparse.RawTextHelpFormatter(prog, max_help_position=10) From c51c3b44119bf68d69af6e15bff4a06c5c72e9ba Mon Sep 17 00:00:00 2001 From: root Date: Mon, 15 Jul 2024 17:51:46 +0000 Subject: [PATCH 12/24] fix: studying apispec for fields conversion --- deepaas/cmd/cli.py | 91 +++++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index e2e517fb..04a7ddeb 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -28,10 +28,11 @@ import shutil import sys import tempfile +import webargs import uuid from datetime import datetime -from marshmallow import fields +from marshmallow import fields #marshmallow from oslo_config import cfg from oslo_log import log @@ -189,6 +190,12 @@ def _get_file_args(fields_in): for k, v in fields_in.items(): if type(v) is fields.Field: file_fields.append(k) + print(f"type of {v} is {type(v)}, key: {k}") + try: + print(f"file? {getattr(v, 'type')}") + except: + print("no type found") + return file_fields @@ -206,6 +213,37 @@ def _get_file_args(fields_in): predict_args = _fields_to_dict(model_obj.get_predict_args()) train_args = _fields_to_dict(model_obj.get_train_args()) +p_schema = webargs.core.dict2schema(model_obj.get_predict_args()) +print(f"Args: {model_obj.get_predict_args()}") + +print(f"Schema: {p_schema}") +#print(f"Fields: {p_schema.fields}") +Meta = getattr(p_schema, "Meta", None) +print(f"Meta: {Meta}") + +### +from pprint import pprint + +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin + +spec = APISpec( + title="AI4OS Module", + version="0.1.0", + openapi_version="3.0.2", + plugins=[MarshmallowPlugin()], +) + +spec.components.schema("Module", schema=p_schema) +method_components = spec.to_dict()["components"]["schemas"] +params = method_components["Module"]["properties"] +required = method_components["Module"]["required"] +print("params:") +pprint(params) +print("required:") +pprint(required) +## + # Find which of the arguments are going to be files file_args = {} file_args["predict"] = _get_file_args(model_obj.get_predict_args()) @@ -339,31 +377,32 @@ def main(): if CONF.deepaas_with_multiprocessing: mp.set_start_method("spawn", force=True) - # Create file wrapper for file args (if provided) - for farg in file_args[CONF.methods.name]: - if getattr(CONF.methods, farg, None): - fpath = conf_vars[farg] - - # create tmp file as later it supposed - # to be deleted by the application - temp = tempfile.NamedTemporaryFile() - temp.close() - # copy original file into tmp file - with open(fpath, "rb") as f: - with open(temp.name, "wb") as f_tmp: - for line in f: - f_tmp.write(line) - - # create file object - file_type = mimetypes.MimeTypes().guess_type(fpath)[0] - file_obj = v2_wrapper.UploadedFile( - name="data", - filename=temp.name, - content_type=file_type, - original_filename=fpath, - ) - # re-write parameter in conf_vars - conf_vars[farg] = file_obj + if CONF.methods.name == "predict" or CONF.methods.name == "train": + # Create file wrapper for file args (if provided) + for farg in file_args[CONF.methods.name]: + if getattr(CONF.methods, farg, None): + fpath = conf_vars[farg] + + # create tmp file as later it supposed + # to be deleted by the application + temp = tempfile.NamedTemporaryFile() + temp.close() + # copy original file into tmp file + with open(fpath, "rb") as f: + with open(temp.name, "wb") as f_tmp: + for line in f: + f_tmp.write(line) + + # create file object + file_type = mimetypes.MimeTypes().guess_type(fpath)[0] + file_obj = v2_wrapper.UploadedFile( + name="data", + filename=temp.name, + content_type=file_type, + original_filename=fpath, + ) + # re-write parameter in conf_vars + conf_vars[farg] = file_obj # debug of input parameters LOG.debug("[DEBUG provided options, conf_vars]: {}".format(conf_vars)) From 12a1d4654f2c65c1fde4165e635154a1afdb7560 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 15 Jul 2024 19:34:37 +0000 Subject: [PATCH 13/24] fix: store BytesIO object in file --- deepaas/cmd/cli.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 04a7ddeb..3d7be47b 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -20,6 +20,7 @@ import argparse import ast import deepaas +import io import json import mimetypes import multiprocessing as mp @@ -214,7 +215,7 @@ def _get_file_args(fields_in): train_args = _fields_to_dict(model_obj.get_train_args()) p_schema = webargs.core.dict2schema(model_obj.get_predict_args()) -print(f"Args: {model_obj.get_predict_args()}") +#print(f"Args: {model_obj.get_predict_args()}") print(f"Schema: {p_schema}") #print(f"Fields: {p_schema.fields}") @@ -230,7 +231,7 @@ def _get_file_args(fields_in): spec = APISpec( title="AI4OS Module", version="0.1.0", - openapi_version="3.0.2", + openapi_version="3.1.0", plugins=[MarshmallowPlugin()], ) @@ -369,7 +370,7 @@ def main(): log.setup(CONF, "deepaas-cli") - LOG.info("[INFO, Method] {} was called.".format(CONF.methods.name)) + LOG.info("[Method] {} was called.".format(CONF.methods.name)) # put all variables in dict, makes life easier... conf_vars = vars(CONF._namespace) @@ -405,12 +406,12 @@ def main(): conf_vars[farg] = file_obj # debug of input parameters - LOG.debug("[DEBUG provided options, conf_vars]: {}".format(conf_vars)) + LOG.debug("[provided options, conf_vars]: {}".format(conf_vars)) if CONF.methods.name == "get_metadata": meta = model_obj.get_metadata() meta_json = json.dumps(meta) - LOG.debug("[DEBUG, get_metadata, Output]: {}".format(meta_json)) + LOG.debug("[get_metadata]: {}".format(meta_json)) if CONF.deepaas_method_output: _store_output(meta_json, CONF.deepaas_method_output) @@ -419,7 +420,7 @@ def main(): elif CONF.methods.name == "warm": # await model_obj.warm() model_obj.warm() - LOG.info("[INFO, warm] Finished warm() method") + LOG.info("[warm] Finished warm() method") elif CONF.methods.name == "predict": # call predict method @@ -449,7 +450,7 @@ def main(): ): # noqa: W503 out_file = out_file + extension LOG.warn( - "[WARNING] You are trying to store {} " + "You are trying to store {} " "type data in the file " "with {} extension!\n" "===================> " @@ -457,15 +458,18 @@ def main(): ) if extension == ".json" or extension is None: results_json = json.dumps(task) - LOG.debug("[DEBUG, predict, Output]: {}".format(results_json)) + LOG.debug("[predict]: {}".format(results_json)) f = open(out_file, "w+") f.write(results_json) f.close() + elif type(task) is io.BytesIO: + with open(out_file, "wb") as f: + f.write(task.getbuffer()) + else: - out_results = task.name - shutil.copy(out_results, out_file) + LOG.info(f"Output of type type({task}), {task}") - LOG.info("[INFO, Output] Output is saved in {}".format(out_file)) + LOG.info("Output is saved in {}".format(out_file)) return task @@ -488,17 +492,17 @@ def main(): # we assume that train always returns JSON ret["result"]["output"] = task results_json = json.dumps(ret) - LOG.info("[INFO] Elapsed time: %s", str(end - start)) - LOG.debug("[DEBUG, train, Output]: {}".format(results_json)) + LOG.info("Elapsed time: %s", str(end - start)) + LOG.debug("[train]: {}".format(results_json)) if CONF.deepaas_method_output: _store_output(results_json, CONF.deepaas_method_output) return results_json else: - LOG.warn("[WARNING] No Method was requested! Return get_metadata()") + LOG.warn("No Method was requested! Return get_metadata()") meta = model_obj.get_metadata() meta_json = json.dumps(meta) - LOG.debug("[DEBUG, get_metadata, Output]: {}".format(meta_json)) + LOG.debug("get_metadata]: {}".format(meta_json)) return meta_json From 1d9facda7229ff1df1830a08b8e03be3e9269d9f Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Mon, 26 Aug 2024 10:19:28 +0000 Subject: [PATCH 14/24] fix: prepare deepaas-cli for the release 2.x --- deepaas/cmd/cli.py | 99 ++++++++++++++-------------------------------- 1 file changed, 29 insertions(+), 70 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 3d7be47b..49c0ea8f 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -43,6 +43,8 @@ CONF = config.CONF +LOG = log.getLogger(__name__) + debug_cli = False # Not all types are covered! If not listed, the type is 'str' @@ -77,7 +79,7 @@ def str2bool(v): fields.URL: str, fields.Url: str, fields.UUID: str, - fields.Field: str, + #fields.Field: str, } @@ -127,7 +129,10 @@ def _fields_to_dict(fields_in): else: val_help += f"\nType: {param['type'].__name__}" if val_type is fields.Field: - val_help += " (filepath)" + if val.metadata.get("type", "") == "file": + val_help += " (filepath)" + else: + val_help += " (WARNING: original type fields.Field)" # Infer "required" param["required"] = val.required @@ -148,6 +153,20 @@ def _fields_to_dict(fields_in): return dict_out +# Function to detect file arguments +def _get_file_args(fields_in): + """Function to retrieve a list of file-type fields + :param fields_in: mashmallow fields + :return: list + """ + file_fields = [] + for k, v in fields_in.items(): + if type(v) is fields.Field: + if v.metadata.get("type", "") == "file": + file_fields.append(k) + + return file_fields + # Function to get a model object def _get_model_name(model_name=None): @@ -182,27 +201,8 @@ def _get_model_name(model_name=None): sys.exit(1) -def _get_file_args(fields_in): - """Function to retrieve a list of file-type fields - :param fields_in: mashmallow fields - :return: list - """ - file_fields = [] - for k, v in fields_in.items(): - if type(v) is fields.Field: - file_fields.append(k) - print(f"type of {v} is {type(v)}, key: {k}") - try: - print(f"file? {getattr(v, 'type')}") - except: - print("no type found") - - return file_fields - - # Get the model name model_name = CONF.model_name - model_name, model_obj = _get_model_name(model_name) # use deepaas.model.v2.wrapper.ModelWrapper(). deepaas>1.2.1dev4 @@ -214,37 +214,6 @@ def _get_file_args(fields_in): predict_args = _fields_to_dict(model_obj.get_predict_args()) train_args = _fields_to_dict(model_obj.get_train_args()) -p_schema = webargs.core.dict2schema(model_obj.get_predict_args()) -#print(f"Args: {model_obj.get_predict_args()}") - -print(f"Schema: {p_schema}") -#print(f"Fields: {p_schema.fields}") -Meta = getattr(p_schema, "Meta", None) -print(f"Meta: {Meta}") - -### -from pprint import pprint - -from apispec import APISpec -from apispec.ext.marshmallow import MarshmallowPlugin - -spec = APISpec( - title="AI4OS Module", - version="0.1.0", - openapi_version="3.1.0", - plugins=[MarshmallowPlugin()], -) - -spec.components.schema("Module", schema=p_schema) -method_components = spec.to_dict()["components"]["schemas"] -params = method_components["Module"]["properties"] -required = method_components["Module"]["required"] -print("params:") -pprint(params) -print("required:") -pprint(required) -## - # Find which of the arguments are going to be files file_args = {} file_args["predict"] = _get_file_args(model_obj.get_predict_args()) @@ -331,13 +300,8 @@ def help_formatter(prog): ), ] - -CONF = cfg.CONF CONF.register_cli_opts(cli_opts) -LOG = log.getLogger(__name__) - - # store DEEPAAS_METHOD output in a file def _store_output(results, out_file): """Function to store model results in the file @@ -349,14 +313,12 @@ def _store_output(results, out_file): if not os.path.exists(out_path): # Create path if does not exist os.makedirs(out_path) - f = open(out_file, "w+") - f.write(results) - f.close() + with open(out_file, "w+") as f: + f.write(results) - LOG.info("[INFO, Output] Output is saved in {}".format(out_file)) + LOG.info("Output is saved in {}".format(out_file)) -# async def main(): def main(): """Executes model's methods with corresponding parameters""" @@ -367,10 +329,9 @@ def main(): log.set_defaults(default_log_levels=log.get_default_log_levels()) CONF(sys.argv[1:], project="deepaas", version=deepaas.__version__) - log.setup(CONF, "deepaas-cli") - LOG.info("[Method] {} was called.".format(CONF.methods.name)) + LOG.info("{} was called.".format(CONF.methods.name)) # put all variables in dict, makes life easier... conf_vars = vars(CONF._namespace) @@ -378,8 +339,9 @@ def main(): if CONF.deepaas_with_multiprocessing: mp.set_start_method("spawn", force=True) + # Create file wrapper for file args, ONLY for "predict()" and "train()" if CONF.methods.name == "predict" or CONF.methods.name == "train": - # Create file wrapper for file args (if provided) + # Create file wrapper for file args (if provided!) for farg in file_args[CONF.methods.name]: if getattr(CONF.methods, farg, None): fpath = conf_vars[farg] @@ -459,13 +421,10 @@ def main(): if extension == ".json" or extension is None: results_json = json.dumps(task) LOG.debug("[predict]: {}".format(results_json)) - f = open(out_file, "w+") - f.write(results_json) - f.close() + _store_output(results_json, out_file) elif type(task) is io.BytesIO: with open(out_file, "wb") as f: f.write(task.getbuffer()) - else: LOG.info(f"Output of type type({task}), {task}") @@ -502,7 +461,7 @@ def main(): LOG.warn("No Method was requested! Return get_metadata()") meta = model_obj.get_metadata() meta_json = json.dumps(meta) - LOG.debug("get_metadata]: {}".format(meta_json)) + LOG.debug("[get_metadata]: {}".format(meta_json)) return meta_json From 8b71eafe6d8e8d55394a7d5bd4ca3c38e3fc15c1 Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Mon, 26 Aug 2024 12:19:03 +0000 Subject: [PATCH 15/24] fix: merge from releases/2.x, replace return with log.info() --- deepaas/cmd/cli.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 1a7fc8bb..e2d52edc 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -201,18 +201,6 @@ def _get_model_name(model_name=None): sys.exit(1) -def _get_file_args(fields_in): - """Function to retrieve a list of file-type fields - :param fields_in: mashmallow fields - :return: list - """ - file_fields = [] - for k, v in fields_in.items(): - if type(v) is fields.Field: - file_fields.append(k) - return file_fields - - # Get the model name model_name = CONF.model_name model_name, model_obj = _get_model_name(model_name) From ce1a46ae6a67cfeafe3fa9bbd18feb112115b477 Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Mon, 26 Aug 2024 22:35:32 +0000 Subject: [PATCH 16/24] fix: merge from releases/2.x, replace return with log.info() --- deepaas/cmd/cli.py | 5 ++--- deepaas/tests/test_cmd.py | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index e2d52edc..2a029b8f 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -175,7 +175,7 @@ def _get_model_name(model_name=None): :param model_name: name of the model :return: model object """ - + model_name = CONF.model_name models = loading.get_available_models("v2") if model_name: model_obj = models.get(model_name) @@ -202,8 +202,7 @@ def _get_model_name(model_name=None): # Get the model name -model_name = CONF.model_name -model_name, model_obj = _get_model_name(model_name) +model_name, model_obj = _get_model_name() # use deepaas.model.v2.wrapper.ModelWrapper(). deepaas>1.2.1dev4 # model_obj = v2_wrapper.ModelWrapper(name=model_name, diff --git a/deepaas/tests/test_cmd.py b/deepaas/tests/test_cmd.py index 10029284..c724c996 100644 --- a/deepaas/tests/test_cmd.py +++ b/deepaas/tests/test_cmd.py @@ -21,6 +21,7 @@ import mock +from deepaas.cmd import cli from deepaas.cmd import execute from deepaas.cmd import run from deepaas.tests import base @@ -63,6 +64,14 @@ def test_run_custom_ip_port(self, m_get_app, m_run_app, m_handle_signals): ) m_handle_signals.assert_called_once() +# W-I-P +class TestCLI(base.TestCase): + @mock.patch("deepaas.cmd.cli.main") + def test_cli_get_metadata(self, m_cli_get_meta): + self.flags(methods="get_metadata") + with mock.patch.object(sys, "argv", ["deepaas-cli"]): + cli.main() + m_cli_get_meta.assert_called_once() class TestExecute(base.TestCase): @mock.patch("deepaas.cmd.execute.prediction") From a0a178010d37a1df3b3d5976d7646764cf49d20f Mon Sep 17 00:00:00 2001 From: "valentin.kozlov" Date: Mon, 26 Aug 2024 23:18:39 +0000 Subject: [PATCH 17/24] fix: remove W-I-P test for cli --- deepaas/tests/test_cmd.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deepaas/tests/test_cmd.py b/deepaas/tests/test_cmd.py index c724c996..08d6d297 100644 --- a/deepaas/tests/test_cmd.py +++ b/deepaas/tests/test_cmd.py @@ -21,7 +21,7 @@ import mock -from deepaas.cmd import cli +#from deepaas.cmd import cli # W-I-P from deepaas.cmd import execute from deepaas.cmd import run from deepaas.tests import base @@ -65,13 +65,13 @@ def test_run_custom_ip_port(self, m_get_app, m_run_app, m_handle_signals): m_handle_signals.assert_called_once() # W-I-P -class TestCLI(base.TestCase): - @mock.patch("deepaas.cmd.cli.main") - def test_cli_get_metadata(self, m_cli_get_meta): - self.flags(methods="get_metadata") - with mock.patch.object(sys, "argv", ["deepaas-cli"]): - cli.main() - m_cli_get_meta.assert_called_once() +#class TestCLI(base.TestCase): +# @mock.patch("deepaas.cmd.cli.main") +# def test_cli_get_metadata(self, m_cli_get_meta): +# self.flags(methods="get_metadata") +# with mock.patch.object(sys, "argv", ["deepaas-cli"]): +# cli.main() +# m_cli_get_meta.assert_called_once() class TestExecute(base.TestCase): @mock.patch("deepaas.cmd.execute.prediction") From 330db841ec13192daee1a1d50082852ca7982490 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:47:49 +0200 Subject: [PATCH 18/24] tests: move to Python 3.12 --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 4b13acde..d2cd137a 100644 --- a/tox.ini +++ b/tox.ini @@ -16,11 +16,11 @@ python = 3.8: py38 3.9: py39 3.10: py310 - 3.11: py311, flake8, black, bandit, pip-missing-reqs, pypi - 3.12: py312 + 3.11: py311 + 3.12: py312, flake8, black, bandit, pip-missing-reqs, pypi [base] -python = python3.11 +python = python3.12 package = deepaas [pytest] From 56138d1173c2acc2123eaf9f1609824bbe9115b2 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:48:47 +0200 Subject: [PATCH 19/24] style: make black happy --- deepaas/cmd/cli.py | 4 +++- deepaas/tests/test_cmd.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 2a029b8f..e147db2a 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -33,7 +33,7 @@ import uuid from datetime import datetime -from marshmallow import fields #marshmallow +from marshmallow import fields # marshmallow from oslo_config import cfg from oslo_log import log @@ -153,6 +153,7 @@ def _fields_to_dict(fields_in): return dict_out + # Function to detect file arguments def _get_file_args(fields_in): """Function to retrieve a list of file-type fields @@ -300,6 +301,7 @@ def help_formatter(prog): CONF.register_cli_opts(cli_opts) + # store DEEPAAS_METHOD output in a file def _store_output(results, out_file): """Function to store model results in the file diff --git a/deepaas/tests/test_cmd.py b/deepaas/tests/test_cmd.py index 08d6d297..b09758bc 100644 --- a/deepaas/tests/test_cmd.py +++ b/deepaas/tests/test_cmd.py @@ -21,7 +21,7 @@ import mock -#from deepaas.cmd import cli # W-I-P +# from deepaas.cmd import cli # W-I-P from deepaas.cmd import execute from deepaas.cmd import run from deepaas.tests import base @@ -64,8 +64,9 @@ def test_run_custom_ip_port(self, m_get_app, m_run_app, m_handle_signals): ) m_handle_signals.assert_called_once() + # W-I-P -#class TestCLI(base.TestCase): +# class TestCLI(base.TestCase): # @mock.patch("deepaas.cmd.cli.main") # def test_cli_get_metadata(self, m_cli_get_meta): # self.flags(methods="get_metadata") @@ -73,6 +74,7 @@ def test_run_custom_ip_port(self, m_get_app, m_run_app, m_handle_signals): # cli.main() # m_cli_get_meta.assert_called_once() + class TestExecute(base.TestCase): @mock.patch("deepaas.cmd.execute.prediction") def test_execute_data(self, m_out_pred): From 4c9c5ded42b2b679c354b2a1d0c2f4e83c7cbb55 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:49:24 +0200 Subject: [PATCH 20/24] fix: remove unused imports --- deepaas/cmd/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index e147db2a..3c83fa01 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -26,10 +26,8 @@ import multiprocessing as mp import os import re -import shutil import sys import tempfile -import webargs import uuid from datetime import datetime From 363766a4cc669785c0f35baa95cf1a55b4770de8 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:49:59 +0200 Subject: [PATCH 21/24] style: remove commented code --- deepaas/tests/test_cmd.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/deepaas/tests/test_cmd.py b/deepaas/tests/test_cmd.py index b09758bc..10029284 100644 --- a/deepaas/tests/test_cmd.py +++ b/deepaas/tests/test_cmd.py @@ -21,7 +21,6 @@ import mock -# from deepaas.cmd import cli # W-I-P from deepaas.cmd import execute from deepaas.cmd import run from deepaas.tests import base @@ -65,16 +64,6 @@ def test_run_custom_ip_port(self, m_get_app, m_run_app, m_handle_signals): m_handle_signals.assert_called_once() -# W-I-P -# class TestCLI(base.TestCase): -# @mock.patch("deepaas.cmd.cli.main") -# def test_cli_get_metadata(self, m_cli_get_meta): -# self.flags(methods="get_metadata") -# with mock.patch.object(sys, "argv", ["deepaas-cli"]): -# cli.main() -# m_cli_get_meta.assert_called_once() - - class TestExecute(base.TestCase): @mock.patch("deepaas.cmd.execute.prediction") def test_execute_data(self, m_out_pred): From 6d870d9670ccc5e9802d29c4289c06ccc2c5fc09 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:53:36 +0200 Subject: [PATCH 22/24] fix: remove unused parameter in function --- deepaas/cmd/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index 3c83fa01..df2af134 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -168,11 +168,11 @@ def _get_file_args(fields_in): # Function to get a model object -def _get_model_name(model_name=None): - """Function to get model_obj from the name of the model. +def _get_model_name(): + """Return et mode name and model fron configured model. + In case of error, prints the list of available models - :param model_name: name of the model - :return: model object + :return: mode name, model object """ model_name = CONF.model_name models = loading.get_available_models("v2") From 32a3f52934ba083f9beafe80f82fb057035206f5 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:54:31 +0200 Subject: [PATCH 23/24] fix: merge ifs statements --- deepaas/cmd/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deepaas/cmd/cli.py b/deepaas/cmd/cli.py index df2af134..a41fc351 100644 --- a/deepaas/cmd/cli.py +++ b/deepaas/cmd/cli.py @@ -160,9 +160,8 @@ def _get_file_args(fields_in): """ file_fields = [] for k, v in fields_in.items(): - if type(v) is fields.Field: - if v.metadata.get("type", "") == "file": - file_fields.append(k) + if (type(v) is fields.Field) and (v.metadata.get("type", "") == "file"): + file_fields.append(k) return file_fields From e8ed37c60f5e842198c5c5601e0ed0b9481e2277 Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Tue, 27 Aug 2024 10:56:54 +0200 Subject: [PATCH 24/24] tests: move to AI4OS Hub CI Images --- .sqa/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sqa/docker-compose.yml b/.sqa/docker-compose.yml index a70c0b9f..bc2abb4f 100644 --- a/.sqa/docker-compose.yml +++ b/.sqa/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.6" services: DEEPaaS-testing: - image: "indigodatacloud/ci-images:python3.11" + image: "ai4oshub/ci-images:python3.12" hostname: "deepaas-testing" volumes: - type: bind