From 0735b7fcd9940f036ce3072ce67676884fcc7848 Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Mon, 25 Dec 2023 00:13:17 +0100 Subject: [PATCH 1/7] add helper function to configure api information --- aiflows/utils/general_helpers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/aiflows/utils/general_helpers.py b/aiflows/utils/general_helpers.py index 80a75a3..3c9bbc7 100644 --- a/aiflows/utils/general_helpers.py +++ b/aiflows/utils/general_helpers.py @@ -524,3 +524,19 @@ def find_replace_in_dict(cfg, key_to_find, new_value,current_path=""): elif isinstance(item, list): cfg[key] = [find_replace_in_dict(x, key_to_find, new_value, new_path) for x in item] return cfg + +def update_api_infos(cfg, api_information): + """Recursively updates the api_infos in a dictionary with the api_information. + :param cfg: The dictionary to update + :type cfg: Dict[str, Any] + :param api_information: The api_information to use + :type api_information: Dict[str, Any] + """ + if isinstance(cfg, dict): + if "api_infos" in cfg and cfg["api_infos"] == "???": + cfg["api_infos"] = api_information + for key, value in cfg.items(): + update_api_infos(value, api_information) + elif isinstance(cfg, list): + for item in cfg: + update_api_infos(item, api_information) \ No newline at end of file From 4b7d237b17de3cb0b75ff43eee396b0d6bcea874 Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Mon, 25 Dec 2023 00:14:11 +0100 Subject: [PATCH 2/7] remove unused helper function to read py library function info --- aiflows/utils/general_helpers.py | 61 -------------------------------- 1 file changed, 61 deletions(-) diff --git a/aiflows/utils/general_helpers.py b/aiflows/utils/general_helpers.py index 3c9bbc7..c0cf99e 100644 --- a/aiflows/utils/general_helpers.py +++ b/aiflows/utils/general_helpers.py @@ -425,67 +425,6 @@ def read_yaml_file(path_to_file, resolve=True): cfg = OmegaConf.to_container(cfg, resolve=resolve) return cfg - -def python_file_path_to_module_path(file_path): - """Converts a python file path to a python module path - - :param file_path: The python file path - :type file_path: str - :return: The python module path - :rtype: str - """ - return file_path.replace("/", ".").replace(".py", "") - - -def python_module_path_to_file_path(module_path): - """Converts a python module path to a python file path - - :param module_path: The python module path - :type module_path: str - :return: The python file path - :rtype: str - """ - return module_path.replace(".", "/") + ".py" - - -def extract_top_level_function_names(python_file_path): - """Extracts the top level function names from a python file (ignores nested) - - :param python_file_path: The path to the python file - :type python_file_path: str - :return: A list of function names - :rtype: List[str] - """ - function_names = [] - with open(python_file_path, "r") as file: - file_content = file.read() - tree = ast.parse(file_content) - - functions = filter(lambda node: isinstance(node, ast.FunctionDef), ast.iter_child_nodes(tree)) - function_names = list(map(lambda node: node.name, functions)) - - return function_names - - -# def get_function_meta_data(function): -# """ Returns the meta data of a function. (docstring)""" -# return function_to_dict(function) - - -def get_function_from_name(function_name, module): - """Returns a function from a module given its name.""" - return getattr(module, function_name) - - -# def get_pyfile_functions_metadata_from_file(python_file_path): -# """ Returns the meta data of all the functions in a python file (docstring)""" -# function_names = extract_top_level_function_names(python_file_path) -# module_path = python_file_path_to_module_path(python_file_path) -# module = importlib.import_module(module_path) -# functions = [get_function_from_name(function_name, module) for function_name in function_names] -# return [get_function_meta_data(function) for function in functions] - - def encode_image(image_path): """Encodes an image to base64.""" with open(image_path, "rb") as image_file: From 018f04a6028b6aa3f745001bfd0ab48271512f3e Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Wed, 27 Dec 2023 15:18:08 +0100 Subject: [PATCH 3/7] add general quick_load method to configure flow configs --- aiflows/utils/general_helpers.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/aiflows/utils/general_helpers.py b/aiflows/utils/general_helpers.py index c0cf99e..cf6e6a5 100644 --- a/aiflows/utils/general_helpers.py +++ b/aiflows/utils/general_helpers.py @@ -478,4 +478,22 @@ def update_api_infos(cfg, api_information): update_api_infos(value, api_information) elif isinstance(cfg, list): for item in cfg: - update_api_infos(item, api_information) \ No newline at end of file + update_api_infos(item, api_information) + +def quick_load(cfg, item, key = "api_infos"): + """Recursively loads the item in a dictionary with key. + :param cfg: The dictionary to update + :type cfg: Dict[str, Any] + :param item: The item to set + :type item: Dict[str, Any] + :param key: The key to use, defaults to "api_infos" + :type key: str, optional + """ + if isinstance(cfg, dict): + if key in cfg and cfg[key] == "???": + cfg[key] = item + for key, value in cfg.items(): + quick_load(value, item, key) + elif isinstance(cfg, list): + for item in cfg: + quick_load(item, item, key) \ No newline at end of file From fbefb7e39b169a637c622c9665e2d52a4c1444bb Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Wed, 27 Dec 2023 15:39:42 +0100 Subject: [PATCH 4/7] type fix in quick_load --- aiflows/utils/general_helpers.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/aiflows/utils/general_helpers.py b/aiflows/utils/general_helpers.py index cf6e6a5..da13366 100644 --- a/aiflows/utils/general_helpers.py +++ b/aiflows/utils/general_helpers.py @@ -480,20 +480,21 @@ def update_api_infos(cfg, api_information): for item in cfg: update_api_infos(item, api_information) -def quick_load(cfg, item, key = "api_infos"): +def quick_load(cfg, item, key="api_infos"): """Recursively loads the item in a dictionary with key. :param cfg: The dictionary to update :type cfg: Dict[str, Any] :param item: The item to set :type item: Dict[str, Any] - :param key: The key to use, defaults to "api_infos" + :param key: The key to use, defaults to 'api_infos' :type key: str, optional """ if isinstance(cfg, dict): if key in cfg and cfg[key] == "???": cfg[key] = item - for key, value in cfg.items(): - quick_load(value, item, key) + for k, v in cfg.items(): + quick_load(v, item, key) elif isinstance(cfg, list): - for item in cfg: - quick_load(item, item, key) \ No newline at end of file + for elem in cfg: + quick_load(elem, item, key) + From 2b42de64443a3f40df59bf13141ca1a3fbd4c9b1 Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Wed, 27 Dec 2023 15:49:46 +0100 Subject: [PATCH 5/7] remove unused update_api_infos method from general helpers --- aiflows/utils/general_helpers.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/aiflows/utils/general_helpers.py b/aiflows/utils/general_helpers.py index da13366..0508442 100644 --- a/aiflows/utils/general_helpers.py +++ b/aiflows/utils/general_helpers.py @@ -464,24 +464,8 @@ def find_replace_in_dict(cfg, key_to_find, new_value,current_path=""): cfg[key] = [find_replace_in_dict(x, key_to_find, new_value, new_path) for x in item] return cfg -def update_api_infos(cfg, api_information): - """Recursively updates the api_infos in a dictionary with the api_information. - :param cfg: The dictionary to update - :type cfg: Dict[str, Any] - :param api_information: The api_information to use - :type api_information: Dict[str, Any] - """ - if isinstance(cfg, dict): - if "api_infos" in cfg and cfg["api_infos"] == "???": - cfg["api_infos"] = api_information - for key, value in cfg.items(): - update_api_infos(value, api_information) - elif isinstance(cfg, list): - for item in cfg: - update_api_infos(item, api_information) - def quick_load(cfg, item, key="api_infos"): - """Recursively loads the item in a dictionary with key. + """Recursively loads the config item in a dictionary with key. :param cfg: The dictionary to update :type cfg: Dict[str, Any] :param item: The item to set From c5439e36e181e207fa57ed2d8699da7e07256d97 Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Wed, 27 Dec 2023 16:36:34 +0100 Subject: [PATCH 6/7] add example for quick_load helper --- aiflows/utils/general_helpers.py | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/aiflows/utils/general_helpers.py b/aiflows/utils/general_helpers.py index 0508442..70dd745 100644 --- a/aiflows/utils/general_helpers.py +++ b/aiflows/utils/general_helpers.py @@ -472,6 +472,50 @@ def quick_load(cfg, item, key="api_infos"): :type item: Dict[str, Any] :param key: The key to use, defaults to 'api_infos' :type key: str, optional + + example: + cfg = { + 'backend': { + 'api_infos': '???', + 'model_name': { + 'openai': 'gpt-4', + 'azure': 'azure/gpt-4' + } + } + 'Executor' : { + 'subflows_config': { + 'backend': { + 'api_infos': '???', + 'model_name': { + 'openai': 'gpt-4', + 'azure': 'azure/gpt-4' + } + } + } + } + } + api_information = [ApiInfo(backend_used="openai", api_key=os.getenv("OPENAI_API_KEY"))] + quick_load(cfg, api_information) + returns: cfg = { + 'backend': { + 'api_infos': [ApiInfo(backend_used="openai", api_key=os.getenv("OPENAI_API_KEY"))], + 'model_name': { + 'openai': 'gpt-4', + 'azure': 'azure/gpt-4' + } + } + 'Executor' : { + 'subflows_config': { + 'backend': { + 'api_infos': [ApiInfo(backend_used="openai", api_key=os.getenv("OPENAI_API_KEY"))], + 'model_name': { + 'openai': 'gpt-4', + 'azure': 'azure/gpt-4' + } + } + } + } + } """ if isinstance(cfg, dict): if key in cfg and cfg[key] == "???": From 2bea0f7d7cd9a8e8277c7af5dd3edf4f4d0c1ccd Mon Sep 17 00:00:00 2001 From: Tachi67 Date: Wed, 27 Dec 2023 17:03:01 +0100 Subject: [PATCH 7/7] update quick start guide to use quick_load helper to configure api infos --- examples/quick_start/runChatAtomicFlow.py | 7 ++----- .../public/docs/getting_started/Quick_Start/quick_start.md | 7 ++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/quick_start/runChatAtomicFlow.py b/examples/quick_start/runChatAtomicFlow.py index 2570872..41d41ac 100644 --- a/examples/quick_start/runChatAtomicFlow.py +++ b/examples/quick_start/runChatAtomicFlow.py @@ -1,11 +1,8 @@ -from aiflows.backends.api_info import ApiInfo -from aiflows.utils.general_helpers import find_replace_in_dict from aiflows.flow_launchers import FlowLauncher from aiflows.backends.api_info import ApiInfo from aiflows import flow_verse -import os from aiflows import logging -from aiflows.utils.general_helpers import read_yaml_file +from aiflows.utils.general_helpers import read_yaml_file, quick_load dependencies = [ @@ -37,7 +34,7 @@ cfg = read_yaml_file("flow_modules/aiflows/ChatFlowModule/demo.yaml") # put the API information in the config - cfg["flow"]["backend"]["api_infos"] = api_information + quick_load(cfg, api_information, key="api_infos") # ~~~ Instantiate the Flow ~~~ flow = ChatAtomicFlow.instantiate_from_default_config(**cfg["flow"]) diff --git a/website/public/docs/getting_started/Quick_Start/quick_start.md b/website/public/docs/getting_started/Quick_Start/quick_start.md index 5bfce8d..19149eb 100644 --- a/website/public/docs/getting_started/Quick_Start/quick_start.md +++ b/website/public/docs/getting_started/Quick_Start/quick_start.md @@ -96,9 +96,14 @@ flow: # Overrides the ChatAtomicFlow config The following overwrites the field with your personal API information: ```python -# put the API information in the config +# recursively find the 'api_infos' entry and put the API information in the config +quick_load(cfg, api_information, key="api_infos") +``` +This is equivalent to the following: +```python cfg["flow"]["backend"]["api_infos"] = api_information ``` +However, with `quick_load`, we are able to quickly set all entries of api_infos, this is useful when we need to configure more than just one config entries. Instantiate your Flow: ```python