From a2c0c052325f783f0194e75cc26086aceac4a2c8 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Mon, 2 Dec 2024 15:13:21 -0800 Subject: [PATCH 1/6] Improvements to DocTAPE Added support for objects like the Aircraft and Mission hierarchies to get_all_keys, get_value, and glue_keys. Fixed a bug where setting display=False for glue_variable would display instead of nothing Added support for including parentheses in the arguments for get_variable_name --- aviary/docs/developer_guide/doctape.ipynb | 24 ------- .../developer_guide/doctape_examples.ipynb | 65 +++++++++++++++++++ aviary/utils/doctape.py | 29 +++++++-- aviary/utils/test/test_doctape.py | 12 ++-- 4 files changed, 95 insertions(+), 35 deletions(-) diff --git a/aviary/docs/developer_guide/doctape.ipynb b/aviary/docs/developer_guide/doctape.ipynb index b4db14ae8..4fc24a74c 100644 --- a/aviary/docs/developer_guide/doctape.ipynb +++ b/aviary/docs/developer_guide/doctape.ipynb @@ -82,41 +82,17 @@ " doctape.glue_variable(key, md_code=True)\n", " class_list += f'- `{key}` {val}\\n'\n", "\n", - "# testing_list = ''\n", - "# for key,val in testing_functions.items():\n", - "# testing_list += f'- `{key}` {val}\\n'\n", - "\n", "utility_list = '```{eval-rst}\\n'\n", "for key in utility_functions:\n", " doctape.glue_variable(key, md_code=True)\n", " utility_list += ' '*4+f'.. autofunction:: aviary.utils.doctape.{key}\\n{\" \"*8}:noindex:\\n\\n'\n", "utility_list += '```'\n", "\n", - "# testing_list = '```{eval-rst}\\n'\n", - "# for key in testing_functions:\n", - "# utils.glue_variable(key, md_code=True)\n", - "# testing_list += ' '*4+f'.. autofunction:: aviary.utils.doctape.{key}\\n{\" \"*8}:noindex:\\n\\n'\n", - "# testing_list += '```'\n", - "\n", - "# testing_list = '
\\n\\nFunction Docs\\n\\n'\n", "testing_list = '```{eval-rst}\\n'\n", "for key in testing_functions:\n", " doctape.glue_variable(key, md_code=True)\n", " testing_list += ' '*4+f'.. autofunction:: aviary.utils.doctape.{key}\\n{\" \"*8}:noindex:\\n\\n'\n", "testing_list += '```'\n", - "# testing_list += '\\n\\n
'\n", - "\n", - "# glue_list = ''\n", - "# for key,val in glue_functions.items():\n", - "# glue_list += f'- `{key}` {key}\\n'\n", - "\n", - "# glue_list = ''\n", - "# for key in glue_functions:\n", - "# # doc_str = inspect.getdoc(imported_functions[key])\n", - "# doc_str = imported_functions[key].__doc__.split('\\n')[1]\n", - "# # doc_str = '\\n'.join([s+' ' for s in imported_functions[key].__doc__.split('\\n')])\n", - "# print(doc_str)\n", - "# glue_list += f'- `{key}`: {doc_str}\\n'\n", "\n", "glue_list = '```{eval-rst}\\n'\n", "for key in glue_functions:\n", diff --git a/aviary/docs/developer_guide/doctape_examples.ipynb b/aviary/docs/developer_guide/doctape_examples.ipynb index 378bfe52e..24edcdfe6 100644 --- a/aviary/docs/developer_guide/doctape_examples.ipynb +++ b/aviary/docs/developer_guide/doctape_examples.ipynb @@ -482,6 +482,71 @@ "p1_alt = get_value(simplified_dict, 'phase1.altitude.val')\n", "print(p1_alt)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "remove-cell" + ] + }, + "outputs": [], + "source": [ + "# Testing Cell\n", + "from aviary.utils.doctape import glue_variable, check_args, get_all_keys, get_previous_line\n", + "from aviary.api import Aircraft, Mission\n", + "\n", + "glue_variable(Aircraft.__name__)\n", + "glue_variable(Mission.__name__)\n", + "\n", + "track_layers = 'track_layers'\n", + "check_args(get_all_keys, track_layers)\n", + "glue_variable(track_layers)\n", + "\n", + "get_all_keys(Mission, track_layers='Mission')\n", + "track_layers_with_name = get_previous_line().split(', ')[1].split(')')[0]\n", + "glue_variable('track_layers_with_Mission', track_layers_with_name, display=False)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "These can also be used to recursively get all of the attributes from a complex object, like the {glue:md}`Aircraft` or {glue:md}`Mission` hierarchies.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from aviary.utils.doctape import get_all_keys, get_value, glue_keys\n", + "from aviary.api import Mission\n", + "\n", + "k1=get_all_keys(Mission)\n", + "print(k1[:5]) # Display the first 5 keys in Mission\n", + "k2=get_all_keys(Mission, track_layers=True)\n", + "print(k2[:5]) # Display the first 5 keys in Mission\n", + "k3=get_all_keys(Mission, track_layers='Mission')\n", + "print(k3[:5]) # Display the first 5 keys in Mission\n", + "\n", + "glue_keys(Mission, False)\n", + "\n", + "print(get_value(Mission,'Constraints.GEARBOX_SHAFT_POWER_RESIDUAL'))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If {glue:md}`get_all_keys` is used on an object like {glue:md}`Mission` without specifying a value for {glue:md}`track_layers` will return all of the uniquely named attributes of the object (such as {glue:md}GEARBOX_SHAFT_POWER_RESIDUAL). Setting {glue:md}`track_layers` to `True` will get all of the attributes in dot notation, but will not include the name of the original object ({glue:md}Constraints.GEARBOX_SHAFT_POWER_RESIDUAL). If you want the full name of the attribute, including the name of the original object, you can use that name as the value of {glue:md}`track_layers` (using {glue:md}track_layers_with_Mission gives us access to {glue:md}Mission.Constraints.GEARBOX_SHAFT_POWER_RESIDUAL)\n", + "\n", + "Using {glue:md}`glue_keys` handles this for us automatically by using the `__name__` attribute of the object passed to it as the value of {glue:md}`track_layers`.\n", + "\n", + "As with the dict_of_dicts, we can recusively get the value of an attribute using the full path along with {glue:md}`get_value`." + ] } ], "metadata": { diff --git a/aviary/utils/doctape.py b/aviary/utils/doctape.py index 13f129410..3900e9632 100644 --- a/aviary/utils/doctape.py +++ b/aviary/utils/doctape.py @@ -1,7 +1,6 @@ import inspect import subprocess import tempfile -import os import numpy as np @@ -107,8 +106,8 @@ def get_variable_name(*variables) -> str: # get the line number that called this function lineno = pframe.f_lineno - first_line if first_line else pframe.f_lineno - 1 # extract the argument and remove all whitespace - arg: str = ''.join(lines[lineno].split( - 'get_variable_name(')[1].split(')')[0].split()) + pre_arg, arg = ''.join(lines[lineno].split()).split('get_variable_name(', 1) + arg = ')'.join(arg.split(')')[:-(1+pre_arg.count('('))]) if ',' in arg: return arg.split(',') else: @@ -290,6 +289,7 @@ def get_attribute_name(object: object, attribute, error_type=AttributeError) -> def get_all_keys(dict_of_dicts: dict, track_layers=False, all_keys=None) -> list: """ Recursively get all of the keys from a dict of dicts + This can also be used to recursively get all of the attributes from a complex object, like the Aircraft hierarchy Note: this will not add duplicates of keys, but will continue deeper even if a key is duplicated @@ -309,9 +309,14 @@ def get_all_keys(dict_of_dicts: dict, track_layers=False, all_keys=None) -> list all_keys : list A list of all the keys in the dict_of_dicts """ + if not isinstance(dict_of_dicts, dict): + dict_of_dicts = dict_of_dicts.__dict__ if all_keys is None: all_keys = [] + for key, val in dict_of_dicts.items(): + if key.startswith('__') and key.endswith('__'): + continue if track_layers is True: current_layer = '' elif track_layers: @@ -320,7 +325,7 @@ def get_all_keys(dict_of_dicts: dict, track_layers=False, all_keys=None) -> list key = current_layer+'.'+key if key not in all_keys: all_keys.append(key) - if isinstance(val, dict): + if isinstance(val, dict) or hasattr(val, '__dict__'): if track_layers: current_layer = key else: @@ -347,6 +352,8 @@ def get_value(dict_of_dicts: dict, comlpete_key: str): """ for key in comlpete_key.split('.'): + if not isinstance(dict_of_dicts, dict): + dict_of_dicts = dict_of_dicts.__dict__ dict_of_dicts = dict_of_dicts[key] return dict_of_dicts @@ -372,13 +379,18 @@ def glue_variable(name: str, val=None, md_code=False, display=True): # local import so myst isn't required unless glue is being used from myst_nb import glue from IPython.display import Markdown + from IPython.utils import io if val is None: val = name if md_code: val = Markdown('`'+val+'`') else: val = Markdown(val) - glue(name, val, display) + + with io.capture_output() as captured: + glue(name, val, display) + if display: + captured.show() def glue_keys(dict_of_dicts: dict, display=True) -> list: @@ -395,7 +407,12 @@ def glue_keys(dict_of_dicts: dict, display=True) -> list: all_keys : list A list of all the keys that were glued """ - all_keys = get_all_keys(dict_of_dicts) + if not isinstance(dict_of_dicts, dict): + track_layers = dict_of_dicts.__name__ + else: + track_layers = False + all_keys = get_all_keys(dict_of_dicts, track_layers) + for key in all_keys: glue_variable(key, md_code=True, display=display) return all_keys diff --git a/aviary/utils/test/test_doctape.py b/aviary/utils/test/test_doctape.py index d13b8d1c1..6e9db35c4 100644 --- a/aviary/utils/test/test_doctape.py +++ b/aviary/utils/test/test_doctape.py @@ -3,7 +3,9 @@ from openmdao.utils.assert_utils import assert_near_equal, assert_equal_numstrings, assert_equal_arrays -from aviary.utils.doctape import gramatical_list, check_value, check_contains, check_args, run_command_no_file_error, get_attribute_name, get_all_keys, get_value, get_previous_line, get_variable_name +from aviary.utils.doctape import (gramatical_list, check_value, check_contains, check_args, + run_command_no_file_error, get_attribute_name, get_all_keys, get_value, get_previous_line, + get_variable_name, glue_variable, glue_keys) class DocTAPETests(unittest.TestCase): @@ -55,12 +57,12 @@ def test_get_variable_name(self): assert_equal_numstrings(name, 'var') # requires IPython shell - # def test_glue_variable(self): - # glue_variable('plain_text') + def test_glue_variable(self): + glue_variable('plain_text', display=False) # requires IPython shell - # def test_glue_keys(self): - # glue_keys({'d1':{'d2':2}}) + def test_glue_keys(self): + glue_keys({'d1': {'d2': 2}}, display=False) if __name__ == '__main__': From ba6f558eef6752ec679d7a06196b697dd1292950 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Mon, 2 Dec 2024 17:40:45 -0800 Subject: [PATCH 2/6] fix for parentheses matching --- aviary/utils/doctape.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/aviary/utils/doctape.py b/aviary/utils/doctape.py index 3900e9632..45932b903 100644 --- a/aviary/utils/doctape.py +++ b/aviary/utils/doctape.py @@ -2,6 +2,7 @@ import subprocess import tempfile import numpy as np +import re """ @@ -88,6 +89,7 @@ def get_previous_line(n=1) -> str: def get_variable_name(*variables) -> str: """ returns the name of the variable passed to the function as a string + # NOTE: You cannot call this function multiple times on one line Parameters ---------- @@ -107,7 +109,14 @@ def get_variable_name(*variables) -> str: lineno = pframe.f_lineno - first_line if first_line else pframe.f_lineno - 1 # extract the argument and remove all whitespace pre_arg, arg = ''.join(lines[lineno].split()).split('get_variable_name(', 1) - arg = ')'.join(arg.split(')')[:-(1+pre_arg.count('('))]) + + num_paren = 1 + for ind, el in enumerate(arg.split(')')): + num_paren += el.count('(')-1 + if num_paren == 0: + break + arg = ')'.join(arg.split(')')[:ind+1]) + if ',' in arg: return arg.split(',') else: From d03c0839f62878062654a42f410c94f914b2aeed Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Mon, 2 Dec 2024 17:41:41 -0800 Subject: [PATCH 3/6] trying regex --- aviary/utils/doctape.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/aviary/utils/doctape.py b/aviary/utils/doctape.py index 45932b903..1ca223117 100644 --- a/aviary/utils/doctape.py +++ b/aviary/utils/doctape.py @@ -108,14 +108,12 @@ def get_variable_name(*variables) -> str: # get the line number that called this function lineno = pframe.f_lineno - first_line if first_line else pframe.f_lineno - 1 # extract the argument and remove all whitespace - pre_arg, arg = ''.join(lines[lineno].split()).split('get_variable_name(', 1) - - num_paren = 1 - for ind, el in enumerate(arg.split(')')): - num_paren += el.count('(')-1 - if num_paren == 0: - break - arg = ')'.join(arg.split(')')[:ind+1]) + arg = ''.join(lines[lineno].split()).split('get_variable_name(', 1)[1] + + # Use regex to match balanced parentheses + match = re.match(r'([^()]*\([^()]*\))*[^()]*', arg) + if match: + arg = match.group(0) if ',' in arg: return arg.split(',') From c4a7bc736973f8588d2edc8db16630be7cc826e3 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Tue, 3 Dec 2024 08:49:01 -0800 Subject: [PATCH 4/6] adding myst-nb to test dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 143676dd5..465292ffd 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ pkgname = "aviary" extras_require = { - "test": ["testflo", "pre-commit", "sphinx_book_theme==1.1.0"], + "test": ["testflo", "pre-commit", "sphinx_book_theme==1.1.0", "myst-nb"], "examples": ["openaerostruct", "ambiance", "itables"], } From 5c390bf6cf7adc325ca39c8d1c09842f109f4792 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Tue, 3 Dec 2024 15:09:51 -0800 Subject: [PATCH 5/6] fix for CI capturing the output of glue without running RichOutput(**kargs).display() prevents myst from finding the glued variable --- aviary/utils/doctape.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/aviary/utils/doctape.py b/aviary/utils/doctape.py index 1ca223117..a63282041 100644 --- a/aviary/utils/doctape.py +++ b/aviary/utils/doctape.py @@ -115,6 +115,13 @@ def get_variable_name(*variables) -> str: if match: arg = match.group(0) + # # Requires Python 3.11, but allows this to be called multiple times on one line + # positions = inspect.getframeinfo(pframe).positions + # calling_lines = lines[positions.lineno-1:positions.end_lineno] + # calling_lines[-1] = calling_lines[-1][:positions.end_col_offset-1] + # calling_lines[0] = calling_lines[0][positions.col_offset:].removeprefix('get_variable_name(') + # arg = ''.join([l.strip() for l in calling_lines]) + if ',' in arg: return arg.split(',') else: @@ -396,8 +403,8 @@ def glue_variable(name: str, val=None, md_code=False, display=True): with io.capture_output() as captured: glue(name, val, display) - if display: - captured.show() + # if display: + captured.show() def glue_keys(dict_of_dicts: dict, display=True) -> list: From 0b9605a890822606e829bddc316cf6889578bf0e Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Tue, 3 Dec 2024 15:48:54 -0800 Subject: [PATCH 6/6] fix for CI --- aviary/docs/developer_guide/doctape_examples.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aviary/docs/developer_guide/doctape_examples.ipynb b/aviary/docs/developer_guide/doctape_examples.ipynb index 24edcdfe6..4fb7f3006 100644 --- a/aviary/docs/developer_guide/doctape_examples.ipynb +++ b/aviary/docs/developer_guide/doctape_examples.ipynb @@ -407,12 +407,12 @@ "outputs": [], "source": [ "# Testing Cell\n", - "from aviary.api import Mission\n", + "from aviary.api import Aircraft\n", "from aviary.utils.doctape import glue_variable, get_previous_line, get_variable_name\n", "\n", - "glue_variable('value', Mission.Design.MACH, md_code=True)\n", + "glue_variable('value', Aircraft.Design.EMPTY_MASS, md_code=True)\n", "glue_variable('var_value_code', get_previous_line(), md_code=True)\n", - "glue_variable(get_variable_name(Mission.Design.MACH), md_code=True)\n", + "glue_variable(get_variable_name(Aircraft.Design.EMPTY_MASS), md_code=True)\n", "glue_variable('var_name_code', get_previous_line(), md_code=True)\n" ] }, @@ -423,7 +423,7 @@ "If you want to glue the name of a variable, instead of the value that variable holds, you can use the {glue:md}`get_variable_name` to extract it.\n", "\n", "For example:\n", - "Using {glue:md}`var_value_code` will result in {glue:md}`value`, whereas using {glue:md}`var_name_code` will result in {glue:md}`Mission.Design.MACH`\n", + "Using {glue:md}`var_value_code` will result in {glue:md}`value`, whereas using {glue:md}`var_name_code` will result in {glue:md}`Aircraft.Design.EMPTY_MASS`\n", "\n", "### {glue:md}`get_attribute_name`\n", "allows users to get the name of object attributes in order to glue them into documentation. This works well for Enums or Class Variables that have unique values."