From 3cd1d2577cbfdb3b9225f6188636f517059c31fd Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 22 Jan 2024 22:31:36 -0700 Subject: [PATCH] [generate_module] First stabe at config module --- misc/generate_module | 238 ++++++++++++++++++++------------------ misc/pfsense_module.py.j2 | 28 ++++- 2 files changed, 153 insertions(+), 113 deletions(-) diff --git a/misc/generate_module b/misc/generate_module index 0de8d042..4a0e56eb 100755 --- a/misc/generate_module +++ b/misc/generate_module @@ -31,6 +31,11 @@ gitconfig = git.GitConfigParser() author_name = gitconfig.get_value('user', 'name') author_email = gitconfig.get_value('user', 'email') package = 'core' +module_base = 'PFSenseModuleBase' +module_key = None +module_node = None +name_param = None +params_xml_only = [] is_package = False args_imports = [] @@ -43,6 +48,7 @@ parser.add_argument('--author-name', default=author_name, help='The full name of parser.add_argument('--author-email', default=author_email, help='The email address of the module author') parser.add_argument('--author-handle', default='', help='The github handle of the module author') parser.add_argument('--module-name', help='The name of the module to generate - defaults to being based on the url') +parser.add_argument('--is-config', action='store_true', help='This is a configuration module', ) parser.add_argument('--name-param', help='The name of the primary module parameter - defaults to the key, but often "name" is used instead of "descr"') parser.add_argument('--type-param', default='type', help='The name of the parameter for selecting different types of elements', ) parser.add_argument('--type-suffix', const=True, default=False, nargs='?', help='Suffix the module name with the item type', ) @@ -55,6 +61,10 @@ parser.add_argument('--keep-params', action=argparse.BooleanOptionalAction, help args = parser.parse_args() +# TODO - require a --module-root arg or search for it +if args.is_config: + module_root = 'system' + # Temporary directory for files tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-') @@ -131,67 +141,68 @@ scp.close() # Parse the config.xml file root = ET.parse(f'{tmpdir.name}/config.xml').getroot() -# Search for any element with our target text, make sure we found only one -xpath = f'.//*[.="{args.item_min}"]' -key_elts = root.findall(xpath) -if len(key_elts) > 1: - sys.exit(f'Found {len(key_elts)} items with path "{xpath}"') -elif len(key_elts) == 0: - sys.exit(f'Cannot find minimally configured item with path "{xpath}"') -else: - key_elt = key_elts[0] +params_full = dict() +if not args.is_config: + # Search for any element with our target text, make sure we found only one + xpath = f'.//*[.="{args.item_min}"]' + key_elts = root.findall(xpath) + if len(key_elts) > 1: + sys.exit(f'Found {len(key_elts)} items with path "{xpath}"') + elif len(key_elts) == 0: + sys.exit(f'Cannot find minimally configured item with path "{xpath}"') + else: + key_elt = key_elts[0] -# This element should be the key for the items -module_key = key_elt.tag -if args.name_param: - name_param = args.name_param -else: - name_param = module_key + # This element should be the key for the items + module_key = key_elt.tag + if args.name_param: + name_param = args.name_param + else: + name_param = module_key -# The full node configuration element will be the parent -node_elt = key_elt.find('..') -module_node = node_elt.tag + # The full node configuration element will be the parent + node_elt = key_elt.find('..') + module_node = node_elt.tag -# The "root" for this type of element is above that -root_elt = node_elt.find('..') -module_root = root_elt.tag + # The "root" for this type of element is above that + root_elt = node_elt.find('..') + module_root = root_elt.tag -# Debug -print('item_min:\t' + ET.tostring(node_elt).decode()) + # Debug + print('item_min:\t' + ET.tostring(node_elt).decode()) -# Let's use our node and key as a check -full_elt = root.find(f'.//{module_node}[{module_key}="{args.item_full}"]') -if full_elt is None: - sys.exit(f'Cannot find fully configured item with path ".//{module_node}[{module_key}="{args.item_full}"]"') + # Let's use our node and key as a check + full_elt = root.find(f'.//{module_node}[{module_key}="{args.item_full}"]') + if full_elt is None: + sys.exit(f'Cannot find fully configured item with path ".//{module_node}[{module_key}="{args.item_full}"]"') -# Debug -print('item_full:\t' + ET.tostring(full_elt).decode()) + # Debug + print('item_full:\t' + ET.tostring(full_elt).decode()) -# Collect the items for comparison with web elements and example values -params_full = dict() -for elt in full_elt: - if elt.tag == '': - continue - param = dict() - addr_elt = elt.find('address') - if addr_elt is not None: - param['example'] = addr_elt.text - param['address'] = True - elif elt.text is not None: - if elt.tag in params_full: - # Copy example and possibly other values from previous copy - param = params_full[elt.tag] - # If we have already need one of these, then it is a list - param['type'] = 'list' - # TODO - can we determine the type? - param['elements'] = 'str' - param['example2'] = elt.text.strip() - else: - param['type'] = 'str' - param['example'] = elt.text.strip() - # else: - # Likely a bool? - params_full[elt.tag] = param + # Collect the items for comparison with web elements and example values + for elt in full_elt: + if elt.tag == '': + continue + param = dict() + addr_elt = elt.find('address') + if addr_elt is not None: + param['example'] = addr_elt.text + param['address'] = True + elif elt.text is not None: + if elt.tag in params_full: + # Copy example and possibly other values from previous copy + param = params_full[elt.tag] + # If we have already need one of these, then it is a list + param['type'] = 'list' + # TODO - can we determine the type? + param['elements'] = 'str' + param['example2'] = elt.text.strip() + else: + param['type'] = 'str' + param['example'] = elt.text.strip() + # else: + # Likely a bool? + params_full[elt.tag] = param print('') @@ -318,9 +329,10 @@ for input in html.forms[0].inputs: params[input.name] = param -# Key is handled separately from other parameters so remove it -# TODO - keep the description, etc? -params.pop(module_key, None) +if not args.is_config: + # Key is handled separately from other parameters so remove it + # TODO - keep the description, etc? + params.pop(module_key, None) # Debug print(f'Web paramters: {params.keys()}') @@ -345,62 +357,67 @@ elif args.type_param in params: else: module_type = None -# Consistency -params_web_only = list(set(params.keys()) - set(params_full.keys())) -print('Web parameters not in xml: ' + str(params_web_only)) - -# Cleanup extra web parameters -for param in params_web_only: - # See if the items are numbered, likely maps to an unnumbered XML tag - newp = re.sub(r'0$', '', param) - if newp != param: - if newp in params_full: - print(f'Renaming {param} to {newp}') - params[newp] = params.pop(param) - continue - - # See if the items are prefixed by a type, likely maps to un-prefixed XML tag - newp = re.sub(f'^{module_type}_', '', param) - if newp != param: - if newp in params_full and newp not in params: - print(f'Renaming {param} to {newp}') - params[newp] = params.pop(param) - continue - - # Common renamings - for f, t in [('dst', 'destination'), ('src', 'source')]: - if param == f and t in params_full: - print(f'Renaming {f} to {t}') - params[t] = params.pop(f) - break - else: - # Otherwise, drop - probably just used to construct the final elements - if param in params and not args.keep_params: - print(f'Removing {param}') - del params[param] - -print('') -params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'}) -print(f'XML parameters not in web: {params_xml_only}\n') -if len(params_xml_only) > 0: - print(f'You may need to use {module_node.upper()}_MAP_PARAMS') - -for param in params_xml_only: - params[param] = params_full[param] - -# Create some sample descriptions -for name, param in params.items(): - # TODO - wrap long descriptions - if 'description' not in param or param['description'] == '': - param['description'] = f'The {name} of the {module_node}.' - if 'example' not in param or param['example'] == '': - if name in params_full and 'example' in params_full[name]: - param['example'] = params_full[name]['example'] - if 'default' in param: - param['description'] += f' Defaults to {param["default"]}.' +if not args.is_config: + # Consistency + params_web_only = list(set(params.keys()) - set(params_full.keys())) + print('Web parameters not in xml: ' + str(params_web_only)) + + # Cleanup extra web parameters + for param in params_web_only: + # See if the items are numbered, likely maps to an unnumbered XML tag + newp = re.sub(r'0$', '', param) + if newp != param: + if newp in params_full: + print(f'Renaming {param} to {newp}') + params[newp] = params.pop(param) + continue + + # See if the items are prefixed by a type, likely maps to un-prefixed XML tag + newp = re.sub(f'^{module_type}_', '', param) + if newp != param: + if newp in params_full and newp not in params: + print(f'Renaming {param} to {newp}') + params[newp] = params.pop(param) + continue + + # Common renamings + for f, t in [('dst', 'destination'), ('src', 'source')]: + if param == f and t in params_full: + print(f'Renaming {f} to {t}') + params[t] = params.pop(f) + break + else: + # Otherwise, drop - probably just used to construct the final elements + if param in params and not args.keep_params: + print(f'Removing {param}') + del params[param] + + print('') + params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'}) + print(f'XML parameters not in web: {params_xml_only}\n') + if len(params_xml_only) > 0: + print(f'You may need to use {module_node.upper()}_MAP_PARAMS') + + for param in params_xml_only: + params[param] = params_full[param] + + # Create some sample descriptions + for name, param in params.items(): + # TODO - wrap long descriptions + if 'description' not in param or param['description'] == '': + param['description'] = f'The {name} of the {module_node}.' + if 'example' not in param or param['example'] == '': + if name in params_full and 'example' in params_full[name]: + param['example'] = params_full[name]['example'] + if 'default' in param: + param['description'] += f' Defaults to {param["default"]}.' + +if args.is_config: + module_base = 'PFSenseModuleConfigBase' # Template variables context = dict( + module_base=module_base, module_name=module_name, module_root=module_root, module_node=module_node, @@ -409,6 +426,7 @@ context = dict( params_xml_only=params_xml_only, name_param=name_param, args_imports=args_imports, + is_config=args.is_config, is_package=is_package, package=package, author_name=args.author_name, diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2 index 88a07d68..e8d3028c 100644 --- a/misc/pfsense_module.py.j2 +++ b/misc/pfsense_module.py.j2 @@ -19,6 +19,7 @@ description: - Manage pfSense {{ module_name }}s.{{ ' This requires the pfSense ' ~ package ~ ' package to be installed.' if is_package else '' }} options: +{% if not is_config %} {{ name_param }}: description: The {{ name_param }} of the {{ module_name }}. required: true @@ -28,6 +29,7 @@ options: default: present choices: ['present', 'absent'] type: str +{% endif %} {% for name, param in params.items() %} {{ name }}: description: {{ "'" if ':' in param['description'] else '' }}{{ param['description'] | default('') }}{{ "'" if ':' in param['description'] else '' }} @@ -44,9 +46,11 @@ author: {{ author_name }} (@{{ author_handle }}) ''' EXAMPLES = r''' -- name: Add myitem {{ module_name }} +- name: {{ 'Configure' if is_config else 'Add myitem' }} {{ module_name }} pfsensible.{{ package }}.pfsense_{{ module_name }}: +{% if not is_config %} {{ name_param }}: myitem +{% endif %} {% for name, param in params.items() %} {% if param['example'] is defined %} {% if param['type'] == 'list' %} @@ -61,23 +65,33 @@ EXAMPLES = r''' {% endif %} {% endfor %} +{% if not is_config %} state: present - name: Remove myitem {{ module_name }} pfsensible.{{ package }}.pfsense_{{ module_name }}: {{ name_param }}: myitem state: absent +{% endif %} ''' RETURN = r''' commands: description: the set of commands that would be pushed to the remote device (if pfSense had a CLI). returned: always type: list +{% if is_config %} + sample: ["update {{ module_name }} set ..."] +{% else %} sample: ["create {{ module_name }} 'myitem'", "update {{ module_name }} 'myitem' set ...", "delete {{ module_name }} 'myitem'"] +{% endif %} ''' from ansible.module_utils.basic import AnsibleModule +{% if is_config %} +from ansible_collections.pfsensible.core.plugins.module_utils.module_config_base import PFSenseModuleConfigBase +{% else %} from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase +{% endif %} {% if args_imports %} from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {{ args_imports | sort | join(', ') }} {% endif %} @@ -85,6 +99,7 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import { # TODO -Change to name of module, extend for needed parameters # TODO -Keep either this or the next compact version of {{ module_name | upper() }}_ARGUMENT_SPEC {{ module_name | upper() }}_ARGUMENT_SPEC = { +{% if not is_config %} # Only {{ name_param }} should be required here - othewise you cannot remove an item with just '{{ name_param }}' # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below '{{ name_param }}': {'required': True, 'type': 'str'}, @@ -93,6 +108,7 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import { 'default': 'present', 'choices': ['present', 'absent'] }, +{% endif %} {% for param in params %} '{{ param }}': { {% if 'choices' in params[param] %} @@ -105,10 +121,12 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import { # Compact style {{ module_name | upper() }}_ARGUMENT_SPEC = dict( +{% if not is_config %} # Only {{ name_param }} should be required here - othewise you cannot remove an item with just '{{ name_param }}' # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below {{ name_param }}=dict(required=True, type='str'), state=dict(type='str', default='present', choices=['present', 'absent']), +{% endif %} {% for param in params %} {{ param }}=dict(type='{{ params[param]['type'] | default('') }}'{% if 'choices' in params[param] %}, choices={{ params[param]['choices'] }},{% endif %}), {% endfor %} @@ -116,12 +134,14 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import { # TODO - check for validity - what parameters are actually required when creating a new {{ module_name }}? {{ module_name | upper() }}_REQUIRED_IF = [ +{% if not is_config %} {% if module_type %} ['state', 'present', ['type']], ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | rejectattr('value.default', 'defined') | map(attribute='key') | join("', '") }}']], {% else %} ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | rejectattr('value.default', 'defined') | map(attribute='key') | join("', '") }}']], {% endif %} +{% endif %} ] {% if params_xml_only %} @@ -154,6 +174,7 @@ PHP_VALIDATION = r''' {% endif %} ) +{% if not is_config %} # TODO - check for validity - what are default values when creating a new {{ module_name }} {{ module_name | upper() }}_CREATE_DEFAULT = dict( {% for item in params | dict2items | selectattr('value.default', 'defined') %} @@ -164,6 +185,7 @@ PHP_VALIDATION = r''' {% endfor %} ) +{% endif %} {% if is_package %} {{ module_name | upper() }}_PHP_COMMAND_SET = r''' require_once("{{ package }}.inc"); @@ -178,7 +200,7 @@ if (filter_configure() == 0) { clear_subsystem_dirty('{{ php_subsystem }}'); } {% endif %} -class PFSense{{ module_name | capitalize() }}Module(PFSenseModuleBase): +class PFSense{{ module_name | capitalize() }}Module({{ module_base }}): """ module managing pfsense {{ module_name }}s """ ############################## @@ -192,7 +214,7 @@ class PFSense{{ module_name | capitalize() }}Module(PFSenseModuleBase): def __init__(self, module, pfsense=None): super(PFSense{{ module_name | capitalize() }}Module, self).__init__(module, pfsense, {{ 'package=\'' ~ package ~ '\', ' if is_package else ''}}root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_name | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }}, - arg_route={{ module_name | upper() }}_ARG_ROUTE{% if params_xml_only %}, map_param={{ module_name | upper() }}_MAP_PARAM{% endif %}, create_default={{ module_name | upper() }}_CREATE_DEFAULT) + arg_route={{ module_name | upper() }}_ARG_ROUTE{% if params_xml_only %}, map_param={{ module_name | upper() }}_MAP_PARAM{% endif %}{% if not is_config %}, create_default={{ module_name | upper() }}_CREATE_DEFAULT{% endif %}) def main():