From ccc4c163d692818532bed1c209d3f2b7e5cae4ea Mon Sep 17 00:00:00 2001 From: Olof Harrysson Date: Tue, 11 Aug 2020 20:12:09 +0200 Subject: [PATCH] Adds support pre/post parsing of input args --- anyfig/anyfig_setup.py | 9 ++-- examples/online_demo.ipynb | 87 ++++++++++++++++++++++++++++++++--- hacking/cli.py | 42 +++++++++++------ notes.txt | 26 +++++------ website/docs/advanced.mdx | 8 ++-- website/docs/introduction.mdx | 13 ++++++ website/docusaurus.config.js | 2 +- 7 files changed, 147 insertions(+), 40 deletions(-) diff --git a/anyfig/anyfig_setup.py b/anyfig/anyfig_setup.py index 73068da..a1b68cf 100644 --- a/anyfig/anyfig_setup.py +++ b/anyfig/anyfig_setup.py @@ -13,12 +13,15 @@ def init_config(default_config, cli_args=None): assert default_config is not None + err_msg = ( "Expected 'default_config' to be a class definition but most likely got an object. E.g. expected ConfigClass but " "got ConfigClass() with parentheses.") assert type(default_config) == type(type), err_msg + err_msg = f"Expected 'default_config' to be an anyfig config class, was {default_config}" assert figutils.is_config_class(default_config), err_msg + if cli_args is not None: err_msg = f"Expected 'cli_args' to be a dict like object, was {type(cli_args)}" assert isinstance(cli_args, Mapping), err_msg @@ -56,14 +59,14 @@ def init_config(default_config, cli_args=None): return config -def parse_cli_args(): - ''' Parses input arguments ''' +def parse_cli_args(raw_args=None): + ''' Parses command line input arguments. If raw_args is None, sys.argv is parsed ''' class NullIO(StringIO): def write(self, txt): pass sys.stdout = NullIO() - args = fire.Fire(lambda **kwargs: kwargs) + args = fire.Fire(lambda **kwargs: kwargs, command=raw_args) sys.stdout = sys.__stdout__ return args diff --git a/examples/online_demo.ipynb b/examples/online_demo.ipynb index 6626ab2..1a8fdae 100644 --- a/examples/online_demo.ipynb +++ b/examples/online_demo.ipynb @@ -1,20 +1,95 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Anyfig Online Demo\n", + "\n", + "This notebook will allow you to play around with Anyfig directly in your browser, no installation required!\n", + "\n", + "For more information about Anyfig, check out the website [https://anyfig.now.sh/](https://anyfig.now.sh/)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Defining Config Classes" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import anyfig\n", + "from pathlib import Path\n", + "import time\n", + "anyfig.unregister_config_classes() # Needed for notebooks\n", + "\n", + "@anyfig.config_class # Registers the class with anyfig\n", + "class MyConfig():\n", + " def __init__(self):\n", + " # Config-parameters goes as attributes\n", + " self.experiment_note = 'Changed stuff'\n", + " self.save_directory = Path('output')\n", + " self.start_time = time.time()\n", + " \n", + " self.inner_config = InnerConfig()\n", + " \n", + "@anyfig.config_class \n", + "class InnerConfig():\n", + " def __init__(self):\n", + " self.inception = \"Yo Dawg\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize the Config\n", + "Anyfig supports command line input but it doesn't work in notebooks. Luckily a dict can be used to simulate the behaviour." + ] + }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Testing binder\n" + "MyConfig:\n", + " experiment_note (str): Changed stuff\n", + " save_directory (PosixPath): output\n", + " start_time (float): 1597155436.506847\n", + " inner_config (InnerConfig): \n", + " inception (str): Yo Dawg\n", + "MyConfig:\n", + " experiment_note (str): new note\n", + " save_directory (PosixPath): string-becomes-path\n", + " start_time (float): 1597155436.507649\n", + " inner_config (InnerConfig): \n", + " inception (str): nested-input-support\n" ] } ], "source": [ - "print(\"Testing binder\")" + "cli_args = {} # No input parameters\n", + "config = anyfig.init_config(default_config=MyConfig, cli_args=cli_args)\n", + "print(config)\n", + "\n", + "# Input parameters of string type\n", + "cli_args = {\n", + " 'experiment_note': 'new note',\n", + " 'save_directory': 'string-becomes-path',\n", + " 'inner_config.inception': 'nested-input-support'\n", + "}\n", + "config = anyfig.init_config(default_config=MyConfig, cli_args=cli_args)\n", + "print(config)" ] }, { @@ -27,9 +102,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "anyfig", "language": "python", - "name": "python3" + "name": "anyfig" }, "language_info": { "codemirror_mode": { @@ -41,7 +116,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.8.3" } }, "nbformat": 4, diff --git a/hacking/cli.py b/hacking/cli.py index dbcc3dd..07bd56b 100644 --- a/hacking/cli.py +++ b/hacking/cli.py @@ -1,29 +1,43 @@ import anyfig from pathlib import Path -import jsonpickle +import time +import argparse -@anyfig.config_class -class MainConfig(): +@anyfig.config_class # Registers the class with anyfig +class MyConfig(): def __init__(self): - # YOOYOYOYOO - self.yo = 0 - # asdasd - self.yo = 123 + # Config-parameters goes as attributes + self.experiment_note = 'Changed stuff' + self.save_directory = Path('output') + self.start_time = time.time() - self.shiiiet = [1, 2, 3] + # self.inner_config = InnerConfig() @anyfig.config_class class InnerConfig(): def __init__(self): - self.save_directory = 12 + self.inner_text = "Yo Dawg" + +# parser = argparse.ArgumentParser() -def main(): - config = anyfig.init_config(default_config=MainConfig) - print(config) +# parser.add_argument("--start_time", +# type=int, +# help="display a square of a given number") +# dict_args = vars(parser.parse_args()) +# print('known', dict_args) +# print('unknown', unknown) +# args = dict(args) +# print(args) +import sys +dict_args = sys.argv[1:] +print(dict_args) +dict_args = anyfig.parse_cli_args(dict_args) +print(dict_args) +dict_args.pop('start_time', None) -if __name__ == '__main__': - main() \ No newline at end of file +config = anyfig.init_config(default_config=MyConfig, cli_args=dict_args) +print(config) diff --git a/notes.txt b/notes.txt index 427d89a..345c7a2 100644 --- a/notes.txt +++ b/notes.txt @@ -1,13 +1,7 @@ -Save and load -Save as pickle and .txt (source and pretty()). - Support json - Have it work out of the box for configs that are pure json-types. Offer ability to extend. -Possibly to hash config class to make sure that other config is exact the same? Loading for example - Versioning support - I'd like to create a way to keep track of / load old configs. Something like a versioning system. Perhaps save this information alongside the config, or create a config-component that has the versioning elements and can be added into a mainconfig. Could add git information via gitpython (or show users how to do it). I like a built in versioning config-class that can save version number, creation time, git commit, maybe the entrypoint script filename -Citation for research - anyfig Implement accessing config values with square brackets/get(name, default_value)? __get_item__ @@ -16,22 +10,23 @@ name wrangling with double underscore? __build_target Post init to help constructions objects from input arguments? To put e.g. objects of lists into the config they can't take input arguments. -Notebook online demo -- Colab? Some other free service without login? - Can I use the "with" statement to maybe say with Anyfig.OnlyOverwrite(self) or something to make sure that people don't add new attributes in configs instead of overwritting existing attributes? Or something like with Anyfig.JsonCompatible() that checks that all attributes are json-compatible / have saving+loading functions? +Or marks a config as allowed/dissallowed from cli +Or used for the helpstring? Some kind of help string about the program that can be sent to init_config or place in a config? for --help output A way to control which arguments that can be overwritten by cli/visible by the --help. I don't see a good way to do this that doesn't require lots of maintainence for the user. Better to do an opt-in system than an opt-out, so whitelist the ones you want exposed. Should also be able to say if the flag should be ignored by Anyfig or raise an error. -input arguments for functions. Could maybe check if old value is function, if so, take the function with the input argument name from the old argument module's namespace - https://github.com/OlofHarrysson/anyfig [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/OlofHarrysson/anyfig/master) +https://mybinder.org/v2/gh/OlofHarrysson/anyfig/master?filepath=examples/online_demo.ipynb +TODO: Test anyfig in a notebook. Maybe have to change the way classes are registered. How to handle input parameters in notebook? + + ~~~~~~ WEBSITE ~~~~~~ Style external links @@ -39,5 +34,10 @@ Some sort of info on the main page. - Add some gifs of code being written? -~~~~~~ Release 0.4 ~~~~~~ -dict cli-arguments readme \ No newline at end of file +~~~~~~ Release 0.2.0 ~~~~~~ +dict cli-arguments readme +readme save/load. Also write why its difficult + + +Off topic: +AI to read lips. Can train with videos that has subtitles. Pretrain to classify the words within a sentence and then let it predict the word from scratch. diff --git a/website/docs/advanced.mdx b/website/docs/advanced.mdx index 167ca9f..69225d0 100644 --- a/website/docs/advanced.mdx +++ b/website/docs/advanced.mdx @@ -312,15 +312,17 @@ class DataProcesser: self.algorithm = algorithm self.data = data - # ... Algorithm methods + def solve(self): + # Use algorithm & data + pass @anyfig.config_class(target=DataProcesser) class MyConfig(): def __init__(self): - self.algorithm = lambda x: x+1 + self.algorithm = lambda x: x + 1 config = anyfig.init_config(default_config=MyConfig) -data = ... # Some complex data that can't be put into the config +data = [1, 2, 3] # Some complex data that can't be put into the config build_args = dict(data=data) data_processor = config.build(build_args) ``` \ No newline at end of file diff --git a/website/docs/introduction.mdx b/website/docs/introduction.mdx index d59c34a..91e24ef 100644 --- a/website/docs/introduction.mdx +++ b/website/docs/introduction.mdx @@ -43,3 +43,16 @@ print(config.start_time) ``` Feel free to play with the [Online Demo](https://pyfiddle.io/fiddle/4de2f70f-e421-4326-bbb8-b06d5efa547d/?i=true) hosted by Pyfiddle or start learning about Anyfig @ [Fundamentals](fundamentals.mdx). + +# Citing Anyfig +Feel free to cite Anyfig in your research: + +``` +@Misc{Anyfig, + author = {Olof Harrysson}, + title = {Anyfig - Configuring complex Python applications}, + howpublished = {Github}, + year = {2020}, + url = {https://github.com/OlofHarrysson/anyfig} +} +``` diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 2f322e8..0efbb2e 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -67,7 +67,7 @@ module.exports = { }, { - to: 'https://pyfiddle.io/fiddle/4de2f70f-e421-4326-bbb8-b06d5efa547d/?i=true', + to: 'https://mybinder.org/v2/gh/OlofHarrysson/anyfig/master?filepath=examples/online_demo.ipynb', label: 'Online Demo', position: 'left', },