From 9966dd667781ef2e76ece96f52c7da35cec214fe Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Fri, 27 Oct 2023 11:41:00 +0300 Subject: [PATCH 1/4] Move the resource allocation script assets under commands.generate --- deployer/{ => commands/generate}/resource_allocation/__init__.py | 0 .../generate}/resource_allocation/generate_choices.py | 0 .../generate}/resource_allocation/node-capacity-info.json | 0 .../generate}/resource_allocation/update_nodeinfo.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename deployer/{ => commands/generate}/resource_allocation/__init__.py (100%) rename deployer/{ => commands/generate}/resource_allocation/generate_choices.py (100%) rename deployer/{ => commands/generate}/resource_allocation/node-capacity-info.json (100%) rename deployer/{ => commands/generate}/resource_allocation/update_nodeinfo.py (100%) diff --git a/deployer/resource_allocation/__init__.py b/deployer/commands/generate/resource_allocation/__init__.py similarity index 100% rename from deployer/resource_allocation/__init__.py rename to deployer/commands/generate/resource_allocation/__init__.py diff --git a/deployer/resource_allocation/generate_choices.py b/deployer/commands/generate/resource_allocation/generate_choices.py similarity index 100% rename from deployer/resource_allocation/generate_choices.py rename to deployer/commands/generate/resource_allocation/generate_choices.py diff --git a/deployer/resource_allocation/node-capacity-info.json b/deployer/commands/generate/resource_allocation/node-capacity-info.json similarity index 100% rename from deployer/resource_allocation/node-capacity-info.json rename to deployer/commands/generate/resource_allocation/node-capacity-info.json diff --git a/deployer/resource_allocation/update_nodeinfo.py b/deployer/commands/generate/resource_allocation/update_nodeinfo.py similarity index 100% rename from deployer/resource_allocation/update_nodeinfo.py rename to deployer/commands/generate/resource_allocation/update_nodeinfo.py From 4c66acd1677711417d0df92f7da650b04428c045 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Fri, 27 Oct 2023 12:08:24 +0300 Subject: [PATCH 2/4] Move resource allocation under generate app and add help strings --- deployer/__main__.py | 4 ++-- .../generate/resource_allocation/generate_choices.py | 10 +++++++--- .../generate/resource_allocation/update_nodeinfo.py | 10 +++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/deployer/__main__.py b/deployer/__main__.py index 6880ee265d..2b638f7111 100644 --- a/deployer/__main__.py +++ b/deployer/__main__.py @@ -9,13 +9,13 @@ import deployer.commands.generate.dedicated_cluster.aws # noqa: F401 import deployer.commands.generate.dedicated_cluster.gcp # noqa: F401 import deployer.commands.generate.helm_upgrade.jobs # noqa: F401 +import deployer.commands.generate.resource_allocation.generate_choices # noqa: F401 +import deployer.commands.generate.resource_allocation.update_nodeinfo # noqa: F401 import deployer.commands.grafana.central_grafana # noqa: F401 import deployer.commands.grafana.deploy_dashboards # noqa: F401 import deployer.commands.grafana.tokens # noqa: F401 import deployer.commands.validate.config # noqa: F401 import deployer.keys.decrypt_age # noqa: F401 -import deployer.resource_allocation.generate_choices # noqa: F401 -import deployer.resource_allocation.update_nodeinfo # noqa: F401 from .cli_app import app diff --git a/deployer/commands/generate/resource_allocation/generate_choices.py b/deployer/commands/generate/resource_allocation/generate_choices.py index 9b5043e29e..ab70aca912 100644 --- a/deployer/commands/generate/resource_allocation/generate_choices.py +++ b/deployer/commands/generate/resource_allocation/generate_choices.py @@ -6,7 +6,7 @@ import typer from ruamel.yaml import YAML -from ..cli_app import app +from deployer.cli_app import generate_app yaml = YAML(typ="rt") @@ -99,8 +99,8 @@ def proportional_memory_strategy( return choices -@app.command() -def generate_resource_allocation_choices( +@generate_app.command() +def resource_allocation_choices( instance_type: str = typer.Argument( ..., help="Instance type to generate Resource Allocation options for" ), @@ -110,6 +110,10 @@ def generate_resource_allocation_choices( help="Strategy to use for generating resource allocation choices choices", ), ): + """ + Generate a custom number of resource allocation choices for a certain instance type, + depending on a certain chosen strategy. + """ with open(HERE / "node-capacity-info.json") as f: nodeinfo = json.load(f) diff --git a/deployer/commands/generate/resource_allocation/update_nodeinfo.py b/deployer/commands/generate/resource_allocation/update_nodeinfo.py index 33fd3e7bec..428d703277 100644 --- a/deployer/commands/generate/resource_allocation/update_nodeinfo.py +++ b/deployer/commands/generate/resource_allocation/update_nodeinfo.py @@ -8,7 +8,7 @@ from kubernetes.utils.quantity import parse_quantity from ruamel.yaml import YAML -from ..cli_app import app +from deployer.cli_app import generate_app HERE = Path(__file__).parent @@ -133,12 +133,16 @@ def get_node_capacity_info(instance_type: str): } -@app.command() -def update_node_capacity_info( +@generate_app.command() +def node_capacity_entry_in_info_json( instance_type: str = typer.Argument( ..., help="Instance type to generate Resource Allocation options for" ), ): + """ + Generates a new entry holding info about the capacity of a node of a certain instance type + or updates existing one in a json file called `node-capacity-info.json`. + """ try: with open(HERE / "node-capacity-info.json") as f: instances_info = json.load(f) From 0d67971530ff72e92f1c8c33a93f4042bc92f346 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Fri, 27 Oct 2023 12:51:38 +0300 Subject: [PATCH 3/4] Add new deployer.generate sub-command and nest everything there --- .../generate/resource_allocation/__init__.py | 0 .../resource_allocation/generate_choices.py | 6 +++--- .../resource_allocation_app.py | 15 +++++++++++++++ .../resource_allocation/update_nodeinfo.py | 6 +++--- 4 files changed, 21 insertions(+), 6 deletions(-) delete mode 100644 deployer/commands/generate/resource_allocation/__init__.py create mode 100644 deployer/commands/generate/resource_allocation/resource_allocation_app.py diff --git a/deployer/commands/generate/resource_allocation/__init__.py b/deployer/commands/generate/resource_allocation/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deployer/commands/generate/resource_allocation/generate_choices.py b/deployer/commands/generate/resource_allocation/generate_choices.py index ab70aca912..08ae9f5191 100644 --- a/deployer/commands/generate/resource_allocation/generate_choices.py +++ b/deployer/commands/generate/resource_allocation/generate_choices.py @@ -6,7 +6,7 @@ import typer from ruamel.yaml import YAML -from deployer.cli_app import generate_app +from .resource_allocation_app import resource_allocation_app yaml = YAML(typ="rt") @@ -99,8 +99,8 @@ def proportional_memory_strategy( return choices -@generate_app.command() -def resource_allocation_choices( +@resource_allocation_app.command() +def choices( instance_type: str = typer.Argument( ..., help="Instance type to generate Resource Allocation options for" ), diff --git a/deployer/commands/generate/resource_allocation/resource_allocation_app.py b/deployer/commands/generate/resource_allocation/resource_allocation_app.py new file mode 100644 index 0000000000..dd5c2555b0 --- /dev/null +++ b/deployer/commands/generate/resource_allocation/resource_allocation_app.py @@ -0,0 +1,15 @@ +""" +Creates a new typer application, which is then +nested as a sub-command named "resource-allocation" +under the `generate` sub-command of the deployer. +""" +import typer + +from deployer.cli_app import generate_app + +resource_allocation_app = typer.Typer(pretty_exceptions_show_locals=False) +generate_app.add_typer( + resource_allocation_app, + name="resource-allocation", + help="Generate the choices for a resource allocation strategy of an instance type and additional helper information", +) diff --git a/deployer/commands/generate/resource_allocation/update_nodeinfo.py b/deployer/commands/generate/resource_allocation/update_nodeinfo.py index 428d703277..c7dd0bf265 100644 --- a/deployer/commands/generate/resource_allocation/update_nodeinfo.py +++ b/deployer/commands/generate/resource_allocation/update_nodeinfo.py @@ -8,7 +8,7 @@ from kubernetes.utils.quantity import parse_quantity from ruamel.yaml import YAML -from deployer.cli_app import generate_app +from .resource_allocation_app import resource_allocation_app HERE = Path(__file__).parent @@ -133,8 +133,8 @@ def get_node_capacity_info(instance_type: str): } -@generate_app.command() -def node_capacity_entry_in_info_json( +@resource_allocation_app.command() +def node_info_update( instance_type: str = typer.Argument( ..., help="Instance type to generate Resource Allocation options for" ), From 1a3808610a153d7d484d794400884fc51babd2c5 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Fri, 27 Oct 2023 13:04:26 +0300 Subject: [PATCH 4/4] Update docs and docstrings --- deployer/README.md | 22 +++++++++++++++---- deployer/cli_app.py | 3 ++- .../resource_allocation/update_nodeinfo.py | 3 ++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/deployer/README.md b/deployer/README.md index db285d9441..c6f5135e32 100644 --- a/deployer/README.md +++ b/deployer/README.md @@ -85,8 +85,13 @@ The `deployer.py` file is the main file, that contains all of the commands regis │   │   │   ├── dedicate_cluster_app.py │   │   │   └── gcp.py │   │   └── helm_upgrade -│   │   ├── decision.py -│   │   └── jobs.py +│   │   | ├── decision.py +│   │   | └── jobs.py +| | └── resource_allocation +│   │   ├── generate_choices.py +│   │   ├── node-capacity-info.json +│   │   ├── resource_allocation_app.py +│   │   └── update_nodeinfo.py │   ├── grafana │   │   ├── central_grafana.py │   │   ├── deploy_dashboards.py @@ -139,8 +144,8 @@ This section descripts some of the subcommands the `deployer` can carry out. │ deploy-support Deploy support components to a cluster │ │ exec Execute a shell in various parts of the infra. It can be used for poking around, or │ │ debugging issues. │ -│ generate Generate various types of assets. It currently supports generating files related to billing │ -│ or new, dedicated clusters. │ +│ generate Generate various types of assets. It currently supports generating files related to │ +│ billing, new dedicated clusters, helm upgrade strategies and resource allocation. │ │ grafana Manages Grafana related workflows. │ │ run-hub-health-check Run a health check on a given hub on a given cluster. Optionally check scaling of dask │ │ workers if the hub is a daskhub. │ @@ -252,6 +257,15 @@ These defaults are described in each file template. The cluster configuration directory is generated based on the templates in: - (`config/clusters/templates/gcp`)[https://github.com/2i2c-org/infrastructure/blob/master/config/clusters/templates/gcp] +#### `generate resource-allocation` + +This sub-command can be used to generate the resource allocation choices for given instance type and a given optimization strategy. + +##### `generate resource-allocation choices` +This generates a custom number of resource allocation choices for a certain instance type, depending on a certain chosen strategy that can be used in the profile list of a hub. + +##### `generate resource-allocation node-info-update` +This updates the json file `node-capacity-info.json` with info about the capacity of a node of a certain type. This file is then used for generating the resource choices. ### The `grafana` sub-command This deployer sub-command manages all of the available functions related to Grafana. diff --git a/deployer/cli_app.py b/deployer/cli_app.py index 3aca1bac60..7026765e91 100644 --- a/deployer/cli_app.py +++ b/deployer/cli_app.py @@ -20,7 +20,8 @@ app.add_typer( generate_app, name="generate", - help="Generate various types of assets. It currently supports generating files related to billing or new, dedicated clusters.", + help="Generate various types of assets. It currently supports generating files related to billing, " + "new dedicated clusters, helm upgrade strategies and resource allocation.", ) app.add_typer( cilogon_client_app, diff --git a/deployer/commands/generate/resource_allocation/update_nodeinfo.py b/deployer/commands/generate/resource_allocation/update_nodeinfo.py index c7dd0bf265..0b9c57e6e4 100644 --- a/deployer/commands/generate/resource_allocation/update_nodeinfo.py +++ b/deployer/commands/generate/resource_allocation/update_nodeinfo.py @@ -141,7 +141,8 @@ def node_info_update( ): """ Generates a new entry holding info about the capacity of a node of a certain instance type - or updates existing one in a json file called `node-capacity-info.json`. + or updates an existing one that is then used to update a json file called `node-capacity-info.json`. + This file is then used for generating the resource choices. """ try: with open(HERE / "node-capacity-info.json") as f: