Add annotations for input parameters in scripts

Add support for easier declaration of script input parameters using typing.Annotated. It supports enum, default, name, and description. It is an experimental feature, enabled by script_annotation flag in global_config. Signed-off-by: GitHub <noreply@github.com>
argoproj-labs · Jul 24, 2023 · 5733e30 · 5733e30
1 parent f1507f4
commit 5733e30
Show file tree

Hide file tree

Showing 261 changed files with 9,437 additions and 4,443 deletions.
diff --git a/Makefile b/Makefile
@@ -25,7 +25,8 @@ codegen: models services examples init-files
 
 .PHONY: check-codegen
 check-codegen: ## Check if the code is up to date
-check-codegen: codegen
+check-codegen:
+	@$(MAKE) codegen
 	git diff --exit-code || "Code is not up-to-date. Please run 'make codegen'"
 
 .PHONY: format
@@ -48,29 +49,31 @@ workflows-models: ## Generate the Workflows models portion of Argo Workflows
 	@poetry run datamodel-codegen \
 		--url $(OPENAPI_SPEC_URL) \
 		--snake-case-field \
-		--target-python-version 3.7 \
+		--target-python-version 3.8 \
 		--output src/hera/workflows/models \
 		--base-class hera.shared._base_model.BaseModel \
 		--wrap-string-literal \
 		--disable-appending-item-suffix \
-		--disable-timestamp
+		--disable-timestamp \
+		--use-default-kwarg
 	@poetry run python scripts/models.py $(OPENAPI_SPEC_URL) workflows
-	@poetry run stubgen -o src -p hera.workflows.models && rm -rf **/__init__.pyi
+	@poetry run stubgen -o src -p hera.workflows.models && find src/hera/workflows/models -name '__init__.pyi' -delete
 	@$(MAKE) format
 
 .PHONY: events-models
 events-models: ## Generate the Events models portion of Argo Workflows
 	@poetry run datamodel-codegen \
 		--url $(OPENAPI_SPEC_URL) \
 		--snake-case-field \
-		--target-python-version 3.7 \
+		--target-python-version 3.8 \
 		--output src/hera/events/models \
 		--base-class hera.shared._base_model.BaseModel \
 		--wrap-string-literal \
 		--disable-appending-item-suffix \
-		--disable-timestamp
+		--disable-timestamp \
+		--use-default-kwarg
 	@poetry run python scripts/models.py $(OPENAPI_SPEC_URL) events
-	@poetry run stubgen -o src -p hera.events.models && rm -rf **/__init__.pyi
+	@poetry run stubgen -o src -p hera.events.models && find src/hera/events/models -name '__init__.pyi' -delete
 	@$(MAKE) format
 
 .PHONY: models

diff --git a/README.md b/README.md
@@ -34,9 +34,6 @@ and its crew were specially protected by the goddess Hera.
 Hera is a Python framework for constructing and submitting Argo Workflows. The main goal of Hera is to make the Argo
 ecosystem accessible by simplifying workflow construction and submission.
 
-You can watch the introductory Hera presentation at the "Argo Workflows and Events Community Meeting 20 Oct
-2021" [here](https://www.youtube.com/watch?v=QETfzfVV-GY&t=181s)!
-
 # Table of content
 
 - [Hera](#hera)
@@ -46,6 +43,7 @@ You can watch the introductory Hera presentation at the "Argo Workflows and Even
 - [Examples](#examples)
     - [Single step script](#single-step-script)
     - [DAG diamond](#dag-diamond)
+- [Presentations](#presentations)
 - [Contributing](#contributing)
 - [Comparison](#comparison)
 
@@ -140,6 +138,18 @@ w.create()
 
 See the [examples](./examples/) directory for a collection of Argo workflow construction and submission via Hera!
 
+# Presentations
+
+- [Argo Workflows and Events Community Meeting 20 Oct 2021 - Hera introductory presentation](https://youtu.be/QETfzfVV-GY?t=181)
+- [Argo Workflows and Events Community Meeting 15 June 2022 - Hera project update](https://youtu.be/sdkBDPOdQ-g?t=231)
+- [KubeCon/ArgoCon EU 2023 - Scaling gene therapy with Argo Workflows and Hera](https://www.youtube.com/watch?v=h2TEw8kd1Ds)
+- [Unsticking ourselves from Glue - Migrating PayIt's Data Pipelines to Argo Workflows and Hera](https://youtu.be/sSLFVIIEKcE?t=2088)
+
+# Blogs
+
+- [Hera introduction and motivation](https://www.dynotx.com/hera-the-missing-argo-workflows-python-sdk/)
+- [Dyno is scaling gene therapy research with cloud-native tools like Argo Workflows and Hera](https://www.dynotx.com/argo-workflows-hera/)
+
 # Contributing
 
 If you plan to submit contributions to Hera you can install Hera in a virtual environment managed by `poetry`:

diff --git a/docs/examples/workflows-examples.md b/docs/examples/workflows-examples.md
@@ -9,5 +9,102 @@ the [DAG Diamond example](workflows/upstream/dag_diamond.md), to demonstrate how
 These examples are generated and compared in Hera's CI/CD pipeline to ensure that all the examples are possible to
 recreate in Hera.
 
+> If you'd like to contribute missing examples, please see the table below and follow the
+> [Contributing Guide](/CONTRIBUTING.md)!
+
 The "Hera" collection shows off features and idiomatic usage of Hera such as the
 [Coinflip example](workflows/coinflip.md) using the script decorator.
+
+## List of missing examples
+| Example |
+|---------|
+| [artifacts-workflowtemplate](https://github.com/argoproj/argo-workflows/blob/master/examples/artifacts-workflowtemplate.yaml) |
+| [buildkit-template](https://github.com/argoproj/argo-workflows/blob/master/examples/buildkit-template.yaml) |
+| [ci](https://github.com/argoproj/argo-workflows/blob/master/examples/ci.yaml) |
+| [ci-workflowtemplate](https://github.com/argoproj/argo-workflows/blob/master/examples/ci-workflowtemplate.yaml) |
+| [cluster-workflow-template/clustertemplates](https://github.com/argoproj/argo-workflows/blob/master/examples/cluster-workflow-template/clustertemplates.yaml) |
+| [conditional-parameters](https://github.com/argoproj/argo-workflows/blob/master/examples/conditional-parameters.yaml) |
+| [conditionals](https://github.com/argoproj/argo-workflows/blob/master/examples/conditionals.yaml) |
+| [conditionals-complex](https://github.com/argoproj/argo-workflows/blob/master/examples/conditionals-complex.yaml) |
+| [configmaps/simple-parameters-configmap](https://github.com/argoproj/argo-workflows/blob/master/examples/configmaps/simple-parameters-configmap.yaml) |
+| [cron-backfill](https://github.com/argoproj/argo-workflows/blob/master/examples/cron-backfill.yaml) |
+| [daemon-step](https://github.com/argoproj/argo-workflows/blob/master/examples/daemon-step.yaml) |
+| [daemoned-stateful-set-with-service](https://github.com/argoproj/argo-workflows/blob/master/examples/daemoned-stateful-set-with-service.yaml) |
+| [dag-coinflip](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-coinflip.yaml) |
+| [dag-conditional-artifacts](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-conditional-artifacts.yaml) |
+| [dag-continue-on-fail](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-continue-on-fail.yaml) |
+| [dag-daemon-task](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-daemon-task.yaml) |
+| [dag-diamond-steps](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-diamond-steps.yaml) |
+| [dag-disable-failFast](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-disable-failFast.yaml) |
+| [dag-inline-clusterworkflowtemplate](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-inline-clusterworkflowtemplate.yaml) |
+| [dag-inline-cronworkflow](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-inline-cronworkflow.yaml) |
+| [dag-inline-workflow](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-inline-workflow.yaml) |
+| [dag-multiroot](https://github.com/argoproj/argo-workflows/blob/master/examples/dag-multiroot.yaml) |
+| [data-transformations](https://github.com/argoproj/argo-workflows/blob/master/examples/data-transformations.yaml) |
+| [dns-config](https://github.com/argoproj/argo-workflows/blob/master/examples/dns-config.yaml) |
+| [exit-code-output-variable](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-code-output-variable.yaml) |
+| [exit-handler-dag-level](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-dag-level.yaml) |
+| [exit-handler-slack](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-slack.yaml) |
+| [exit-handler-with-param](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-with-param.yaml) |
+| [exit-handlers](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) |
+| [expression-destructure-json](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-destructure-json.yaml) |
+| [expression-reusing-verbose-snippets](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-reusing-verbose-snippets.yaml) |
+| [expression-tag-template-workflow](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) |
+| [fibonacci-seq-conditional-param](https://github.com/argoproj/argo-workflows/blob/master/examples/fibonacci-seq-conditional-param.yaml) |
+| [fun-with-gifs](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) |
+| [gc-ttl](https://github.com/argoproj/argo-workflows/blob/master/examples/gc-ttl.yaml) |
+| [global-outputs](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) |
+| [global-parameters-from-configmap](https://github.com/argoproj/argo-workflows/blob/master/examples/global-parameters-from-configmap.yaml) |
+| [handle-large-output-results](https://github.com/argoproj/argo-workflows/blob/master/examples/handle-large-output-results.yaml) |
+| [hello-hybrid](https://github.com/argoproj/argo-workflows/blob/master/examples/hello-hybrid.yaml) |
+| [hello-windows](https://github.com/argoproj/argo-workflows/blob/master/examples/hello-windows.yaml) |
+| [hello-world](https://github.com/argoproj/argo-workflows/blob/master/examples/hello-world.yaml) |
+| [http-hello-world](https://github.com/argoproj/argo-workflows/blob/master/examples/http-hello-world.yaml) |
+| [http-success-condition](https://github.com/argoproj/argo-workflows/blob/master/examples/http-success-condition.yaml) |
+| [influxdb-ci](https://github.com/argoproj/argo-workflows/blob/master/examples/influxdb-ci.yaml) |
+| [k8s-jobs](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml) |
+| [k8s-orchestration](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-orchestration.yaml) |
+| [k8s-owner-reference](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml) |
+| [k8s-patch](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-patch.yaml) |
+| [k8s-patch-basic](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-patch-basic.yaml) |
+| [k8s-set-owner-reference](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-set-owner-reference.yaml) |
+| [k8s-wait-wf](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-wait-wf.yaml) |
+| [label-value-from-workflow](https://github.com/argoproj/argo-workflows/blob/master/examples/label-value-from-workflow.yaml) |
+| [life-cycle-hooks-tmpl-level](https://github.com/argoproj/argo-workflows/blob/master/examples/life-cycle-hooks-tmpl-level.yaml) |
+| [loops-param-argument](https://github.com/argoproj/argo-workflows/blob/master/examples/loops-param-argument.yaml) |
+| [loops-sequence](https://github.com/argoproj/argo-workflows/blob/master/examples/loops-sequence.yaml) |
+| [map-reduce](https://github.com/argoproj/argo-workflows/blob/master/examples/map-reduce.yaml) |
+| [memoize-simple](https://github.com/argoproj/argo-workflows/blob/master/examples/memoize-simple.yaml) |
+| [nested-workflow](https://github.com/argoproj/argo-workflows/blob/master/examples/nested-workflow.yaml) |
+| [parallelism-nested-workflow](https://github.com/argoproj/argo-workflows/blob/master/examples/parallelism-nested-workflow.yaml) |
+| [parameter-aggregation](https://github.com/argoproj/argo-workflows/blob/master/examples/parameter-aggregation.yaml) |
+| [parameter-aggregation-dag](https://github.com/argoproj/argo-workflows/blob/master/examples/parameter-aggregation-dag.yaml) |
+| [parameter-aggregation-script](https://github.com/argoproj/argo-workflows/blob/master/examples/parameter-aggregation-script.yaml) |
+| [pod-gc-strategy](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-gc-strategy.yaml) |
+| [pod-gc-strategy-with-label-selector](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-gc-strategy-with-label-selector.yaml) |
+| [pod-metadata](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-metadata.yaml) |
+| [pod-metadata-wf-field](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-metadata-wf-field.yaml) |
+| [pod-spec-from-previous-step](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-spec-from-previous-step.yaml) |
+| [pod-spec-patch](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-spec-patch.yaml) |
+| [pod-spec-yaml-patch](https://github.com/argoproj/argo-workflows/blob/master/examples/pod-spec-yaml-patch.yaml) |
+| [recursive-for-loop](https://github.com/argoproj/argo-workflows/blob/master/examples/recursive-for-loop.yaml) |
+| [resource-flags](https://github.com/argoproj/argo-workflows/blob/master/examples/resource-flags.yaml) |
+| [resubmit](https://github.com/argoproj/argo-workflows/blob/master/examples/resubmit.yaml) |
+| [retry-conditional](https://github.com/argoproj/argo-workflows/blob/master/examples/retry-conditional.yaml) |
+| [retry-container-to-completion](https://github.com/argoproj/argo-workflows/blob/master/examples/retry-container-to-completion.yaml) |
+| [retry-on-error](https://github.com/argoproj/argo-workflows/blob/master/examples/retry-on-error.yaml) |
+| [retry-with-steps](https://github.com/argoproj/argo-workflows/blob/master/examples/retry-with-steps.yaml) |
+| [scripts-bash](https://github.com/argoproj/argo-workflows/blob/master/examples/scripts-bash.yaml) |
+| [scripts-javascript](https://github.com/argoproj/argo-workflows/blob/master/examples/scripts-javascript.yaml) |
+| [scripts-python](https://github.com/argoproj/argo-workflows/blob/master/examples/scripts-python.yaml) |
+| [secrets](https://github.com/argoproj/argo-workflows/blob/master/examples/secrets.yaml) |
+| [status-reference](https://github.com/argoproj/argo-workflows/blob/master/examples/status-reference.yaml) |
+| [step-level-timeout](https://github.com/argoproj/argo-workflows/blob/master/examples/step-level-timeout.yaml) |
+| [synchronization-mutex-wf-level](https://github.com/argoproj/argo-workflows/blob/master/examples/synchronization-mutex-wf-level.yaml) |
+| [template-defaults](https://github.com/argoproj/argo-workflows/blob/master/examples/template-defaults.yaml) |
+| [testvolume](https://github.com/argoproj/argo-workflows/blob/master/examples/testvolume.yaml) |
+| [timeouts-step](https://github.com/argoproj/argo-workflows/blob/master/examples/timeouts-step.yaml) |
+| [work-avoidance](https://github.com/argoproj/argo-workflows/blob/master/examples/work-avoidance.yaml) |
+| [workflow-count-resourcequota](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-count-resourcequota.yaml) |
+| [workflow-event-binding/event-consumer-workfloweventbinding](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml) |
+| [workflow-template/templates](https://github.com/argoproj/argo-workflows/blob/master/examples/workflow-template/templates.yaml) |
diff --git a/docs/examples/workflows/artifact_with_fanout.md b/docs/examples/workflows/artifact_with_fanout.md
@@ -8,25 +8,25 @@
 === "Hera"
 
     ```python linenums="1"
-    from hera.workflows import DAG, Artifact, Workflow, script
+    from hera.workflows import DAG, Artifact, NoneArchiveStrategy, Workflow, script
 
 
-    @script(outputs=Artifact(name="test", path="/file"))
+    @script(outputs=Artifact(name="out-art", path="/tmp/file", archive=NoneArchiveStrategy()))
     def writer():
         import json
 
-        with open("/file", "w+") as f:
+        with open("/tmp/file", "w+") as f:
             for i in range(10):
                 f.write(json.dumps(i) + "\n")
 
 
-    @script(inputs=Artifact(name="test", path="/file"))
+    @script(inputs=Artifact(name="in-art", path="/tmp/file"))
     def fanout():
         import json
         import sys
 
         indices = []
-        with open("/file", "r") as f:
+        with open("/tmp/file", "r") as f:
             for line in f.readlines():
                 indices.append(line.strip())
         json.dump(indices, sys.stdout)
@@ -37,15 +37,12 @@
         print(i)
 
 
-    # assumes you used `hera.set_global_token` and `hera.set_global_host` so that the workflow can be submitted
     with Workflow(generate_name="artifact-with-fanout-", entrypoint="d") as w:
         with DAG(name="d"):
             w_ = writer()
-            f = fanout(arguments={"w": w_.get_artifact("test")})
+            f = fanout(arguments=w_.get_artifact("out-art").as_name("in-art"))
             c = consumer(with_param=f.result)
             w_ >> f >> c
-
-    w.to_yaml()
     ```
 
 === "YAML"
@@ -63,12 +60,10 @@
           - name: writer
             template: writer
           - arguments:
-              parameters:
-              - name: w
-                value: '{"name": "test", "archive": null, "archive_logs": null, "artifact_gc":
-                  null, "deleted": null, "from_": "{{tasks.writer.outputs.artifacts.test}}",
-                  "from_expression": null, "global_name": null, "mode": null, "path":
-                  "/file", "recurse_mode": null, "sub_path": null}'
+              artifacts:
+              - from: '{{tasks.writer.outputs.artifacts.out-art}}'
+                name: in-art
+                path: /tmp/file
             depends: writer
             name: fanout
             template: fanout
@@ -84,27 +79,30 @@
       - name: writer
         outputs:
           artifacts:
-          - name: test
-            path: /file
+          - archive:
+              none: {}
+            name: out-art
+            path: /tmp/file
         script:
           command:
           - python
           image: python:3.8
           source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\
-            with open('/file', 'w+') as f:\n    for i in range(10):\n        f.write(json.dumps(i)\
+            with open('/tmp/file', 'w+') as f:\n    for i in range(10):\n        f.write(json.dumps(i)\
             \ + '\\n')"
       - inputs:
           artifacts:
-          - name: test
-            path: /file
+          - name: in-art
+            path: /tmp/file
         name: fanout
         script:
           command:
           - python
           image: python:3.8
           source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\
-            import sys\nindices = []\nwith open('/file', 'r') as f:\n    for line in f.readlines():\n\
-            \        indices.append(line.strip())\njson.dump(indices, sys.stdout)"
+            import sys\nindices = []\nwith open('/tmp/file', 'r') as f:\n    for line\
+            \ in f.readlines():\n        indices.append(line.strip())\njson.dump(indices,\
+            \ sys.stdout)"
       - inputs:
           parameters:
           - name: i