Merge remote-tracking branch 'origin' into bugs/nodename

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>

.github/workflows/single-binary.yml

@@ -159,6 +159,8 @@ jobs:
         with:
           python-version: "3.12"
       - uses: unionai/flytectl-setup-action@v0.0.3
+        with:
+          version: '0.9.2'
       - name: Setup sandbox
         run: |
           mkdir -p ~/.flyte/sandbox
@@ -197,6 +199,9 @@ jobs:
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/$line;
           done < flytesnacks/flyte_tests.txt
+      - name: Install Pytest
+        run: |
+          pip install pytest
       - name: End2End
         run: |
           make end2end_execute

.gitignore

@@ -38,3 +38,4 @@ docs/examples
 docs/_src
 docs/_projects
 docs/tests
+empty-config.yaml

CHANGELOG/CHANGELOG-v1.14.1.md

@@ -0,0 +1,19 @@
+# Flyte 1.14.1 Release Notes
+
+* Update flytestdlib and affected tools (copilot) for missing config.
+
+## What's Changed
+* docs: Refactor merge sort code example to use literalinclude by @davidlin20dev in https://github.com/flyteorg/flyte/pull/6091
+* [DOCS] Using ImageSpec in ContainerTask by @machichima in https://github.com/flyteorg/flyte/pull/6095
+* Eager doc updates by @wild-endeavor in https://github.com/flyteorg/flyte/pull/6099
+* Revert "fix: return the config file not found error" by @eapolinario in https://github.com/flyteorg/flyte/pull/6100
+* Remove notes on deprecated Batch size by @wild-endeavor in https://github.com/flyteorg/flyte/pull/6102
+* Upstream: Add labels to published execution events by @katrogan in https://github.com/flyteorg/flyte/pull/6104
+* Fix: Make distributed error aggregation opt-in by @fg91 in https://github.com/flyteorg/flyte/pull/6103
+* Add default labels and annotations to Ray worker pods too. by @katrogan in https://github.com/flyteorg/flyte/pull/6107
+* Fix: Remove the default search dialog if it exists (on CMD + K) by @chmod77 in https://github.com/flyteorg/flyte/pull/6106
+
+## New Contributors
+* @chmod77 made their first contribution in https://github.com/flyteorg/flyte/pull/6106
+
+**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.14.0...v1.14.1

boilerplate/flyte/end2end/Makefile

@@ -7,8 +7,12 @@
 end2end_execute: export FLYTESNACKS_PRIORITIES ?= P0
 end2end_execute: export FLYTESNACKS_VERSION ?= $(shell curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name)
 end2end_execute:
-	./boilerplate/flyte/end2end/end2end.sh ./boilerplate/flyte/end2end/functional-test-config.yaml --return_non_zero_on_failure
-
+	pytest ./boilerplate/flyte/end2end/test_run.py \
+		--flytesnacks_release_tag=$(FLYTESNACKS_VERSION) \
+		--priorities=$(FLYTESNACKS_PRIORITIES) \
+		--config_file=./boilerplate/flyte/end2end/functional-test-config.yaml \
+		--return_non_zero_on_failure
+
 .PHONY: k8s_integration_execute
 k8s_integration_execute:
 	echo "pass"

boilerplate/flyte/end2end/conftest.py

@@ -0,0 +1,47 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption("--flytesnacks_release_tag", required=True)
+    parser.addoption("--priorities", required=True)
+    parser.addoption("--config_file", required=True)
+    parser.addoption(
+        "--return_non_zero_on_failure",
+        action="store_true",
+        default=False,
+        help="Return a non-zero exit status if any workflow fails",
+    )
+    parser.addoption(
+        "--terminate_workflow_on_failure",
+        action="store_true",
+        default=False,
+        help="Abort failing workflows upon exit",
+    )
+    parser.addoption(
+        "--test_project_name", 
+        default="flytesnacks",
+        help="Name of project to run functional tests on"
+    )
+    parser.addoption(
+        "--test_project_domain", 
+        default="development", 
+        help="Name of domain in project to run functional tests on"
+    )
+    parser.addoption(
+        "--cluster_pool_name",
+        required=False,
+        type=str,
+        default=None,
+    )
+
+@pytest.fixture
+def setup_flytesnacks_env(pytestconfig):
+    return {
+        "flytesnacks_release_tag": pytestconfig.getoption("--flytesnacks_release_tag"),
+        "priorities": pytestconfig.getoption("--priorities"),
+        "config_file": pytestconfig.getoption("--config_file"),
+        "return_non_zero_on_failure": pytestconfig.getoption("--return_non_zero_on_failure"),
+        "terminate_workflow_on_failure": pytestconfig.getoption("--terminate_workflow_on_failure"),
+        "test_project_name": pytestconfig.getoption("--test_project_name"),
+        "test_project_domain": pytestconfig.getoption("--test_project_domain"),
+        "cluster_pool_name": pytestconfig.getoption("--cluster_pool_name"),
+    }

boilerplate/flyte/end2end/run-tests.py → boilerplate/flyte/end2end/test_run.py

@@ -5,7 +5,7 @@
 import traceback
 from typing import Dict, List, Optional
 
-import click
+import pytest
 import requests
 from flytekit.configuration import Config
 from flytekit.models.core.execution import WorkflowExecutionPhase
@@ -15,7 +15,6 @@
 WAIT_TIME = 10
 MAX_ATTEMPTS = 200
 
-
 def execute_workflow(
     remote: FlyteRemote,
     version,
@@ -27,7 +26,6 @@ def execute_workflow(
     wf = remote.fetch_workflow(name=workflow_name, version=version)
     return remote.execute(wf, inputs=inputs, wait=False, cluster_pool=cluster_pool_name)
 
-
 def executions_finished(
     executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]
 ) -> bool:
@@ -36,7 +34,6 @@ def executions_finished(
             return False
     return True
 
-
 def sync_executions(
     remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]
 ):
@@ -50,13 +47,11 @@ def sync_executions(
         print("GOT TO THE EXCEPT")
         print("COUNT THIS!")
 
-
 def report_executions(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]):
     for executions in executions_by_wfgroup.values():
         for execution in executions:
             print(execution)
 
-
 def schedule_workflow_groups(
     tag: str,
     workflow_groups: List[str],
@@ -65,10 +60,6 @@ def schedule_workflow_groups(
     parsed_manifest: List[dict],
     cluster_pool_name: Optional[str] = None,
 ) -> Dict[str, bool]:
-    """
-    Schedule workflows executions for all workflow groups and return True if all executions succeed, otherwise
-    return False.
-    """
     executions_by_wfgroup = {}
     # Schedule executions for each workflow group,
     for wf_group in workflow_groups:
@@ -120,30 +111,32 @@ def schedule_workflow_groups(
         results[wf_group] = len(non_succeeded_executions) == 0
     return results
 
-
 def valid(workflow_group, parsed_manifest):
     """
     Return True if a workflow group is contained in parsed_manifest,
     False otherwise.
     """
     return workflow_group in set(wf_group["name"] for wf_group in parsed_manifest)
 
+def test_run(setup_flytesnacks_env):
+
+    env = setup_flytesnacks_env
+
+    flytesnacks_release_tag = env["flytesnacks_release_tag"]
+    priorities = env["priorities"]
+    config_file_path = env["config_file"]
+    terminate_workflow_on_failure = env["terminate_workflow_on_failure"]
+    test_project_name = env["test_project_name"]
+    test_project_domain = env["test_project_domain"]
+    cluster_pool_name = env["cluster_pool_name"]
+    return_non_zero_on_failure = env["return_non_zero_on_failure"]
 
-def run(
-    flytesnacks_release_tag: str,
-    priorities: List[str],
-    config_file_path,
-    terminate_workflow_on_failure: bool,
-    test_project_name: str,
-    test_project_domain: str,
-    cluster_pool_name: Optional[str] = None,
-) -> List[Dict[str, str]]:
     remote = FlyteRemote(
         Config.auto(config_file=config_file_path),
         test_project_name,
         test_project_domain,
     )
-
+    
     # For a given release tag and priority, this function filters the workflow groups from the flytesnacks
     # manifest file. For example, for the release tag "v0.2.224" and the priority "P0" it returns [ "core" ].
     manifest_url = (
@@ -210,75 +203,15 @@ def run(
             "color": background_color,
         }
         results.append(result)
-    return results
-
-
-@click.command()
-@click.argument("flytesnacks_release_tag")
-@click.argument("priorities")
-@click.argument("config_file")
-@click.option(
-    "--return_non_zero_on_failure",
-    default=False,
-    is_flag=True,
-    help="Return a non-zero exit status if any workflow fails",
-)
-@click.option(
-    "--terminate_workflow_on_failure",
-    default=False,
-    is_flag=True,
-    help="Abort failing workflows upon exit",
-)
-@click.option(
-    "--test_project_name",
-    default="flytesnacks",
-    type=str,
-    is_flag=False,
-    help="Name of project to run functional tests on",
-)
-@click.option(
-    "--test_project_domain",
-    default="development",
-    type=str,
-    is_flag=False,
-    help="Name of domain in project to run functional tests on",
-)
-@click.argument(
-    "cluster_pool_name",
-    required=False,
-    type=str,
-    default=None,
-)
-def cli(
-    flytesnacks_release_tag,
-    priorities,
-    config_file,
-    return_non_zero_on_failure,
-    terminate_workflow_on_failure,
-    test_project_name,
-    test_project_domain,
-    cluster_pool_name,
-):
-    print(f"return_non_zero_on_failure={return_non_zero_on_failure}")
-    results = run(
-        flytesnacks_release_tag,
-        priorities,
-        config_file,
-        terminate_workflow_on_failure,
-        test_project_name,
-        test_project_domain,
-        cluster_pool_name,
-    )
 
-    # Write a json object in its own line describing the result of this run to stdout
     print(f"Result of run:\n{json.dumps(results)}")
 
-    # Return a non-zero exit code if core fails
     if return_non_zero_on_failure:
-        for result in results:
-            if result["status"] not in ("passing", "coming soon"):
-                sys.exit(1)
-
-
-if __name__ == "__main__":
-    cli()
+        fail_results = [result for result in results if result["status"] not in ("passing", "coming soon")]
+        if fail_results:
+            fail_msgs = [
+                f"Workflow '{r['label']}' failed with status '{r['status']}'" for r in fail_results
+            ]
+            pytest.fail("\n".join(fail_msgs))
+
+    assert results == [{"label": "core", "status": "passing", "color": "green"}]

charts/flyte-binary/README.md

@@ -41,7 +41,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.13.2"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.14.1"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

charts/flyte-binary/values.yaml

@@ -167,7 +167,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot  # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.13.2  # FLYTECOPILOT_TAG
+      tag: v1.14.1  # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent:

charts/flyte-core/README.md

@@ -110,8 +110,8 @@ helm install gateway bitnami/contour -n flyte
 | configmap.clusters.clusterConfigs | list | `[]` |  |
 | configmap.clusters.labelClusterMap | object | `{}` |  |
 | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI |
-| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
-| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
+| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.14.1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
+| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.14.1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
 | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","literal-offloading-config":{"enabled":true},"max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
 | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). |
 | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","literal-offloading-config":{"enabled":true},"max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
@@ -145,7 +145,7 @@ helm install gateway bitnami/contour -n flyte
 | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment |
-| datacatalog.image.tag | string | `"v1.13.2"` | Docker image tag |
+| datacatalog.image.tag | string | `"v1.14.1"` | Docker image tag |
 | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment |
 | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods |
 | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables |
@@ -182,7 +182,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
-| flyteadmin.image.tag | string | `"v1.13.2"` |  |
+| flyteadmin.image.tag | string | `"v1.14.1"` |  |
 | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create |
 | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment |
 | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods |
@@ -223,7 +223,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` |  |
 | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment |
-| flyteconsole.image.tag | string | `"v1.17.1"` |  |
+| flyteconsole.image.tag | string | `"v1.19.0"` |  |
 | flyteconsole.imagePullSecrets | list | `[]` | ImagePullSecrets to assign to the Flyteconsole deployment |
 | flyteconsole.livenessProbe | object | `{}` |  |
 | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment |
@@ -253,7 +253,7 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment |
-| flytepropeller.image.tag | string | `"v1.13.2"` |  |
+| flytepropeller.image.tag | string | `"v1.14.1"` |  |
 | flytepropeller.manager | bool | `false` |  |
 | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment |
 | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods |
@@ -285,7 +285,7 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
 | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment |
-| flytescheduler.image.tag | string | `"v1.13.2"` | Docker image tag |
+| flytescheduler.image.tag | string | `"v1.14.1"` | Docker image tag |
 | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment |
 | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods |
 | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables |