Merge branch 'master' into upstream-array-node-idl-change

Signed-off-by: Paul Dittamo <pvdittamo@gmail.com>

.github/workflows/publish.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Before Build
         run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ inputs.dockerfile }}

.github/workflows/sandbox.yml

@@ -53,7 +53,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64

.github/workflows/single-binary.yml

@@ -73,7 +73,7 @@ jobs:
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64
@@ -84,7 +84,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -108,7 +108,7 @@ jobs:
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push Image
         if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -144,7 +144,7 @@ jobs:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           load: true
@@ -240,7 +240,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"

Dockerfile

@@ -5,8 +5,8 @@ FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
 FROM --platform=${BUILDPLATFORM} golang:1.22-bookworm AS flytebuilder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 WORKDIR /flyteorg/build
 
@@ -29,10 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r
 FROM debian:bookworm-slim
 
 ARG FLYTE_VERSION
-ENV FLYTE_VERSION "${FLYTE_VERSION}"
+ENV FLYTE_VERSION="${FLYTE_VERSION}"
 
-ENV DEBCONF_NONINTERACTIVE_SEEN true
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Install core packages
 RUN apt-get update && apt-get install --no-install-recommends --yes \

Dockerfile.datacatalog

@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 

Dockerfile.flyteadmin

@@ -3,11 +3,11 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -41,7 +41,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytecopilot

@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -32,7 +32,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/lyft/flyteplugins
+LABEL org.opencontainers.image.source=https://github.com/lyft/flyteplugins
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytepropeller

@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -33,7 +33,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flytepropeller
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytepropeller
 
 COPY --from=builder /artifacts /bin
 

Dockerfile.flytescheduler

@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -36,7 +36,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

Makefile

@@ -135,6 +135,7 @@ go-tidy:
 	make -C flyteplugins go-tidy
 	make -C flytestdlib go-tidy
 	make -C flytecopilot go-tidy
+	make -C flytectl go-tidy
 
 .PHONY: lint-helm-charts
 lint-helm-charts:

charts/flyte-binary/README.md

@@ -63,6 +63,7 @@ Chart for basic single Flyte executable deployment
 | configuration.logging.plugins.kubernetes.templateUri | string | `""` |  |
 | configuration.logging.plugins.stackdriver.enabled | bool | `false` |  |
 | configuration.logging.plugins.stackdriver.templateUri | string | `""` |  |
+| configuration.propeller.createCRDs | bool | `true` |  |
 | configuration.storage.metadataContainer | string | `"my-organization-flyte-container"` |  |
 | configuration.storage.provider | string | `"s3"` |  |
 | configuration.storage.providerConfig.azure.account | string | `"storage-account-name"` |  |

charts/flyte-binary/templates/configmap.yaml

@@ -40,7 +40,7 @@ data:
       show-source: true
       level: {{ default 1 .Values.configuration.logging.level }}
     propeller:
-      create-flyteworkflow-crd: true
+      create-flyteworkflow-crd: {{ .Values.configuration.propeller.createCRDs }}
     webhook:
       certDir: /var/run/flyte/certs
       localCert: true

charts/flyte-binary/templates/crds/flyteworkflow.yaml

@@ -0,0 +1,32 @@
+{{- if not .Values.configuration.propeller.createCRDs }}
+{{- if $.Capabilities.APIVersions.Has "apiextensions.k8s.io/v1/CustomResourceDefinition" }}
+apiVersion: apiextensions.k8s.io/v1
+{{- else }}
+apiVersion: apiextensions.k8s.io/v1beta1
+{{- end }}
+kind: CustomResourceDefinition
+metadata:
+  name: flyteworkflows.flyte.lyft.com
+spec:
+  group: flyte.lyft.com
+  names:
+    kind: FlyteWorkflow
+    plural: flyteworkflows
+    shortNames:
+      - fly
+    singular: flyteworkflow
+  scope: Namespaced
+{{- if $.Capabilities.APIVersions.Has "apiextensions.k8s.io/v1/CustomResourceDefinition" }}
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          x-kubernetes-preserve-unknown-fields: true
+          properties:
+{{- else }}
+  version: v1alpha1
+{{- end }}
+{{- end }}

charts/flyte-binary/values.yaml

@@ -176,6 +176,10 @@ configuration:
       timeouts:
         GetTask: 10s
       defaultTimeout: 10s
+  # propeller Specify configuration for Flyte Propeller
+  propeller:
+    # createCRDs If true, Propeller will install CRDs at runtime, if false, CRDs will be installed during helm install
+    createCRDs: true
   # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
   # If set, no Flyte configuration will be generated by this chart
   externalConfigMap: ""

charts/flyte-core/README.md

@@ -191,7 +191,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment |
 | flyteadmin.secrets | object | `{}` |  |
 | flyteadmin.securityContext | object | `{"fsGroup":65534,"fsGroupChangePolicy":"Always","runAsNonRoot":true,"runAsUser":1001,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for flyteadmin pod(s). |
-| flyteadmin.service | object | `{"additionalPorts":[],"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin |
+| flyteadmin.service | object | `{"additionalPorts":[],"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"appProtocols":{"enabled":false},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin |
 | flyteadmin.service.additionalPorts | list | `[]` | Appends additional ports to the service spec. |
 | flyteadmin.serviceAccount | object | `{"alwaysCreate":false,"annotations":{},"clusterRole":{"apiGroups":["","flyte.lyft.com","rbac.authorization.k8s.io"],"resources":["configmaps","flyteworkflows","namespaces","pods","resourcequotas","roles","rolebindings","secrets","services","serviceaccounts","spark-role","limitranges"],"verbs":["*"]},"create":true,"createClusterRole":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlyteAdmin |
 | flyteadmin.serviceAccount.alwaysCreate | bool | `false` | Should a service account always be created for flyteadmin even without an actual flyteadmin deployment running (e.g. for multi-cluster setups) |
@@ -234,7 +234,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment |
 | flyteconsole.resources | object | `{"limits":{"cpu":"500m","memory":"250Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Flyteconsole deployment |
 | flyteconsole.securityContext | object | `{"fsGroupChangePolicy":"OnRootMismatch","runAsNonRoot":true,"runAsUser":1000,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for flyteconsole pod(s). |
-| flyteconsole.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Flyteconsole |
+| flyteconsole.service | object | `{"annotations":{},"appProtocols":{"enabled":false},"type":"ClusterIP"}` | Service settings for Flyteconsole |
 | flyteconsole.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flyteconsole service monitor |
 | flyteconsole.serviceMonitor.enabled | bool | `false` | If enabled create the flyteconsole service monitor |
 | flyteconsole.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |

charts/flyte-core/templates/admin/service.yaml

@@ -20,22 +20,30 @@ spec:
     - name: http
       port: 80
       protocol: TCP
+      {{- if .Values.flyteadmin.service.appProtocols.enabled }}
       appProtocol: TCP
+      {{- end }}
       targetPort: 8088
     - name: grpc
       port: 81
       protocol: TCP
       # intentionally set to TCP instead of grpc
+      {{- if .Values.flyteadmin.service.appProtocols.enabled }}
       appProtocol: TCP
+      {{- end }}
       targetPort: 8089
     - name: redoc
       protocol: TCP
+      {{- if .Values.flyteadmin.service.appProtocols.enabled }}
       appProtocol: TCP
+      {{- end }}
       port: 87
       targetPort: 8087
     - name: http-metrics
       protocol: TCP
+      {{- if .Values.flyteadmin.service.appProtocols.enabled }}
       appProtocol: TCP
+      {{- end }}
       port: 10254
     {{- with .Values.flyteadmin.service.additionalPorts -}}
     {{ tpl (toYaml .) $ | nindent 4 }}

charts/flyte-core/templates/console/service.yaml

@@ -16,7 +16,9 @@ spec:
   - name: http
     port: 80
     protocol: TCP
+    {{- if .Values.flyteconsole.service.appProtocols.enabled }}
     appProtocol: TCP
+    {{- end }}
     targetPort: 8080
     {{- if .Values.flyteconsole.serviceMonitor.enabled  }}
   - name: http-metrics

charts/flyte-core/values.yaml

@@ -50,6 +50,8 @@ flyteadmin:
     - flyteexamples
   # -- Service settings for Flyteadmin
   service:
+    appProtocols:
+      enabled: false
     annotations:
       projectcontour.io/upstream-protocol.h2c: grpc
     type: ClusterIP
@@ -407,6 +409,8 @@ flyteconsole:
       memory: 50Mi
   # -- Service settings for Flyteconsole
   service:
+    appProtocols:
+      enabled: false
     annotations: {}
     type: ClusterIP
   # -- Annotations for Flyteconsole pods

deployment/eks/flyte_aws_scheduler_helm_generated.yaml

@@ -751,22 +751,18 @@ spec:
     - name: http
       port: 80
       protocol: TCP
-      appProtocol: TCP
       targetPort: 8088
     - name: grpc
       port: 81
       protocol: TCP
       # intentionally set to TCP instead of grpc
-      appProtocol: TCP
       targetPort: 8089
     - name: redoc
       protocol: TCP
-      appProtocol: TCP
       port: 87
       targetPort: 8087
     - name: http-metrics
       protocol: TCP
-      appProtocol: TCP
       port: 10254
   selector: 
     app.kubernetes.io/name: flyteadmin
@@ -789,7 +785,6 @@ spec:
   - name: http
     port: 80
     protocol: TCP
-    appProtocol: TCP
     targetPort: 8080
   selector: 
     app.kubernetes.io/name: flyteconsole