Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into upstream-array-node-idl-change
Browse files Browse the repository at this point in the history
Signed-off-by: Paul Dittamo <pvdittamo@gmail.com>
pvditt committed Nov 25, 2024

Verified

This commit was signed with the committer’s verified signature.
marc-aurele-besner Marc-Aurele Besner
2 parents 12cde77 + 172e816 commit 39122f5
Showing 98 changed files with 4,579 additions and 1,287 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ jobs:
- name: Before Build
run: ${{ inputs.before-build }}
- name: Build and Push Image
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
2 changes: 1 addition & 1 deletion .github/workflows/sandbox.yml
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@ jobs:
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
password: "${{ secrets.FLYTE_BOT_PAT }}"
- name: Build and push DIND Image
uses: docker/build-push-action@v2
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/arm64, linux/amd64
10 changes: 5 additions & 5 deletions .github/workflows/single-binary.yml
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ jobs:
run: |
mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
- name: Export ARM64 Image
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/arm64
@@ -84,7 +84,7 @@ jobs:
file: Dockerfile
outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
- name: Export AMD64 Image
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64
@@ -108,7 +108,7 @@ jobs:
password: "${{ secrets.FLYTE_BOT_PAT }}"
- name: Build and push Image
if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/arm64, linux/amd64
@@ -144,7 +144,7 @@ jobs:
driver-opts: image=moby/buildkit:master
buildkitd-flags: "--allow-insecure-entitlement security.insecure"
- name: Build sandbox image for functional tests
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: docker/sandbox-bundled
load: true
@@ -240,7 +240,7 @@ jobs:
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
password: "${{ secrets.FLYTE_BOT_PAT }}"
- name: Build and push multi-arch image
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: docker/sandbox-bundled
allow: "security.insecure"
10 changes: 5 additions & 5 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -5,8 +5,8 @@ FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
FROM --platform=${BUILDPLATFORM} golang:1.22-bookworm AS flytebuilder

ARG TARGETARCH
ENV GOARCH "${TARGETARCH}"
ENV GOOS linux
ENV GOARCH="${TARGETARCH}"
ENV GOOS=linux

WORKDIR /flyteorg/build

@@ -29,10 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r
FROM debian:bookworm-slim

ARG FLYTE_VERSION
ENV FLYTE_VERSION "${FLYTE_VERSION}"
ENV FLYTE_VERSION="${FLYTE_VERSION}"

ENV DEBCONF_NONINTERACTIVE_SEEN true
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
ENV DEBIAN_FRONTEND=noninteractive

# Install core packages
RUN apt-get update && apt-get install --no-install-recommends --yes \
6 changes: 3 additions & 3 deletions Dockerfile.datacatalog
Original file line number Diff line number Diff line change
@@ -3,12 +3,12 @@
#
# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst

FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder


ARG TARGETARCH
ENV GOARCH "${TARGETARCH}"
ENV GOOS linux
ENV GOARCH="${TARGETARCH}"
ENV GOOS=linux

RUN apk add git openssh-client make curl

8 changes: 4 additions & 4 deletions Dockerfile.flyteadmin
Original file line number Diff line number Diff line change
@@ -3,11 +3,11 @@
#
# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst

FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder

ARG TARGETARCH
ENV GOARCH "${TARGETARCH}"
ENV GOOS linux
ENV GOARCH="${TARGETARCH}"
ENV GOOS=linux

RUN apk add git openssh-client make curl

@@ -41,7 +41,7 @@ ENV PATH="/artifacts:${PATH}"

# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
FROM alpine:3.18
LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin

COPY --from=builder /artifacts /bin

8 changes: 4 additions & 4 deletions Dockerfile.flytecopilot
Original file line number Diff line number Diff line change
@@ -3,12 +3,12 @@
#
# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst

FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder


ARG TARGETARCH
ENV GOARCH "${TARGETARCH}"
ENV GOOS linux
ENV GOARCH="${TARGETARCH}"
ENV GOOS=linux

RUN apk add git openssh-client make curl

@@ -32,7 +32,7 @@ ENV PATH="/artifacts:${PATH}"

# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
FROM alpine:3.18
LABEL org.opencontainers.image.source https://github.com/lyft/flyteplugins
LABEL org.opencontainers.image.source=https://github.com/lyft/flyteplugins

COPY --from=builder /artifacts /bin

8 changes: 4 additions & 4 deletions Dockerfile.flytepropeller
Original file line number Diff line number Diff line change
@@ -4,12 +4,12 @@
# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst


FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder


ARG TARGETARCH
ENV GOARCH "${TARGETARCH}"
ENV GOOS linux
ENV GOARCH="${TARGETARCH}"
ENV GOOS=linux

RUN apk add git openssh-client make curl

@@ -33,7 +33,7 @@ ENV PATH="/artifacts:${PATH}"

# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
FROM alpine:3.18
LABEL org.opencontainers.image.source https://github.com/flyteorg/flytepropeller
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytepropeller

COPY --from=builder /artifacts /bin

8 changes: 4 additions & 4 deletions Dockerfile.flytescheduler
Original file line number Diff line number Diff line change
@@ -4,12 +4,12 @@
# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst


FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder


ARG TARGETARCH
ENV GOARCH "${TARGETARCH}"
ENV GOOS linux
ENV GOARCH="${TARGETARCH}"
ENV GOOS=linux

RUN apk add git openssh-client make curl

@@ -36,7 +36,7 @@ ENV PATH="/artifacts:${PATH}"

# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
FROM alpine:3.18
LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin

COPY --from=builder /artifacts /bin

1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -135,6 +135,7 @@ go-tidy:
make -C flyteplugins go-tidy
make -C flytestdlib go-tidy
make -C flytecopilot go-tidy
make -C flytectl go-tidy

.PHONY: lint-helm-charts
lint-helm-charts:
4 changes: 1 addition & 3 deletions README.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions charts/flyte-binary/README.md
Original file line number Diff line number Diff line change
@@ -63,6 +63,7 @@ Chart for basic single Flyte executable deployment
| configuration.logging.plugins.kubernetes.templateUri | string | `""` | |
| configuration.logging.plugins.stackdriver.enabled | bool | `false` | |
| configuration.logging.plugins.stackdriver.templateUri | string | `""` | |
| configuration.propeller.createCRDs | bool | `true` | |
| configuration.storage.metadataContainer | string | `"my-organization-flyte-container"` | |
| configuration.storage.provider | string | `"s3"` | |
| configuration.storage.providerConfig.azure.account | string | `"storage-account-name"` | |
2 changes: 1 addition & 1 deletion charts/flyte-binary/templates/configmap.yaml
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ data:
show-source: true
level: {{ default 1 .Values.configuration.logging.level }}
propeller:
create-flyteworkflow-crd: true
create-flyteworkflow-crd: {{ .Values.configuration.propeller.createCRDs }}
webhook:
certDir: /var/run/flyte/certs
localCert: true
32 changes: 32 additions & 0 deletions charts/flyte-binary/templates/crds/flyteworkflow.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{{- if not .Values.configuration.propeller.createCRDs }}
{{- if $.Capabilities.APIVersions.Has "apiextensions.k8s.io/v1/CustomResourceDefinition" }}
apiVersion: apiextensions.k8s.io/v1
{{- else }}
apiVersion: apiextensions.k8s.io/v1beta1
{{- end }}
kind: CustomResourceDefinition
metadata:
name: flyteworkflows.flyte.lyft.com
spec:
group: flyte.lyft.com
names:
kind: FlyteWorkflow
plural: flyteworkflows
shortNames:
- fly
singular: flyteworkflow
scope: Namespaced
{{- if $.Capabilities.APIVersions.Has "apiextensions.k8s.io/v1/CustomResourceDefinition" }}
versions:
- name: v1alpha1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
properties:
{{- else }}
version: v1alpha1
{{- end }}
{{- end }}
4 changes: 4 additions & 0 deletions charts/flyte-binary/values.yaml
Original file line number Diff line number Diff line change
@@ -176,6 +176,10 @@ configuration:
timeouts:
GetTask: 10s
defaultTimeout: 10s
# propeller Specify configuration for Flyte Propeller
propeller:
# createCRDs If true, Propeller will install CRDs at runtime, if false, CRDs will be installed during helm install
createCRDs: true
# externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
# If set, no Flyte configuration will be generated by this chart
externalConfigMap: ""
4 changes: 2 additions & 2 deletions charts/flyte-core/README.md
Original file line number Diff line number Diff line change
@@ -191,7 +191,7 @@ helm install gateway bitnami/contour -n flyte
| flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment |
| flyteadmin.secrets | object | `{}` | |
| flyteadmin.securityContext | object | `{"fsGroup":65534,"fsGroupChangePolicy":"Always","runAsNonRoot":true,"runAsUser":1001,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for flyteadmin pod(s). |
| flyteadmin.service | object | `{"additionalPorts":[],"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin |
| flyteadmin.service | object | `{"additionalPorts":[],"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"appProtocols":{"enabled":false},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin |
| flyteadmin.service.additionalPorts | list | `[]` | Appends additional ports to the service spec. |
| flyteadmin.serviceAccount | object | `{"alwaysCreate":false,"annotations":{},"clusterRole":{"apiGroups":["","flyte.lyft.com","rbac.authorization.k8s.io"],"resources":["configmaps","flyteworkflows","namespaces","pods","resourcequotas","roles","rolebindings","secrets","services","serviceaccounts","spark-role","limitranges"],"verbs":["*"]},"create":true,"createClusterRole":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlyteAdmin |
| flyteadmin.serviceAccount.alwaysCreate | bool | `false` | Should a service account always be created for flyteadmin even without an actual flyteadmin deployment running (e.g. for multi-cluster setups) |
@@ -234,7 +234,7 @@ helm install gateway bitnami/contour -n flyte
| flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment |
| flyteconsole.resources | object | `{"limits":{"cpu":"500m","memory":"250Mi"},"requests":{"cpu":"10m","memory":"50Mi"}}` | Default resources requests and limits for Flyteconsole deployment |
| flyteconsole.securityContext | object | `{"fsGroupChangePolicy":"OnRootMismatch","runAsNonRoot":true,"runAsUser":1000,"seLinuxOptions":{"type":"spc_t"}}` | Sets securityContext for flyteconsole pod(s). |
| flyteconsole.service | object | `{"annotations":{},"type":"ClusterIP"}` | Service settings for Flyteconsole |
| flyteconsole.service | object | `{"annotations":{},"appProtocols":{"enabled":false},"type":"ClusterIP"}` | Service settings for Flyteconsole |
| flyteconsole.serviceMonitor | object | `{"enabled":false,"interval":"60s","labels":{},"scrapeTimeout":"30s"}` | Settings for flyteconsole service monitor |
| flyteconsole.serviceMonitor.enabled | bool | `false` | If enabled create the flyteconsole service monitor |
| flyteconsole.serviceMonitor.interval | string | `"60s"` | Sets the interval at which metrics will be scraped by prometheus |
8 changes: 8 additions & 0 deletions charts/flyte-core/templates/admin/service.yaml
Original file line number Diff line number Diff line change
@@ -20,22 +20,30 @@ spec:
- name: http
port: 80
protocol: TCP
{{- if .Values.flyteadmin.service.appProtocols.enabled }}
appProtocol: TCP
{{- end }}
targetPort: 8088
- name: grpc
port: 81
protocol: TCP
# intentionally set to TCP instead of grpc
{{- if .Values.flyteadmin.service.appProtocols.enabled }}
appProtocol: TCP
{{- end }}
targetPort: 8089
- name: redoc
protocol: TCP
{{- if .Values.flyteadmin.service.appProtocols.enabled }}
appProtocol: TCP
{{- end }}
port: 87
targetPort: 8087
- name: http-metrics
protocol: TCP
{{- if .Values.flyteadmin.service.appProtocols.enabled }}
appProtocol: TCP
{{- end }}
port: 10254
{{- with .Values.flyteadmin.service.additionalPorts -}}
{{ tpl (toYaml .) $ | nindent 4 }}
2 changes: 2 additions & 0 deletions charts/flyte-core/templates/console/service.yaml
Original file line number Diff line number Diff line change
@@ -16,7 +16,9 @@ spec:
- name: http
port: 80
protocol: TCP
{{- if .Values.flyteconsole.service.appProtocols.enabled }}
appProtocol: TCP
{{- end }}
targetPort: 8080
{{- if .Values.flyteconsole.serviceMonitor.enabled }}
- name: http-metrics
4 changes: 4 additions & 0 deletions charts/flyte-core/values.yaml
Original file line number Diff line number Diff line change
@@ -50,6 +50,8 @@ flyteadmin:
- flyteexamples
# -- Service settings for Flyteadmin
service:
appProtocols:
enabled: false
annotations:
projectcontour.io/upstream-protocol.h2c: grpc
type: ClusterIP
@@ -407,6 +409,8 @@ flyteconsole:
memory: 50Mi
# -- Service settings for Flyteconsole
service:
appProtocols:
enabled: false
annotations: {}
type: ClusterIP
# -- Annotations for Flyteconsole pods
5 changes: 0 additions & 5 deletions deployment/eks/flyte_aws_scheduler_helm_generated.yaml
Original file line number Diff line number Diff line change
@@ -751,22 +751,18 @@ spec:
- name: http
port: 80
protocol: TCP
appProtocol: TCP
targetPort: 8088
- name: grpc
port: 81
protocol: TCP
# intentionally set to TCP instead of grpc
appProtocol: TCP
targetPort: 8089
- name: redoc
protocol: TCP
appProtocol: TCP
port: 87
targetPort: 8087
- name: http-metrics
protocol: TCP
appProtocol: TCP
port: 10254
selector:
app.kubernetes.io/name: flyteadmin
@@ -789,7 +785,6 @@ spec:
- name: http
port: 80
protocol: TCP
appProtocol: TCP
targetPort: 8080
selector:
app.kubernetes.io/name: flyteconsole
Loading

0 comments on commit 39122f5

Please sign in to comment.