feat: feat: backport from 1.28.0 (1.26.6) (#208)

* feat: backport features from 1.28.0

* chore: update to latest changes

* chore: upgrade go model file

* docs: update supported matrix versions

* feat: add release notes

* feat: added e2e upgrade tests in drone

* fix: mark new test executable

---------

Co-authored-by: Samuele Chiocca <samuele@sighup.io>

.drone.yml

@@ -159,7 +159,7 @@ steps:
     environment:
       CLUSTER_NAME: ${DRONE_REPO_NAME}-${DRONE_BUILD_NUMBER}
       KUBECONFIG: /drone/src/kubeconfig
-      FURYCTL_VERSION: v0.27.3-rc.2
+      FURYCTL_VERSION: v0.28.0-rc.4
     depends_on: [create Kind cluster]
     commands:
       - export KUBECONFIG=/drone/src/kubeconfig
@@ -197,14 +197,119 @@ volumes:
   - name: dockersock
     host:
       path: /var/run/docker.sock
+---
+name: e2e-kubernetes-1.26.5-1.26.6
+kind: pipeline
+type: docker
+
+depends_on:
+  - qa
+
+clone:
+  depth: 1
+
+platform:
+  os: linux
+  arch: amd64
+
+trigger:
+  ref:
+    include:
+      - refs/tags/**
+      - refs/heads/main
+      - refs/heads/release-v**
+    exclude:
+      - refs/tags/**-docs*
 
+steps:
+  - name: create Kind cluster
+    image: quay.io/sighup/dind-kind-kubectl-kustomize:0.20.0_1.29.1_3.10.0
+    pull: always
+    volumes:
+      - name: dockersock
+        path: /var/run/docker.sock
+    environment:
+      CLUSTER_VERSION: v1.26.4
+      CLUSTER_NAME: ${DRONE_REPO_NAME}-${DRONE_BUILD_NUMBER}-upgrades
+      # /drone/src is the default workdir for the pipeline
+      # using this folder we don't need to mount another
+      # shared volume between the steps
+      KUBECONFIG: /drone/src/kubeconfig-upgrades
+    commands:
+      # create a custom config to disable Kind's default CNI so
+      # we can test using KFD's networking module.
+      - |
+        cat <<EOF > kind-config.yaml
+        kind: Cluster
+        apiVersion: kind.x-k8s.io/v1alpha4
+        networking:
+          disableDefaultCNI: true
+        nodes:
+          - role: control-plane
+          - role: worker
+        EOF
+      # NOTE: kind's `--wait` flag that waits for the control-plane ot be ready
+      # does not work when disabling the default CNI. It will always go in timeout.
+      - kind create cluster --name $${CLUSTER_NAME} --image registry.sighup.io/fury/kindest/node:$${CLUSTER_VERSION} --config kind-config.yaml
+      # save the kubeconfig so we can use it from other steps.
+      - kind get kubeconfig --name $${CLUSTER_NAME} > $${KUBECONFIG}
+
+  - name: e2e-kfddistribution
+    # KUBECTL_KUSTOMIZE_HELM_YQ_ISTIOCTL_FURYCTL_BATS
+    image: quay.io/sighup/e2e-testing:1.1.0_0.11.0_3.1.1_1.9.4_1.26.3_3.5.3_4.33.3
+    pull: always
+    # we need to use host network to access Kind API port that is listening on the worker's loopback
+    # beacuse we mount the host's Docker socket to run Kind.
+    network_mode: host
+    environment:
+      CLUSTER_NAME: ${DRONE_REPO_NAME}-${DRONE_BUILD_NUMBER}-upgrades
+      KUBECONFIG: /drone/src/kubeconfig-upgrades
+      FURYCTL_VERSION: v0.28.0-rc.4
+    depends_on: [create Kind cluster]
+    commands:
+      - export KUBECONFIG=/drone/src/kubeconfig-upgrades
+      # We change the loopback IP in the kubeconfig to use the service hostname and keep the port.
+      # - 'sed -Ei "s#(server: https://)(.*)(:.*)#\1kind-cluster\3#" $${KUBECONFIG}'
+      - echo "Installing the correct furyctl version..."
+      - curl -L "https://github.com/sighupio/furyctl/releases/download/$${FURYCTL_VERSION}/furyctl-$(uname -s)-amd64.tar.gz" -o /tmp/furyctl.tar.gz && tar xfz /tmp/furyctl.tar.gz -C /tmp
+      # to use furyctl latest, use the following instead:
+      # - curl -L "https://github.com/sighupio/furyctl/releases/latest/download/furyctl-$(uname -s)-amd64.tar.gz" -o /tmp/furyctl.tar.gz && tar xfz /tmp/furyctl.tar.gz -C /tmp
+      - chmod +x /tmp/furyctl
+      # check that the kind cluster is ready before we move on
+      # - kubectl wait --timeout=180s --for=condition=ready pod --all -n kube-system
+      - until kubectl get serviceaccount default > /dev/null 2>&1; do echo "waiting for control-plane" && sleep 1; done
+      # finally, run the e2e tests
+      - tests/e2e-kfddistribution-upgrades.sh
+
+  - name: delete-kind-cluster
+    image: quay.io/sighup/dind-kind-kubectl-kustomize:0.20.0_1.29.1_3.10.0
+    volumes:
+      - name: dockersock
+        path: /var/run/docker.sock
+    environment:
+      CLUSTER_NAME: ${DRONE_REPO_NAME}-${DRONE_BUILD_NUMBER}-upgrades
+    commands:
+      # does not matter if the command fails
+      - kind delete cluster --name $${CLUSTER_NAME} || true
+    depends_on:
+      - e2e-kfddistribution
+    when:
+      status:
+        - success
+        - failure
+
+volumes:
+  - name: dockersock
+    host:
+      path: /var/run/docker.sock
 ---
 name: release
 kind: pipeline
 type: docker
 
 depends_on:
   - e2e-kubernetes-1.26
+  - e2e-kubernetes-1.26.5-1.26.6
 
 platform:
   os: linux

Furyfile.yaml

@@ -4,15 +4,15 @@
 
 ---
 versions:
-  auth: v0.1.0
-  aws: v4.1.0
-  dr: v2.2.0
-  ingress: v2.2.0
-  logging: v3.3.1
-  monitoring: v3.0.1
-  opa: v1.11.1
-  networking: v1.15.0
-  tracing: v1.0.2
+  auth: v0.2.0
+  aws: v4.2.0
+  dr: v2.3.0
+  ingress: v2.3.0
+  logging: v3.4.0
+  monitoring: v3.1.0
+  opa: v1.12.0
+  networking: v1.16.0
+  tracing: v1.0.3
 
 bases:
   - name: auth

Makefile

@@ -47,12 +47,10 @@ formattag:
 	| xargs -I {} sh -c 'formattag -file {}'
 
 .PHONY: lint-go
-
 lint-go:
 	@golangci-lint -v run --color=always --config=.rules/.golangci.yml ./...
 
 .PHONY: tools-go
-
 tools-go:
 	@go install github.com/evanphx/json-patch/cmd/json-patch@v5.6.0
 	@go install github.com/google/addlicense@v1.1.1
@@ -61,10 +59,10 @@ tools-go:
 	@go install github.com/daixiang0/gci@v0.10.1
 	@go install github.com/momaek/formattag@v0.0.9
 	@go install github.com/santhosh-tekuri/jsonschema/cmd/jv@v0.4.0
+	@go install github.com/sighupio/go-jsonschema@latest
 
-.PHONY: generate-private-schema dump-go-models
-
-generate-go-models: dump-private-schema
+.PHONY: _generate-go-models
+_generate-go-models: dump-private-schema
 	@go-jsonschema \
 		--package public \
 		--resolve-extension json \
@@ -86,6 +84,10 @@ generate-go-models: dump-private-schema
 		--output pkg/apis/onpremises/v1alpha2/public/schema.go \
 		schemas/public/onpremises-kfd-v1alpha2.json
 
+.PHONY: generate-go-models
+generate-go-models: _generate-go-models format-go
+
+.PHONY: dump-private-schema
 dump-private-schema:
 	@cat schemas/public/ekscluster-kfd-v1alpha2.json | \
 	json-patch -p schemas/private/ekscluster-kfd-v1alpha2.patch.json | \

README.md

@@ -1,14 +1,14 @@
-<!-- markdownlint-disable MD033 -->
+<!-- markdownlint-disable MD033 MD045 -->
 <h1 align="center">
   <img src="docs/assets/fury-epta-white.png" width="200px"/><br/>
   Kubernetes Fury Distribution
 </h1>
 
 <p align="center">Kubernetes Fury Distribution (KFD) is a certified battle-tested Kubernetes distribution based purely on upstream Kubernetes.</p>
-<!-- markdownlint-enable MD033 -->
+<!-- markdownlint-enable MD033 MD045 -->
 
-[![Build Status](http://ci.sighup.io/api/badges/sighupio/fury-distribution/status.svg?ref=refs/tags/v1.26.5)](http://ci.sighup.io/sighupio/fury-distribution)
-[![Release](https://img.shields.io/badge/release-v1.26.5-blue?label=FuryDistributionRelease)](https://github.com/sighupio/fury-distribution/releases/latest)
+[![Build Status](http://ci.sighup.io/api/badges/sighupio/fury-distribution/status.svg?ref=refs/tags/v1.26.6)](http://ci.sighup.io/sighupio/fury-distribution)
+[![Release](https://img.shields.io/badge/release-v1.26.6-blue?label=FuryDistributionRelease)](https://github.com/sighupio/fury-distribution/releases/latest)
 [![Slack](https://img.shields.io/badge/slack-@kubernetes/fury-yellow.svg?logo=slack)](https://kubernetes.slack.com/archives/C0154HYTAQH)
 [![License](https://img.shields.io/github/license/sighupio/fury-distribution)](https://github.com/sighupio/fury-distribution/blob/main/LICENSE)
 
@@ -31,11 +31,11 @@ KFD uses an **un-distribution model**. This means that we:
 
 ## Architecture 🏗
 
-<!-- markdownlint-disable MD033 -->
+<!-- markdownlint-disable MD033 MD045 -->
 <p align="center">
   <img src="docs/assets/kfd-architecture.png" width="800px"/>
 </p>
-<!-- markdownlint-enable MD033 -->
+<!-- markdownlint-enable MD033 MD045 -->
 
 Kubernetes Fury Distribution is structured on modules, and each module has a set of packages.
 
@@ -46,8 +46,40 @@ Kubernetes Fury Distribution is structured on modules, and each module has a set
 
 The standard way to deploy KFD is to:
 
-- Deploy all the [Core Modules](#core-modules-) of the distribution
-- Deploy (if needed) any of the [Addon modules](#add-on-modules-)
+- Deploy all the [Core Modules](#core-modules-) of the distribution using furyctl providers
+- Deploy (if needed) any of the [Addon modules](#add-on-modules-) using furyctl plugin feature
+
+### Recommended Hardware Requirements
+
+KFD is a modular and composable system, so hardware requirements ultimately depend on the modules and configuration chosen. Having said that, for a production-grade cluster a good starting point would be:
+
+A KFD production grade cluster will be composed of 3 node pools:
+
+- Control Plane: 3 nodes in HA.
+- Infrastructure: 3 nodes dedicated to running the infrastructural components of KFD (monitoring, logging, policy enforcement, etc., i.e. the modules).
+- Workers: where the application workload will run. This is up to you.
+- Load Balancers (optional): for on-premises installations, 2 load balancers in HA can be deployed to forward traffic to the control plane and the ingress controllers running in the infrastructure nodes.
+
+#### Nodes sizing
+
+| Node Role      | CPU (cores) | RAM (GB) | Disk (GB) | Qty. |
+| -------------- | ----------- | -------- | --------- | ---- |
+| Control Plane  | 2           | 8        | 50        | 3    |
+| Infrastructure | 4           | 16       | 50        | 3    |
+| Load Balancer  | 2           | 2        | 50        | 2    |
+
+#### Storage
+
+Some modules rely on persistent storage via PersistentVolumeClaims, by default (but configurable) the following capacity will be used:
+
+| Description                                | Size (GB) |
+| ------------------------------------------ | --------- |
+| Prometheus (metrics storage)               | 150       |
+| MinIO Monitoring (metrics storage, 20GBx6) | 120       |
+| MinIO Logging (logs storage, 20GBx6)       | 120       |
+| OpenSearch (logs storage)                  | 30        |
+| MinIO Tracing (traces storage)             | 120       |
+| **Total**                                  | **540**   |
 
 ### Core Modules 📦
 
@@ -94,11 +126,11 @@ If you are looking to run KFD in production and would like to learn more, SIGHUP
 
 Current supported versions of KFD are:
 
-|                                  KFD Version                                   | Kubernetes Version |
-| :----------------------------------------------------------------------------: | :----------------: |
-| [`1.27.3`](https://github.com/sighupio/fury-distribution/releases/tag/v1.27.3) |      `1.27.x`      |
-| [`1.26.5`](https://github.com/sighupio/fury-distribution/releases/tag/v1.26.5) |      `1.26.x`      |
-| [`1.25.10`](https://github.com/sighupio/fury-distribution/releases/tag/v1.25.10) |      `1.25.x`      |
+|                                  KFD Version                                     | Kubernetes Version |
+| :------------------------------------------------------------------------------: | :----------------: |
+| [`1.28.0`](https://github.com/sighupio/fury-distribution/releases/tag/v1.28.0)   |      `1.28.x`      |
+| [`1.27.5`](https://github.com/sighupio/fury-distribution/releases/tag/v1.27.5)   |      `1.27.x`      |
+| [`1.26.6`](https://github.com/sighupio/fury-distribution/releases/tag/v1.26.6)   |      `1.26.x`      |
 
 Check the [compatibility matrix][compatibility-matrix] for additional information about previous releases of the Distribution and the compatibility with `furyctl`.
 
@@ -163,5 +195,5 @@ KFD is open-source software and it's released under the following [LICENSE](LICE
 
 <!-- Misc -->
 [sighup-site]: https://sighup.io
-[CNCF]: https://landscape.cncf.io/card-mode?category=certified-kubernetes-distribution&grouping=category&organization=sighup
+[CNCF]: https://landscape.cncf.io/?group=certified-partners-and-providers&item=platform--certified-kubernetes-distribution--fury-distribution
 [cncf-conformance]: https://www.cncf.io/certification/software-conformance/

defaults/ekscluster-kfd-v1alpha2.yaml

@@ -8,6 +8,7 @@ data:
     secretGenerator: []
     patches: []
     patchesStrategicMerge: []
+    images: []
   # the common section will be used by all the templates in all modules, everything defined here is something used by all the KFD modules.
   common:
     # where all the KFD modules are downloaded
@@ -251,7 +252,7 @@ data:
           dex:
             host: ""
             ingressClass: ""
-          gangway: # only needed as default
+          gangplank: # only needed as default
             host: ""
             ingressClass: ""
         tolerations: null
@@ -274,6 +275,7 @@ data:
       dex:
         # see dex documentation for more information
         connectors: []
+        additionalStaticClients: []
       oidcKubernetesAuth: # only needed as default
         enabled: false
       baseDomain: ""

defaults/kfddistribution-kfd-v1alpha2.yaml

@@ -8,6 +8,7 @@ data:
     secretGenerator: []
     patches: []
     patchesStrategicMerge: []
+    images: []
   # the common section will be used by all the templates in all modules, everything defined here is something used by all the KFD modules.
   common:
     # where all the KFD modules are downloaded
@@ -238,7 +239,7 @@ data:
           dex:
             host: ""
             ingressClass: ""
-          gangway: # only needed as default
+          gangplank: # only needed as default
             host: ""
             ingressClass: ""
         tolerations: null
@@ -261,6 +262,7 @@ data:
       dex:
         # see dex documentation for more information
         connectors: []
+        additionalStaticClients: []
       oidcKubernetesAuth: # only needed as default
         enabled: false
       baseDomain: ""

defaults/onpremises-kfd-v1alpha2.yaml

@@ -8,6 +8,7 @@ data:
     secretGenerator: []
     patches: []
     patchesStrategicMerge: []
+    images: []
   # the common section will be used by all the templates in all modules, everything defined here is something used by all the KFD modules.
   common:
     # where all the KFD modules are downloaded
@@ -238,7 +239,7 @@ data:
           dex:
             host: ""
             ingressClass: ""
-          gangway:
+          gangplank:
             host: ""
             ingressClass: ""
         tolerations: null
@@ -261,6 +262,7 @@ data:
       dex:
         # see dex documentation for more information
         connectors: []
+        additionalStaticClients: []
       oidcKubernetesAuth:
         enabled: false
       baseDomain: ""