diff --git a/README.md b/README.md index 3569d9a..f5245b9 100644 --- a/README.md +++ b/README.md @@ -251,15 +251,15 @@ Once you have created the terraform.tfvars file edit the main.tf file (always in | `k3s_server_pool_size` | `no` | Number of k3s servers deployed. Default 1 | | `k3s_worker_pool_size` | `no` | Number of k3s workers deployed. Default 2 | | `k3s_extra_worker_node` | `no` | Boolean value, default true. Deploy the third worker nodes. The node will be deployed outside the worker instance pools. Using OCI always free account you can't create instance pools with more than two servers. This workaround solve this problem. | -| `install_nginx_ingress` | `no` | Boolean value, install kubernetes nginx ingress controller instead of Traefik. Default: true. For more information see [Nginx ingress controller](#nginx-ingress-controller) | -| `install_traefik2` | `no` | Boolean value, install [Traefik 2](https://traefik.io/) ingress controller instead of the [embedded k3s Traefik](https://docs.k3s.io/networking#traefik-ingress-controller). Default: false. | +| `ingress_controller` | `no` | Define the ingress controller to use. Valid values are: [default](https://docs.k3s.io/networking#traefik-ingress-controller), [nginx](#nginx-ingress-controller), [traefik2](https://traefik.io/) or [istio](https://istio.io/latest/docs/tasks/traffic-management/ingress/kubernetes-ingress/) | | `disable_ingress` | `no` | Boolean value, disable all ingress controllers. Default: false | | `ingress_controller_http_nodeport` | `no` | NodePort where nginx ingress will listen for http traffic. Default 30080 | | `ingress_controller_https_nodeport` | `no` | NodePort where nginx ingress will listen for https traffic. Default 30443 | | `install_longhorn` | `no` | Boolean value, install longhorn "Cloud native distributed block storage for Kubernetes". Default: true. To use longhorn set the *k3s_version* < v1.25.x [Ref.](https://github.com/longhorn/longhorn/issues/4003) | -| `longhorn_release` | `no` | Longhorn release. Default: v1.2.3 | +| `longhorn_release` | `no` | Longhorn release. Default: v1.4.0 | | `install_certmanager` | `no` | Boolean value, install [cert manager](https://cert-manager.io/) "Cloud native certificate management". Default: true | -| `certmanager_release` | `no` | Cert manager release. Default: v1.8.2 | +| `nginx_ingress_release` | `no` | Longhorn release. Default: v1.5.1 | +| `certmanager_release` | `no` | Cert manager release. Default: v1.11.0 | | `certmanager_email_address` | `no` | Email address used for signing https certificates. Defaul: changeme@example.com | | `install_argocd` | `no` | Boolean value, install [Argo CD](https://argo-cd.readthedocs.io/en/stable/) "a declarative, GitOps continuous delivery tool for Kubernetes.". Default: true | | `argocd_release` | `no` | Argo CD release. Default: v2.4.11 | diff --git a/data.tf b/data.tf index 45c295c..d06303d 100644 --- a/data.tf +++ b/data.tf @@ -15,9 +15,9 @@ data "cloudinit_config" "k3s_server_tpl" { k3s_token = random_password.k3s_token.result, is_k3s_server = true, disable_ingress = var.disable_ingress, - install_traefik2 = var.install_traefik2, - install_nginx_ingress = var.install_nginx_ingress, + ingress_controller = var.ingress_controller, nginx_ingress_release = var.nginx_ingress_release, + istio_release = var.istio_release, install_certmanager = var.install_certmanager, certmanager_release = var.certmanager_release, certmanager_email_address = var.certmanager_email_address, @@ -53,8 +53,6 @@ data "cloudinit_config" "k3s_worker_tpl" { disable_ingress = var.disable_ingress, k3s_url = oci_load_balancer_load_balancer.k3s_load_balancer.ip_address_details[0].ip_address, http_lb_port = var.http_lb_port, - install_traefik2 = var.install_traefik2, - install_nginx_ingress = var.install_nginx_ingress, install_longhorn = var.install_longhorn, https_lb_port = var.https_lb_port, ingress_controller_http_nodeport = var.ingress_controller_http_nodeport, diff --git a/example/.terraform.lock.hcl b/example/.terraform.lock.hcl index ac6b937..8a186e4 100644 --- a/example/.terraform.lock.hcl +++ b/example/.terraform.lock.hcl @@ -39,24 +39,24 @@ provider "registry.terraform.io/hashicorp/random" { } provider "registry.terraform.io/oracle/oci" { - version = "4.98.0" + version = "4.103.0" constraints = ">= 4.64.0" hashes = [ - "h1:+snjrBb1V2mejOejPzEK8Zmq8yfJljBQq5EDm2CJeqk=", - "zh:0eaf1b384d9eabbdd9ac5d97ca1f2cb378cda50beedcd64c3e2e362135a2d671", - "zh:329b4ad85442095028c95051135b0c5a1b79069b4666ca9aba00b5c9c7c36fd4", - "zh:437b131102be4da4beeee93eb603ff0054e2c4fbcc10a7848d7d6a8309360e24", - "zh:4735dd41ffba8fb7e609d5d8624d087e59eb62051952401aa1c5b37fb46afe85", - "zh:69064721b5c19c5524d5f44829cc977a55c82d83eadc96b683dbae31b3f31ab0", - "zh:9987c04e93708cfe98acc5e900c09c059bbb856a1ce21cff8822acac0ba80faf", + "h1:S2W+Ws/6rUPbWHJfY4opXJXCP7qkiIrhK7tUSxvRqgg=", + "zh:4353fcba6ecc24a9002620fb9be90f69aea8944be3e066661babd53e12e73070", + "zh:552aba3d96e2a243ddff23b28ad473e6299ca15e63b4de1fa0cde17733477663", + "zh:5fe58faa187364d73182c0131995a0a089a199767e6017b1107eedf288548027", + "zh:714e52b8937129f229528f20ac60acb2d5d1ea2ae491c1fc246e0e3d57ebfc4a", + "zh:7d72e2af797346457227a98557d8eb7807f24cd05700a9c309fe5615332363a3", + "zh:7d9d99753272232ac05bc19cce68fed88ec3874e8c0dbbc720b6afc38d37e933", + "zh:8acf325c514506473c8cebfc827fd69c4320468c23dc56eca603ef823d4c369d", + "zh:932ce1cb255f2d6859b8120be5b1e71b6c276cfc59022c6f79d8996a8c678ddd", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9d7ca69e185d637adf9d68bff56ac58a22606f94ba5ec590855a753b50e1ee09", - "zh:a3a243be0c44fdc368f5bb18efa767aafe69918196979abdae30a3ef1b490bde", - "zh:a3da991b7a2d4619fc18396858687ff757fc42a97c518325a26cf78f4adf27ad", - "zh:b0b9c0d3917757f3902a64aace04ed6fc3e3428e64f578302925dbe805fa7d6c", - "zh:bc63e96d3af8a25771bfe89a0731c706d218694224a76fe66d3e881dfcc3877d", - "zh:e05637ac506508c72ca41ca17444f4642d3ad5150d1fb5cf9eed0e61b67a0d42", - "zh:e9b55c344670135e89fac6a7450553dc30f0e587e0ab3d47567f68bd5cc9b7c3", - "zh:f83699e8397f3d8c36dbae57a8650a64194b91cfef2db110d505fa2a3308f111", + "zh:a6d83b758296989ef620808681640053419a99be9b507cf9f031e13ecc6b545c", + "zh:afe0c587bbec6680aed5c37a3946be0abc96c1edfee8104d76c8712d26bfa400", + "zh:eaef779b65d5f61be8d89dfd9b6aa22ee96f0dc980fcbb106d7601bb4e5caa5a", + "zh:eb2de2828a1f107acbb334f59745def95ce115f1bfdf73c84c32c00e250db5f3", + "zh:fac1395be2d866835d4dd99b41d19811d044c84ae0cc77c59c0d45e1bb97e947", + "zh:fcdd1dc048d6931fb6a116fbeda415b007715626e34005f2b175f4c80b943149", ] } diff --git a/files/k3s-install-agent.sh b/files/k3s-install-agent.sh index 2e3975d..f838435 100644 --- a/files/k3s-install-agent.sh +++ b/files/k3s-install-agent.sh @@ -67,13 +67,6 @@ if [[ "$operating_system" == "ubuntu" ]]; then apt-get update apt-get install -y software-properties-common jq DEBIAN_FRONTEND=noninteractive apt-get upgrade -y - %{ if install_nginx_ingress } - install_oci_cli_ubuntu - %{ endif } - - %{ if install_traefik2 } - install_oci_cli_ubuntu - %{ endif } %{ if ! disable_ingress } install_oci_cli_ubuntu @@ -96,13 +89,6 @@ if [[ "$operating_system" == "oraclelinux" ]]; then dnf -y update dnf -y install jq curl - %{ if install_nginx_ingress } - install_oci_cli_oracle - %{ endif } - - %{ if install_traefik2 } - install_oci_cli_oracle - %{ endif } %{ if ! disable_ingress } install_oci_cli_oracle @@ -268,14 +254,6 @@ nginx -t systemctl restart nginx } -%{ if install_nginx_ingress } -proxy_protocol_stuff -%{ endif } - -%{ if install_traefik2 } -proxy_protocol_stuff -%{ endif } - %{ if ! disable_ingress } proxy_protocol_stuff %{ endif } diff --git a/files/k3s-install-server.sh b/files/k3s-install-server.sh index d24fa12..c6ac19f 100644 --- a/files/k3s-install-server.sh +++ b/files/k3s-install-server.sh @@ -41,6 +41,263 @@ install_helm() { /root/get_helm.sh } + +render_istio_config(){ +cat << 'EOF' > "$ISTIO_CONFIG_FILE" +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +metadata: + namespace: istio-system +spec: + hub: docker.io/istio + tag: ${istio_release} + + # You may override parts of meshconfig by uncommenting the following lines. + meshConfig: + defaultConfig: + proxyMetadata: {} + enablePrometheusMerge: true + # Opt-out of global http2 upgrades. + # Destination rule is used to opt-in. + # h2_upgrade_policy: DO_NOT_UPGRADE + + # Traffic management feature + components: + base: + enabled: true + pilot: + enabled: true + + # Istio Gateway feature + ingressGateways: + - name: istio-ingressgateway + enabled: true + k8s: + resources: + requests: + cpu: 10m + memory: 40Mi + service: + type: NodePort + ports: + ## You can add custom gateway ports in user values overrides, but it must include those ports since helm replaces. + # Note that AWS ELB will by default perform health checks on the first port + # on this list. Setting this to the health check port will ensure that health + # checks always work. https://github.com/istio/istio/issues/12503 + - port: 15021 + targetPort: 15021 + nodePort: 30521 + name: status-port + - port: 80 + targetPort: 8080 + nodePort: ${ingress_controller_http_nodeport} + name: http2 + - port: 443 + targetPort: 8443 + nodePort: ${ingress_controller_https_nodeport} + name: https + - port: 31400 + targetPort: 31400 + nodePort: 31400 + name: tcp + # This is the port where sni routing happens + - port: 15443 + targetPort: 15443 + nodePort: 30543 + name: tls + egressGateways: + - name: istio-egressgateway + enabled: false + + # Istio CNI feature + cni: + enabled: false + + # Remote and config cluster configuration for an external istiod + istiodRemote: + enabled: false + + # Global values passed through to helm global.yaml. + # Please keep this in sync with manifests/charts/global.yaml + values: + defaultRevision: "" + global: + istioNamespace: istio-system + istiod: + enableAnalysis: false + logging: + level: "default:info" + logAsJson: false + pilotCertProvider: istiod + jwtPolicy: third-party-jwt + proxy: + image: proxyv2 + clusterDomain: "cluster.local" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 2000m + memory: 1024Mi + logLevel: warning + componentLogLevel: "misc:error" + privileged: false + enableCoreDump: false + statusPort: 15020 + readinessInitialDelaySeconds: 1 + readinessPeriodSeconds: 2 + readinessFailureThreshold: 30 + includeIPRanges: "*" + excludeIPRanges: "" + excludeOutboundPorts: "" + excludeInboundPorts: "" + autoInject: enabled + tracer: "zipkin" + proxy_init: + image: proxyv2 + resources: + limits: + cpu: 2000m + memory: 1024Mi + requests: + cpu: 10m + memory: 10Mi + # Specify image pull policy if default behavior isn't desired. + # Default behavior: latest images will be Always else IfNotPresent. + imagePullPolicy: "" + operatorManageWebhooks: false + tracer: + lightstep: {} + zipkin: {} + datadog: {} + stackdriver: {} + imagePullSecrets: [] + oneNamespace: false + defaultNodeSelector: {} + configValidation: true + multiCluster: + enabled: false + clusterName: "" + omitSidecarInjectorConfigMap: false + network: "" + defaultResources: + requests: + cpu: 10m + defaultPodDisruptionBudget: + enabled: true + priorityClassName: "" + useMCP: false + sds: + token: + aud: istio-ca + sts: + servicePort: 0 + meshNetworks: {} + mountMtlsCerts: false + base: + enableCRDTemplates: false + validationURL: "" + pilot: + autoscaleEnabled: true + autoscaleMin: 1 + autoscaleMax: 5 + replicaCount: 1 + image: pilot + traceSampling: 1.0 + env: {} + cpu: + targetAverageUtilization: 80 + nodeSelector: {} + keepaliveMaxServerConnectionAge: 30m + enableProtocolSniffingForOutbound: true + enableProtocolSniffingForInbound: true + deploymentLabels: + podLabels: {} + configMap: true + + telemetry: + enabled: true + v2: + enabled: true + metadataExchange: + wasmEnabled: false + prometheus: + wasmEnabled: false + enabled: true + stackdriver: + enabled: false + logging: false + monitoring: false + topology: false + configOverride: {} + + istiodRemote: + injectionURL: "" + + gateways: + istio-egressgateway: + env: {} + autoscaleEnabled: true + type: ClusterIP + name: istio-egressgateway + secretVolumes: + - name: egressgateway-certs + secretName: istio-egressgateway-certs + mountPath: /etc/istio/egressgateway-certs + - name: egressgateway-ca-certs + secretName: istio-egressgateway-ca-certs + mountPath: /etc/istio/egressgateway-ca-certs + + istio-ingressgateway: + autoscaleEnabled: true + type: LoadBalancer + name: istio-ingressgateway + env: {} + secretVolumes: + - name: ingressgateway-certs + secretName: istio-ingressgateway-certs + mountPath: /etc/istio/ingressgateway-certs + - name: ingressgateway-ca-certs + secretName: istio-ingressgateway-ca-certs + mountPath: /etc/istio/ingressgateway-ca-certs +EOF + +cat << 'EOF' > "$ENABLE_ISTIO_PROXY_PROTOCOL" +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: proxy-protocol + namespace: istio-system +spec: + configPatches: + - applyTo: LISTENER + patch: + operation: MERGE + value: + listener_filters: + - name: envoy.listener.proxy_protocol + - name: envoy.listener.tls_inspector + workloadSelector: + labels: + istio: ingressgateway +EOF +} + +install_and_configure_istio(){ + echo "Installing Istio..." + curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${istio_release} sh - + + ISTIO_CONFIG_FILE=/root/custom_istio_config.yaml + ENABLE_ISTIO_PROXY_PROTOCOL=/root/enable_istio_proxy_protocol.yaml + render_istio_config + + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + mv /istio-${istio_release} /opt + /opt/istio-${istio_release}/bin/istioctl install -y -f $ISTIO_CONFIG_FILE + kubectl apply -f $ENABLE_ISTIO_PROXY_PROTOCOL +} + install_and_configure_traefik2() { export KUBECONFIG=/etc/rancher/k3s/k3s.yaml @@ -53,11 +310,11 @@ install_and_configure_traefik2() { helm repo update TRAEFIK_VALUES_FILE=/root/traefik2_values.yaml - render_traefil2_config + render_traefik2_config helm install --namespace=traefik -f $TRAEFIK_VALUES_FILE traefik traefik/traefik } -render_traefil2_config() { +render_traefik2_config() { cat << 'EOF' > "$TRAEFIK_VALUES_FILE" service: enabled: true @@ -237,6 +494,26 @@ metadata: EOF } +install_and_configure_nginx(){ + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-${nginx_ingress_release}/deploy/static/provider/baremetal/deploy.yaml + NGINX_RESOURCES_FILE=/root/nginx-ingress-resources.yaml + render_nginx_config + kubectl apply -f $NGINX_RESOURCES_FILE +} + +install_ingress(){ + INGRESS_CONTROLLER=$1 + if [[ "$INGRESS_CONTROLLER" == "nginx" ]]; then + install_and_configure_nginx + elif [[ "$INGRESS_CONTROLLER" == "traefik2" ]]; then + install_and_configure_traefik2 + elif [[ "$INGRESS_CONTROLLER" == "istio" ]]; then + install_and_configure_istio + else + echo "Ingress controller not supported" + fi +} + render_staging_issuer(){ STAGING_ISSUER_RESOURCE=$1 cat << 'EOF' > "$STAGING_ISSUER_RESOURCE" @@ -352,12 +629,10 @@ k3s_install_params+=("--flannel-iface $flannel_iface") k3s_install_params+=("--disable traefik") %{ endif } -%{ if install_nginx_ingress } +%{ if ! disable_ingress } +%{ if ingress_controller != "default" } k3s_install_params+=("--disable traefik") %{ endif } - -%{ if install_traefik2 } -k3s_install_params+=("--disable traefik") %{ endif } %{ if expose_kubeapi } @@ -408,26 +683,12 @@ if [[ "$first_instance" == "$instance_id" ]]; then fi %{ endif } -%{ if install_nginx_ingress } -if [[ "$first_instance" == "$instance_id" ]]; then - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-${nginx_ingress_release}/deploy/static/provider/baremetal/deploy.yaml - NGINX_RESOURCES_FILE=/root/nginx-ingress-resources.yaml - render_nginx_config - kubectl apply -f $NGINX_RESOURCES_FILE -fi -%{ endif } - - %{ if ! disable_ingress } +%{ if ingress_controller != "default" } if [[ "$first_instance" == "$instance_id" ]]; then - install_and_configure_traefik2 + install_ingress ${ingress_controller} fi %{ endif } - -%{ if install_traefik2 } -if [[ "$first_instance" == "$instance_id" ]]; then - install_and_configure_traefik2 -fi %{ endif } %{ if install_certmanager } diff --git a/vars.tf b/vars.tf index 9ebc9b3..344ac29 100644 --- a/vars.tf +++ b/vars.tf @@ -164,9 +164,9 @@ variable "my_public_ip_cidr" { description = "My public ip CIDR" } -variable "install_traefik2" { - type = bool - default = false +variable "istio_release" { + type = string + default = "1.16.1" } variable "disable_ingress" { @@ -174,14 +174,18 @@ variable "disable_ingress" { default = false } -variable "install_nginx_ingress" { - type = bool - default = true +variable "ingress_controller" { + type = string + default = "default" + validation { + condition = contains(["default", "nginx", "traefik2", "istio"], var.ingress_controller) + error_message = "Supported ingress controllers are: default, nginx, traefik2, istio" + } } variable "nginx_ingress_release" { type = string - default = "v1.3.1" + default = "v1.5.1" } variable "install_certmanager" { @@ -191,7 +195,7 @@ variable "install_certmanager" { variable "certmanager_release" { type = string - default = "v1.9.1" + default = "v1.11.0" } variable "certmanager_email_address" { @@ -206,7 +210,7 @@ variable "install_longhorn" { variable "longhorn_release" { type = string - default = "v1.3.1" + default = "v1.4.0" } variable "install_argocd" {