From b0326377b415916943006310e56620fb1403a4a0 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Oct 2024 15:59:19 +0200 Subject: [PATCH] feat: Support non-default cluster domain (#870) * adapt to op-rs 0.79.0 * adapt changelog * cargo +nightly update --breaking -Z unstable-options * Update rust/crd/src/lib.rs Co-authored-by: Sebastian Bernauer * improve error handling * adapt to op-rs 0.80.0 --------- Co-authored-by: Sebastian Bernauer Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 6 + Cargo.lock | 180 ++++++++++++------- Cargo.toml | 2 +- deploy/helm/zookeeper-operator/values.yaml | 4 + rust/crd/src/authentication.rs | 3 +- rust/crd/src/lib.rs | 19 +- rust/crd/src/security.rs | 48 +++-- rust/operator-binary/src/discovery.rs | 6 +- rust/operator-binary/src/main.rs | 9 +- rust/operator-binary/src/zk_controller.rs | 115 ++++++++---- rust/operator-binary/src/znode_controller.rs | 39 ++-- 11 files changed, 292 insertions(+), 139 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a52d45..1fc97a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- The operator can now run on Kubernetes clusters using a non-default cluster domain. + Use the env var `KUBERNETES_CLUSTER_DOMAIN` or the operator Helm chart property `kubernetesClusterDomain` to set a non-default cluster domain ([#870]). + ### Changed - Reduce CRD size from `483KB` to `57KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#853]): @@ -20,6 +25,7 @@ All notable changes to this project will be documented in this file. [#853]: https://github.com/stackabletech/zookeeper-operator/pull/853 [#857]: https://github.com/stackabletech/zookeeper-operator/pull/857 +[#870]: https://github.com/stackabletech/zookeeper-operator/pull/870 [#872]: https://github.com/stackabletech/zookeeper-operator/pull/872 ## [24.7.0] - 2024-07-24 diff --git a/Cargo.lock b/Cargo.lock index 7dc49dde..883a16f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "async-broadcast" @@ -146,7 +146,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -157,7 +157,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "built" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4" +checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" dependencies = [ "chrono", "git2", @@ -270,9 +270,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.30" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "jobserver", "libc", @@ -329,7 +329,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -444,7 +444,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -455,7 +455,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -466,7 +466,7 @@ checksum = "bc2323e10c92e1cf4d86e11538512e6dc03ceb586842970b6332af3d4046a046" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -525,6 +525,18 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.82", +] + [[package]] name = "either" version = "1.13.0" @@ -549,6 +561,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "enum-ordinalize" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.82", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -673,7 +705,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1115,9 +1147,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.95.0" +version = "0.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa21063c854820a77c5d7f8deeb7ffa55246d8304e4bcd8cce2956752c6604f8" +checksum = "efffeb3df0bd4ef3e5d65044573499c0e4889b988070b08c50b25b1329289a1f" dependencies = [ "k8s-openapi", "kube-client", @@ -1128,9 +1160,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.95.0" +version = "0.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c2355f5c9d8a11900e71a6fe1e47abd5ec45bf971eb4b162ffe97b46db9bb7" +checksum = "8bf471ece8ff8d24735ce78dac4d091e9fcb8d74811aeb6b75de4d1c3f5de0f1" dependencies = [ "base64 0.22.1", "bytes", @@ -1166,9 +1198,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.95.0" +version = "0.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3030bd91c9db544a50247e7d48d7db9cf633c172732dce13351854526b1e666" +checksum = "f42346d30bb34d1d7adc5c549b691bce7aa3a1e60254e68fab7e2d7b26fe3d77" dependencies = [ "chrono", "form_urlencoded", @@ -1184,29 +1216,29 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.95.0" +version = "0.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa98be978eddd70a773aa8e86346075365bfb7eb48783410852dbf7cb57f0c27" +checksum = "f9364e04cc5e0482136c6ee8b7fb7551812da25802249f35b3def7aaa31e82ad" dependencies = [ "darling", "proc-macro2", "quote", "serde_json", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] name = "kube-runtime" -version = "0.95.0" +version = "0.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5895cb8aa641ac922408f128b935652b34c2995f16ad7db0984f6caa50217914" +checksum = "d3fbf1f6ffa98e65f1d2a9a69338bb60605d46be7edf00237784b89e62c9bd44" dependencies = [ "ahash", "async-broadcast", "async-stream", "async-trait", "backoff", - "derivative", + "educe", "futures 0.3.31", "hashbrown 0.14.5", "json-patch", @@ -1231,9 +1263,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libgit2-sys" @@ -1423,7 +1455,7 @@ dependencies = [ "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.3.0", + "ordered-float 4.4.0", "percent-encoding", "rand", "thiserror", @@ -1442,9 +1474,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537" +checksum = "83e7ccb95e240b7c9506a3d544f10d935e142cc90b0a1d56954fb44d89ad6b97" dependencies = [ "num-traits", ] @@ -1531,7 +1563,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1562,7 +1594,7 @@ checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1600,9 +1632,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" dependencies = [ "unicode-ident", ] @@ -1738,9 +1770,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "log", "once_cell", @@ -1846,7 +1878,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1857,11 +1889,10 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "secrecy" -version = "0.8.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" dependencies = [ - "serde", "zeroize", ] @@ -1921,7 +1952,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1932,14 +1963,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2060,7 +2091,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2081,8 +2112,8 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "stackable-operator" -version = "0.76.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.76.0#a7e70f174fb043a1766e0a80de95834cb4f7513d" +version = "0.80.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.80.0#6fbe32300b60f95e0baa2ab0ff2daf961b06531c" dependencies = [ "chrono", "clap", @@ -2092,6 +2123,7 @@ dependencies = [ "dockerfile-parser", "either", "futures 0.3.31", + "indexmap", "json-patch", "k8s-openapi", "kube", @@ -2106,6 +2138,7 @@ dependencies = [ "serde_yaml", "snafu 0.8.5", "stackable-operator-derive", + "stackable-shared", "strum", "tokio", "tracing", @@ -2118,12 +2151,24 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.76.0#a7e70f174fb043a1766e0a80de95834cb4f7513d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.80.0#6fbe32300b60f95e0baa2ab0ff2daf961b06531c" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", +] + +[[package]] +name = "stackable-shared" +version = "0.0.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.80.0#6fbe32300b60f95e0baa2ab0ff2daf961b06531c" +dependencies = [ + "kube", + "semver", + "serde", + "serde_yaml", + "snafu 0.8.5", ] [[package]] @@ -2188,7 +2233,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2210,15 +2255,21 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "thiserror" version = "1.0.64" @@ -2236,7 +2287,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2343,7 +2394,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2400,14 +2451,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.13" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" dependencies = [ "futures-core", "futures-util", - "pin-project", "pin-project-lite", + "sync_wrapper", "tokio", "tokio-util", "tower-layer", @@ -2417,16 +2468,15 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bitflags 2.6.0", "bytes", "http", "http-body", - "http-body-util", "mime", "pin-project-lite", "tower-layer", @@ -2478,7 +2528,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2681,7 +2731,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", "wasm-bindgen-shared", ] @@ -2703,7 +2753,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2861,7 +2911,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8f6d63f9..b3d7c737 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.76.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.80.0" } product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } diff --git a/deploy/helm/zookeeper-operator/values.yaml b/deploy/helm/zookeeper-operator/values.yaml index 2ecded77..ad77b7ac 100644 --- a/deploy/helm/zookeeper-operator/values.yaml +++ b/deploy/helm/zookeeper-operator/values.yaml @@ -47,3 +47,7 @@ nodeSelector: {} tolerations: [] affinity: {} + +# When running on a non-default Kubernetes cluster domain, the cluster domain can be configured here. +# See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details. +# kubernetesClusterDomain: my-cluster.local diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index 0a850c64..cf271139 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -73,7 +73,8 @@ impl ResolvedAuthenticationClasses { AuthenticationClassProvider::Tls(_) => {} AuthenticationClassProvider::Ldap(_) | AuthenticationClassProvider::Oidc(_) - | AuthenticationClassProvider::Static(_) => { + | AuthenticationClassProvider::Static(_) + | AuthenticationClassProvider::Kerberos(_) => { return Err(Error::AuthenticationMethodNotSupported { authentication_class: ObjectRef::from_obj(auth_class), method: auth_class.spec.provider.to_string(), diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 17f196c2..fa78578d 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -29,6 +29,7 @@ use stackable_operator::{ schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, time::Duration, + utils::cluster_info::KubernetesClusterInfo, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; @@ -482,11 +483,12 @@ impl ZookeeperCluster { } /// The fully-qualified domain name of the role-level load-balanced Kubernetes `Service` - pub fn server_role_service_fqdn(&self) -> Option { + pub fn server_role_service_fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Option { Some(format!( - "{}.{}.svc.cluster.local", - self.server_role_service_name()?, - self.metadata.namespace.as_ref()? + "{role_service_name}.{namespace}.svc.{cluster_domain}", + role_service_name = self.server_role_service_name()?, + namespace = self.metadata.namespace.as_ref()?, + cluster_domain = cluster_info.cluster_domain )) } @@ -666,10 +668,13 @@ pub struct ZookeeperPodRef { } impl ZookeeperPodRef { - pub fn fqdn(&self) -> String { + pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> String { format!( - "{}.{}.{}.svc.cluster.local", - self.pod_name, self.role_group_service_name, self.namespace + "{pod_name}.{service_name}.{namespace}.svc.{cluster_domain}", + pod_name = self.pod_name, + service_name = self.role_group_service_name, + namespace = self.namespace, + cluster_domain = cluster_info.cluster_domain ) } } diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs index 21d121e0..8dede828 100644 --- a/rust/crd/src/security.rs +++ b/rust/crd/src/security.rs @@ -8,13 +8,16 @@ use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::pod::{ - container::ContainerBuilder, - volume::{ - SecretFormat, SecretOperatorVolumeSourceBuilder, - SecretOperatorVolumeSourceBuilderError, VolumeBuilder, + builder::{ + self, + pod::{ + container::ContainerBuilder, + volume::{ + SecretFormat, SecretOperatorVolumeSourceBuilder, + SecretOperatorVolumeSourceBuilderError, VolumeBuilder, + }, + PodBuilder, }, - PodBuilder, }, client::Client, commons::authentication::AuthenticationClassProvider, @@ -35,6 +38,14 @@ pub enum Error { source: SecretOperatorVolumeSourceBuilderError, volume_name: String, }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { source: builder::pod::Error }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: builder::pod::container::Error, + }, } /// Helper struct combining TLS settings for server and quorum with the resolved AuthenticationClasses @@ -141,17 +152,25 @@ impl ZookeeperSecurity { if let Some(secret_class) = tls_secret_class { let tls_volume_name = "server-tls"; - cb_zookeeper.add_volume_mount(tls_volume_name, Self::SERVER_TLS_DIR); - pod_builder.add_volume(Self::create_tls_volume(tls_volume_name, secret_class)?); + cb_zookeeper + .add_volume_mount(tls_volume_name, Self::SERVER_TLS_DIR) + .context(AddVolumeMountSnafu)?; + pod_builder + .add_volume(Self::create_tls_volume(tls_volume_name, secret_class)?) + .context(AddVolumeSnafu)?; } // quorum let tls_volume_name = "quorum-tls"; - cb_zookeeper.add_volume_mount(tls_volume_name, Self::QUORUM_TLS_DIR); - pod_builder.add_volume(Self::create_tls_volume( - tls_volume_name, - &self.quorum_secret_class, - )?); + cb_zookeeper + .add_volume_mount(tls_volume_name, Self::QUORUM_TLS_DIR) + .context(AddVolumeMountSnafu)?; + pod_builder + .add_volume(Self::create_tls_volume( + tls_volume_name, + &self.quorum_secret_class, + )?) + .context(AddVolumeSnafu)?; Ok(()) } @@ -264,7 +283,8 @@ impl ZookeeperSecurity { AuthenticationClassProvider::Tls(tls) => tls.client_cert_secret_class.as_ref(), AuthenticationClassProvider::Ldap(_) | AuthenticationClassProvider::Oidc(_) - | AuthenticationClassProvider::Static(_) => None, + | AuthenticationClassProvider::Static(_) + | AuthenticationClassProvider::Kerberos(_) => None, }) .or(self.server_secret_class.as_ref()) } diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index c83a426f..14671b66 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -6,6 +6,7 @@ use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service}, kube::{runtime::reflector::ObjectRef, Resource, ResourceExt}, + utils::cluster_info::KubernetesClusterInfo, }; use stackable_zookeeper_crd::{security::ZookeeperSecurity, ZookeeperCluster, ZookeeperRole}; @@ -84,7 +85,7 @@ pub async fn build_discovery_configmaps( &namespace, controller_name, chroot, - pod_hosts(zk, zookeeper_security)?, + pod_hosts(zk, zookeeper_security, &client.kubernetes_cluster_info)?, resolved_product_image, )?]; if zk.spec.cluster_config.listener_class @@ -171,11 +172,12 @@ fn build_discovery_configmap( fn pod_hosts<'a>( zk: &'a ZookeeperCluster, zookeeper_security: &'a ZookeeperSecurity, + cluster_info: &'a KubernetesClusterInfo, ) -> Result + 'a> { Ok(zk .pods() .context(ExpectedPodsSnafu)? - .map(|pod_ref| (pod_ref.fqdn(), zookeeper_security.client_port()))) + .map(|pod_ref| (pod_ref.fqdn(cluster_info), zookeeper_security.client_port()))) } /// Lists all nodes currently hosting Pods participating in the [`Service`] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 83840b63..ccb2238d 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -54,6 +54,7 @@ async fn main() -> anyhow::Result<()> { product_config, watch_namespace, tracing_target, + cluster_info_opts, }) => { stackable_operator::logging::initialize_logging( "ZOOKEEPER_OPERATOR_LOG", @@ -72,8 +73,12 @@ async fn main() -> anyhow::Result<()> { "deploy/config-spec/properties.yaml", "/etc/stackable/zookeeper-operator/config-spec/properties.yaml", ])?; - let client = - stackable_operator::client::create_client(Some(OPERATOR_NAME.to_string())).await?; + let client = stackable_operator::client::initialize_operator( + Some(OPERATOR_NAME.to_string()), + &cluster_info_opts, + ) + .await?; + let zk_controller_builder = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index dce670fb..c09c7c80 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -17,6 +17,7 @@ use product_config::{ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ + self, configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{container::ContainerBuilder, resources::ResourceRequirementsBuilder, PodBuilder}, @@ -49,7 +50,9 @@ use stackable_operator::{ product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, product_logging::{ self, - framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, + framework::{ + create_vector_shutdown_file_command, remove_vector_shutdown_file_command, LoggingError, + }, spec::{ ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, CustomContainerLogConfig, @@ -61,7 +64,7 @@ use stackable_operator::{ statefulset::StatefulSetConditionBuilder, }, time::Duration, - utils::COMMON_BASH_TRAP_FUNCTIONS, + utils::{cluster_info::KubernetesClusterInfo, COMMON_BASH_TRAP_FUNCTIONS}, }; use stackable_zookeeper_crd::{ security::{self, ZookeeperSecurity}, @@ -255,6 +258,22 @@ pub enum Error { #[snafu(display("failed to add TLS volume mounts"))] AddTlsVolumeMounts { source: security::Error }, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { source: LoggingError }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { source: builder::pod::Error }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: builder::pod::container::Error, + }, + + #[snafu(display("failed to create cluster resources"))] + CreateClusterResources { + source: stackable_operator::cluster_resources::Error, + }, } impl ReconcilerError for Error { @@ -296,6 +315,10 @@ impl ReconcilerError for Error { Error::BuildLabel { .. } => None, Error::ObjectMeta { .. } => None, Error::AddTlsVolumeMounts { .. } => None, + Error::ConfigureLogging { .. } => None, + Error::AddVolume { .. } => None, + Error::AddVolumeMount { .. } => None, + Error::CreateClusterResources { .. } => None, } } } @@ -323,7 +346,7 @@ pub async fn reconcile_zk( &zk.object_ref(&()), ClusterResourceApplyStrategy::from(&zk.spec.cluster_operation), ) - .unwrap(); + .context(CreateClusterResourcesSnafu)?; let validated_config = validate_all_roles_and_groups_config( &resolved_product_image.app_version_label, @@ -411,6 +434,7 @@ pub async fn reconcile_zk( &resolved_product_image, vector_aggregator_address.as_deref(), &zookeeper_security, + &client.kubernetes_cluster_info, )?; let rg_statefulset = build_server_rolegroup_statefulset( zk, @@ -561,6 +585,7 @@ fn build_server_rolegroup_config_map( resolved_product_image: &ResolvedProductImage, vector_aggregator_address: Option<&str>, zookeeper_security: &ZookeeperSecurity, + cluster_info: &KubernetesClusterInfo, ) -> Result { let mut zoo_cfg: BTreeMap<_, _> = zk .pods() @@ -571,7 +596,7 @@ fn build_server_rolegroup_config_map( format!("server.{}", pod.zookeeper_myid), format!( "{}:2888:3888;{}", - pod.fqdn(), + pod.fqdn(cluster_info), zookeeper_security.client_port() ), ) @@ -816,9 +841,13 @@ fn build_server_rolegroup_statefulset( ..EnvVar::default() }]) .add_volume_mount("data", STACKABLE_DATA_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("log", STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? .resources( ResourceRequirementsBuilder::new() .with_cpu_request("200m") @@ -875,10 +904,15 @@ fn build_server_rolegroup_statefulset( .add_container_port("zk-election", 3888) .add_container_port("metrics", 9505) .add_volume_mount("data", STACKABLE_DATA_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) + .context(AddVolumeMountSnafu)? .add_volume_mount("log", STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? .resources(resources) .build(); @@ -907,6 +941,7 @@ fn build_server_rolegroup_statefulset( }), ..Volume::default() }) + .context(AddVolumeSnafu)? .add_volume(Volume { empty_dir: Some(EmptyDirVolumeSource { medium: None, @@ -915,12 +950,14 @@ fn build_server_rolegroup_statefulset( name: "rwconfig".to_string(), ..Volume::default() }) + .context(AddVolumeSnafu)? .add_empty_dir_volume( "log", Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_ZK_LOG_FILES_SIZE, MAX_PREPARE_LOG_FILE_SIZE], )), ) + .context(AddVolumeSnafu)? .security_context(PodSecurityContext { run_as_user: Some(ZK_UID), run_as_group: Some(0), @@ -936,38 +973,45 @@ fn build_server_rolegroup_statefulset( })), }) = logging.containers.get(&Container::Zookeeper) { - pod_builder.add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: config_map.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }); + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: config_map.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; } else { - pod_builder.add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: rolegroup_ref.object_name(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }); + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: rolegroup_ref.object_name(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; } if logging.enable_vector_agent { - pod_builder.add_container(product_logging::framework::vector_container( - resolved_product_image, - "config", - "log", - logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - )); + pod_builder.add_container( + product_logging::framework::vector_container( + resolved_product_image, + "config", + "log", + logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + ) + .context(ConfigureLoggingSnafu)?, + ); } add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?; @@ -1033,6 +1077,8 @@ pub fn error_policy( #[cfg(test)] mod tests { + use stackable_operator::commons::networking::DomainName; + use super::*; #[test] @@ -1108,7 +1154,9 @@ mod tests { let mut zookeeper: ZookeeperCluster = serde_yaml::from_str(zookeeper_yaml).expect("illegal test input"); zookeeper.metadata.uid = Some("42".to_owned()); - + let cluster_info = KubernetesClusterInfo { + cluster_domain: DomainName::try_from("cluster.local").unwrap(), + }; let resolved_product_image = zookeeper .spec .image @@ -1161,6 +1209,7 @@ mod tests { &resolved_product_image, None, &zookeeper_security, + &cluster_info, ) .unwrap() } diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index e6424fd6..d5542731 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -17,6 +17,7 @@ use stackable_operator::{ }, logging::controller::ReconcilerError, time::Duration, + utils::cluster_info::KubernetesClusterInfo, }; use stackable_zookeeper_crd::{ security::ZookeeperSecurity, ZookeeperCluster, ZookeeperZnode, ZookeeperZnodeStatus, @@ -273,12 +274,15 @@ async fn reconcile_apply( ) .unwrap(); - znode_mgmt::ensure_znode_exists(&zk_mgmt_addr(&zk, &zookeeper_security)?, znode_path) - .await - .with_context(|_| EnsureZnodeSnafu { - zk: ObjectRef::from_obj(&zk), - znode_path, - })?; + znode_mgmt::ensure_znode_exists( + &zk_mgmt_addr(&zk, &zookeeper_security, &client.kubernetes_cluster_info)?, + znode_path, + ) + .await + .with_context(|_| EnsureZnodeSnafu { + zk: ObjectRef::from_obj(&zk), + znode_path, + })?; let server_role_service = client .get::( @@ -340,22 +344,29 @@ async fn reconcile_cleanup( .context(FailedToInitializeSecurityContextSnafu)?; // Clean up znode from the ZooKeeper cluster before letting Kubernetes delete the object - znode_mgmt::ensure_znode_missing(&zk_mgmt_addr(&zk, &zookeeper_security)?, znode_path) - .await - .with_context(|_| EnsureZnodeMissingSnafu { - zk: ObjectRef::from_obj(&zk), - znode_path, - })?; + znode_mgmt::ensure_znode_missing( + &zk_mgmt_addr(&zk, &zookeeper_security, &client.kubernetes_cluster_info)?, + znode_path, + ) + .await + .with_context(|_| EnsureZnodeMissingSnafu { + zk: ObjectRef::from_obj(&zk), + znode_path, + })?; // No need to delete the ConfigMap, since that has an OwnerReference on the ZookeeperZnode object Ok(controller::Action::await_change()) } -fn zk_mgmt_addr(zk: &ZookeeperCluster, zookeeper_security: &ZookeeperSecurity) -> Result { +fn zk_mgmt_addr( + zk: &ZookeeperCluster, + zookeeper_security: &ZookeeperSecurity, + cluster_info: &KubernetesClusterInfo, +) -> Result { // Rust ZooKeeper client does not support client-side load-balancing, so use // (load-balanced) global service instead. Ok(format!( "{}:{}", - zk.server_role_service_fqdn() + zk.server_role_service_fqdn(cluster_info) .with_context(|| NoZkFqdnSnafu { zk: ObjectRef::from_obj(zk), })?,