diff --git a/CHANGELOG.md b/CHANGELOG.md index b362cf1b..65a979d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,11 @@ - Default stackableVersion to operator version ([#385]). - Configuration overrides for the JVM security properties, such as DNS caching ([#389]). - Support PodDisruptionBudgets ([#399]). +- Support graceful shutdown ([#402]). ### Changed - `vector` `0.26.0` -> `0.31.0` ([#382]). -- `operator-rs` `0.44.0` -> `0.52.1` ([#385], [#399]). ### Fixed @@ -22,6 +22,7 @@ [#389]: https://github.com/stackabletech/hbase-operator/pull/389 [#394]: https://github.com/stackabletech/hbase-operator/pull/394 [#399]: https://github.com/stackabletech/hbase-operator/pull/399 +[#402]: https://github.com/stackabletech/hbase-operator/pull/402 ## [23.7.0] - 2023-07-14 diff --git a/Cargo.lock b/Cargo.lock index e7cb4607..e0d636a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,9 +31,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -61,9 +61,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -75,15 +75,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys", @@ -121,7 +121,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -223,9 +223,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle", @@ -305,7 +305,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -439,7 +439,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -461,7 +461,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -628,7 +628,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -722,9 +722,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" dependencies = [ "ahash", "allocator-api2", @@ -893,12 +893,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.1", ] [[package]] @@ -935,9 +935,9 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -953,9 +953,9 @@ dependencies = [ [[package]] name = "json-patch" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7765dccf8c39c3a470fc694efe322969d791e713ca46bc7b5c506886157572" +checksum = "55ff1e1486799e3f64129f8ccad108b38290df9cd7015cd31bed17239f0789d6" dependencies = [ "serde", "serde_json", @@ -1080,7 +1080,7 @@ dependencies = [ "backoff", "derivative", "futures 0.3.28", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "json-patch", "k8s-openapi", "kube-client", @@ -1103,9 +1103,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libgit2-sys" @@ -1158,9 +1158,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "mime" @@ -1200,9 +1200,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -1301,7 +1301,7 @@ dependencies = [ "futures-util", "once_cell", "opentelemetry_api", - "ordered-float 3.9.1", + "ordered-float 3.9.2", "percent-encoding", "rand", "regex", @@ -1312,18 +1312,18 @@ dependencies = [ [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] [[package]] name = "ordered-float" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" dependencies = [ "num-traits", ] @@ -1404,7 +1404,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1435,7 +1435,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1464,9 +1464,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -1537,14 +1537,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.8", - "regex-syntax 0.7.5", + "regex-automata 0.4.1", + "regex-syntax 0.8.0", ] [[package]] @@ -1558,13 +1558,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.0", ] [[package]] @@ -1575,9 +1575,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" [[package]] name = "relative-path" @@ -1625,7 +1625,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.37", + "syn 2.0.38", "unicode-ident", ] @@ -1783,9 +1783,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" dependencies = [ "serde", ] @@ -1805,7 +1805,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float 2.10.0", + "ordered-float 2.10.1", "serde", ] @@ -1817,7 +1817,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1837,7 +1837,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -1858,7 +1858,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -1878,9 +1878,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b21f559e07218024e7e9f90f96f601825397de0e25420135f7f952453fed0b" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -2054,7 +2054,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2082,7 +2082,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2098,9 +2098,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -2124,7 +2124,7 @@ checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2155,7 +2155,7 @@ dependencies = [ "byteorder", "integer-encoding", "log", - "ordered-float 2.10.0", + "ordered-float 2.10.1", "threadpool", ] @@ -2176,9 +2176,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -2211,7 +2211,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2277,7 +2277,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "serde", "serde_spanned", "toml_datetime", @@ -2355,7 +2355,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2556,7 +2556,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -2578,7 +2578,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2698,9 +2698,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" dependencies = [ "memchr", ] diff --git a/deploy/helm/hbase-operator/crds/crds.yaml b/deploy/helm/hbase-operator/crds/crds.yaml index 032082a3..56746624 100644 --- a/deploy/helm/hbase-operator/crds/crds.yaml +++ b/deploy/helm/hbase-operator/crds/crds.yaml @@ -596,6 +596,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string hbaseOpts: nullable: true type: string @@ -606,6 +610,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -613,13 +618,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -635,14 +641,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -656,9 +664,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -671,10 +680,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -4032,6 +4044,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string hbaseOpts: nullable: true type: string @@ -4042,6 +4058,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -4049,13 +4066,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4071,14 +4089,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4092,9 +4112,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -4107,10 +4128,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -7485,6 +7509,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string hbaseOpts: nullable: true type: string @@ -7495,6 +7523,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -7502,13 +7531,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7524,14 +7554,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7545,9 +7577,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -7560,10 +7593,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -10921,6 +10957,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string hbaseOpts: nullable: true type: string @@ -10931,6 +10971,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -10938,13 +10979,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -10960,14 +11002,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -10981,9 +11025,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -10996,10 +11041,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -14374,6 +14422,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string hbaseOpts: nullable: true type: string @@ -14384,6 +14436,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -14391,13 +14444,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -14413,14 +14467,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -14434,9 +14490,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -14449,10 +14506,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object @@ -17810,6 +17870,10 @@ spec: type: array type: object type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string hbaseOpts: nullable: true type: string @@ -17820,6 +17884,7 @@ spec: default: enableVectorAgent: null containers: {} + description: Logging configuration properties: containers: additionalProperties: @@ -17827,13 +17892,14 @@ spec: - required: - custom - {} - description: Fragment derived from `ContainerLogConfigChoice` + description: Log configuration of the container properties: console: + description: Configuration for the console appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -17849,14 +17915,16 @@ spec: description: Custom log configuration provided in a ConfigMap properties: configMap: + description: ConfigMap containing the log configuration files nullable: true type: string type: object file: + description: Configuration for the file appender nullable: true properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -17870,9 +17938,10 @@ spec: type: object loggers: additionalProperties: + description: Configuration of a logger properties: level: - description: Log levels + description: The log level threshold. Log events with a lower log level are discarded. enum: - TRACE - DEBUG @@ -17885,10 +17954,13 @@ spec: type: string type: object default: {} + description: Configuration per logger type: object type: object + description: Log configuration per container type: object enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent nullable: true type: boolean type: object diff --git a/docs/modules/hbase/pages/usage-guide/operations/graceful-shutdown.adoc b/docs/modules/hbase/pages/usage-guide/operations/graceful-shutdown.adoc index 37ee4a40..afa5e8fd 100644 --- a/docs/modules/hbase/pages/usage-guide/operations/graceful-shutdown.adoc +++ b/docs/modules/hbase/pages/usage-guide/operations/graceful-shutdown.adoc @@ -1,7 +1,41 @@ = Graceful shutdown -Graceful shutdown of HBase nodes is either not supported by the product itself -or we have not implemented it yet. +You can configure the graceful shutdown as described in xref:concepts:operations/graceful_shutdown.adoc[]. -Outstanding implementation work for the graceful shutdowns of all products where this functionality is relevant is tracked in -https://github.com/stackabletech/issues/issues/357 +== Masters + +As a default, masters have `20 minutes` to shut down gracefully. + +The HBase master process will always run as PID `1` and will receive a `SIGTERM` signal when Kubernetes wants to terminate the Pod. +After the graceful shutdown timeout runs out, and the process still didn't exit, Kubernetes will issue a `SIGKILL` signal. + +This is equivalent to executing the `bin/hbase-daemon.sh stop master` command, which internally executes `kill ` (https://github.com/apache/hbase/blob/8382f55b15be6ae190f8d202a5e6a40af177ec76/bin/hbase-daemon.sh#L338[code]), waits for a configurable period of time (defaults to 20 minutes), and finally executes `kill -9 ` to `SIGKILL` the master (https://github.com/apache/hbase/blob/8382f55b15be6ae190f8d202a5e6a40af177ec76/bin/hbase-common.sh#L20-L41[code]). + +However, there is no acknowledge message in the log indicating a graceful shutdown. + +== RegionServers + +As a default, RegionServers have `60 minutes` to shut down gracefully. + +They use the same mechanism described above. +In contrast to the Master servers, they will, however, acknowledge the graceful shutdown with a message in the logs: + +[source,text] +---- +2023-10-11 12:38:05,059 INFO [shutdown-hook-0] regionserver.ShutdownHook: Shutdown hook starting; hbase.shutdown.hook=true; fsShutdownHook=org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer@5875de6a +2023-10-11 12:38:05,060 INFO [shutdown-hook-0] regionserver.HRegionServer: ***** STOPPING region server 'test-hbase-regionserver-default-0.test-hbase-regionserver-default.kuttl-test-topical-parakeet.svc.cluster.local,16020,1697027870348' ***** +---- + +== RestServers + +As a default, RestServers have `5 minutes` to shut down gracefully. + +They use the same mechanism described above. +In contrast to the Master servers, they will, however, acknowledge the graceful shutdown with a message in the logs: + +[source,text] +---- +2023-10-11 12:40:42,309 INFO [JettyShutdownThread] server.AbstractConnector: Stopped ServerConnector@62dae540{HTTP/1.1, (http/1.1)}{0.0.0.0:8080} +2023-10-11 12:40:42,309 INFO [JettyShutdownThread] server.session: node0 Stopped scavenging +2023-10-11 12:40:42,316 INFO [main] RESTServer: ***** STOPPING service 'RESTServer' ***** +---- diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 1dc67881..43c387c1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -20,6 +20,7 @@ use stackable_operator::{ role_utils::{GenericRoleConfig, Role, RoleGroup, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, + time::Duration, }; use strum::{Display, EnumIter, EnumString}; @@ -58,6 +59,11 @@ pub const METRICS_PORT: i32 = 8081; pub const JVM_HEAP_FACTOR: f32 = 0.8; +const DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(20); +const DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = + Duration::from_minutes_unchecked(60); +const DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("the role [{role}] is invalid and does not exist in HBase"))] @@ -236,12 +242,19 @@ impl HbaseRole { }, }; + let graceful_shutdown_timeout = match &self { + HbaseRole::Master => DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, + HbaseRole::RegionServer => DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, + HbaseRole::RestServer => DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, + }; + HbaseConfigFragment { hbase_rootdir: None, hbase_opts: None, resources, logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, self, hdfs_discovery_cm_name), + graceful_shutdown_timeout: Some(graceful_shutdown_timeout), } } } @@ -308,6 +321,10 @@ pub struct HbaseConfig { pub logging: Logging, #[fragment_attrs(serde(default))] pub affinity: StackableAffinity, + + /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + #[fragment_attrs(serde(default))] + pub graceful_shutdown_timeout: Option, } impl Configuration for HbaseConfigFragment { diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 3ba69906..96b08a92 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -27,16 +27,16 @@ use stackable_operator::{ product_image_selection::ResolvedProductImage, rbac::{build_rbac_resources, service_account_name}, }, - k8s_openapi::{api::core::v1::Volume, DeepMerge}, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ ConfigMap, ConfigMapVolumeSource, ContainerPort, HTTPGetAction, Probe, Service, - ServicePort, ServiceSpec, TCPSocketAction, + ServicePort, ServiceSpec, TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + DeepMerge, }, kube::{runtime::controller::Action, Resource}, labels::{role_group_selector_labels, role_selector_labels, ObjectLabels}, @@ -45,6 +45,7 @@ use stackable_operator::{ product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, product_logging::{ self, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, spec::{ ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, CustomContainerLogConfig, @@ -56,12 +57,13 @@ use stackable_operator::{ statefulset::StatefulSetConditionBuilder, }, time::Duration, + utils::COMMON_BASH_TRAP_FUNCTIONS, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ discovery::build_discovery_configmap, - operations::pdb::add_pdbs, + operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{ extend_role_group_config_map, resolve_vector_aggregator_address, LOG4J_CONFIG_FILE, }, @@ -219,6 +221,11 @@ pub enum Error { FailedToCreatePdb { source: crate::operations::pdb::Error, }, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, } type Result = std::result::Result; @@ -310,7 +317,7 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< for (rolegroup_name, rolegroup_config) in group_config.iter() { let rolegroup = hbase.server_rolegroup_ref(role_name, rolegroup_name); - let config = hbase + let merged_config = hbase .merged_config( &hbase_role, &rolegroup.role_group, @@ -325,7 +332,7 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< &rolegroup, rolegroup_config, &zookeeper_connection_information, - &config, + &merged_config, &resolved_product_image, vector_aggregator_address.as_deref(), )?; @@ -333,7 +340,7 @@ pub async fn reconcile_hbase(hbase: Arc, ctx: Arc) -> Result< &hbase, &hbase_role, &rolegroup, - &config, + &merged_config, &resolved_product_image, )?; cluster_resources @@ -674,28 +681,33 @@ fn build_rolegroup_statefulset( "pipefail".to_string(), "-c".to_string(), ]) - .args(vec![[ - format!("mkdir -p {}", CONFIG_DIR_NAME), - format!( - "cp {}/hdfs-site.xml {}", - HDFS_DISCOVERY_TMP_DIR, CONFIG_DIR_NAME - ), - format!( - "cp {}/core-site.xml {}", - HDFS_DISCOVERY_TMP_DIR, CONFIG_DIR_NAME - ), - format!("cp {}/* {}", HBASE_CONFIG_TMP_DIR, CONFIG_DIR_NAME), - format!("cp {HBASE_LOG_CONFIG_TMP_DIR}/{LOG4J_CONFIG_FILE} {CONFIG_DIR_NAME}",), - format!( - "bin/hbase {} start", - match hbase_role { - HbaseRole::Master => "master", - HbaseRole::RegionServer => "regionserver", - HbaseRole::RestServer => "rest", - } - ), - ] - .join(" && ")]) + .args(vec![format!( + "\ +mkdir -p {CONFIG_DIR_NAME} +cp {HDFS_DISCOVERY_TMP_DIR}/hdfs-site.xml {CONFIG_DIR_NAME} +cp {HDFS_DISCOVERY_TMP_DIR}/core-site.xml {CONFIG_DIR_NAME} +cp {HBASE_CONFIG_TMP_DIR}/* {CONFIG_DIR_NAME} +cp {HBASE_LOG_CONFIG_TMP_DIR}/{LOG4J_CONFIG_FILE} {CONFIG_DIR_NAME} + +{COMMON_BASH_TRAP_FUNCTIONS} +{remove_vector_shutdown_file_command} +prepare_signal_handlers +bin/hbase {hbase_role_name_in_command} start & +wait_for_termination $! +{create_vector_shutdown_file_command} +", + hbase_role_name_in_command = match hbase_role { + HbaseRole::Master => "master", + HbaseRole::RegionServer => "regionserver", + // Of course it is not called "restserver", so we need to have this match + // instead of just letting the Display impl do it's thing ;P + HbaseRole::RestServer => "rest", + }, + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + )]) .add_env_var("HBASE_CONF_DIR", CONFIG_DIR_NAME) // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 .add_env_var("HADOOP_CONF_DIR", CONFIG_DIR_NAME) @@ -795,6 +807,8 @@ fn build_rolegroup_statefulset( )); } + add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?; + let mut pod_template = pod_builder.build_template(); if let Some(role) = role { pod_template.merge_from(role.config.pod_overrides.clone()); diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/operations/graceful_shutdown.rs new file mode 100644 index 00000000..e0442669 --- /dev/null +++ b/rust/operator-binary/src/operations/graceful_shutdown.rs @@ -0,0 +1,26 @@ +use snafu::{ResultExt, Snafu}; +use stackable_hbase_crd::HbaseConfig; +use stackable_operator::builder::PodBuilder; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("Failed to set terminationGracePeriod"))] + SetTerminationGracePeriod { + source: stackable_operator::builder::pod::Error, + }, +} + +pub fn add_graceful_shutdown_config( + merged_config: &HbaseConfig, + pod_builder: &mut PodBuilder, +) -> Result<(), Error> { + // This must be always set by the merge mechanism, as we provide a default value, + // users can not disable graceful shutdown. + if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { + pod_builder + .termination_grace_period(&graceful_shutdown_timeout) + .context(SetTerminationGracePeriodSnafu)?; + } + + Ok(()) +} diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs index d3cf6e9c..92ca2ec7 100644 --- a/rust/operator-binary/src/operations/mod.rs +++ b/rust/operator-binary/src/operations/mod.rs @@ -1 +1,2 @@ +pub mod graceful_shutdown; pub mod pdb; diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml index c3b9f698..cae4a019 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -9,33 +9,45 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: test-hbase-master-default +spec: + template: + spec: + terminationGracePeriodSeconds: 1200 status: - readyReplicas: 1 - replicas: 1 + readyReplicas: 2 + replicas: 2 --- apiVersion: apps/v1 kind: StatefulSet metadata: name: test-hbase-regionserver-default +spec: + template: + spec: + terminationGracePeriodSeconds: 3600 status: - readyReplicas: 1 - replicas: 1 + readyReplicas: 2 + replicas: 2 --- apiVersion: apps/v1 kind: StatefulSet metadata: name: test-hbase-restserver-default +spec: + template: + spec: + terminationGracePeriodSeconds: 300 status: - readyReplicas: 1 - replicas: 1 + readyReplicas: 2 + replicas: 2 --- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: test-hbase-master status: - expectedPods: 1 - currentHealthy: 1 + expectedPods: 2 + currentHealthy: 2 disruptionsAllowed: 1 --- apiVersion: policy/v1 @@ -43,8 +55,8 @@ kind: PodDisruptionBudget metadata: name: test-hbase-regionserver status: - expectedPods: 1 - currentHealthy: 1 + expectedPods: 2 + currentHealthy: 2 disruptionsAllowed: 1 --- apiVersion: policy/v1 @@ -52,6 +64,6 @@ kind: PodDisruptionBudget metadata: name: test-hbase-restserver status: - expectedPods: 1 - currentHealthy: 1 + expectedPods: 2 + currentHealthy: 2 disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 index 214c3fdc..ec30caef 100644 --- a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 @@ -23,7 +23,7 @@ spec: hbase-site.xml: phoenix.log.saltBuckets: "2" hbase.regionserver.wal.codec: "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec" - replicas: 1 + replicas: 2 regionServers: config: logging: @@ -34,7 +34,7 @@ spec: hbase-site.xml: phoenix.log.saltBuckets: "2" hbase.regionserver.wal.codec: "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec" - replicas: 1 + replicas: 2 restServers: config: logging: @@ -45,4 +45,4 @@ spec: hbase-site.xml: phoenix.log.saltBuckets: "2" hbase.regionserver.wal.codec: "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec" - replicas: 1 + replicas: 2 diff --git a/tests/templates/kuttl/smoke/70-unleash-the-chaosmonkey.yaml.j2.DISABLED b/tests/templates/kuttl/smoke/70-unleash-the-chaosmonkey.yaml.j2.DISABLED new file mode 100644 index 00000000..92df0300 --- /dev/null +++ b/tests/templates/kuttl/smoke/70-unleash-the-chaosmonkey.yaml.j2.DISABLED @@ -0,0 +1,72 @@ +# WARNING +# This test is disabled as everything is broken +# See https://github.com/stackabletech/hbase-operator/issues/404 for details +# WARNING + +# Tribute to https://github.com/Netflix/chaosmonkey + +# We need to reduce the number of monkeys, otherwise the tests literally take days +# We only run them on some hand-picked test cases +{% if test_scenario['values']['listener-class'] == 'cluster-internal' and test_scenario['values']['hdfs'] == test_scenario['values']['hdfs-latest'] and test_scenario['values']['zookeeper'] == test_scenario['values']['zookeeper-latest'] %} + +# We need to force-delete the Pods, because IONOS is sometimes unable to delete the pod (it's stuck in Terminating for > 20 minutes) +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 3600 +commands: + # First, let's delete the first pod of every HBase service + # Should trigger failover of the namenode to 1 + - script: kubectl -n $NAMESPACE delete pod --force test-hbase-master-default-0 test-hbase-regionserver-default-0 test-hbase-restserver-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hbase test-hbase --timeout 10m + timeout: 600 + + # Also delete the last pod of every HBase service + # Should trigger failover of the namenode back to 0 + - script: kubectl -n $NAMESPACE delete pod --force test-hbase-master-default-1 test-hbase-regionserver-default-1 test-hbase-restserver-default-1 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hbase test-hbase --timeout 10m + timeout: 600 + + # Also delete the Zookeeper + - script: kubectl -n $NAMESPACE delete pod --force test-zk-server-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available zookeepercluster test-zk --timeout 10m + timeout: 600 + + # Also delete some HDFS Pods + - script: kubectl -n $NAMESPACE delete pod --force test-hdfs-namenode-default-0 test-hdfs-datanode-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs test-hdfs --timeout 10m + timeout: 600 + + # And now everything +{% for n in range(3) %} + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hbase + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hdfs + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=zookeeper + timeout: 600 + - script: sleep 10 + # Delete just after they have started up again, just to make things worse + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hbase + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hdfs + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=zookeeper + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available zookeepercluster test-zk --timeout 10m + timeout: 600 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs test-hdfs --timeout 10m + timeout: 600 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hbase test-hbase --timeout 10m + timeout: 600 +{% endfor %} +{% endif %}