From d507f10b9d8ec1820101f0f82a7a4581c3faccfe Mon Sep 17 00:00:00 2001 From: Adam Hendel Date: Wed, 29 May 2024 07:35:01 -0500 Subject: [PATCH] add podmonitor to coredb spec (#809) --- charts/tembo-operator/Chart.yaml | 2 +- charts/tembo-operator/templates/crd.yaml | 16 + tembo-operator/Cargo.lock | 2 +- tembo-operator/Cargo.toml | 2 +- tembo-operator/src/app_service/manager.rs | 106 +- tembo-operator/src/app_service/types.rs | 12 + tembo-operator/src/lib.rs | 1 + tembo-operator/src/prometheus/mod.rs | 1 + .../src/prometheus/podmonitor_crd.rs | 962 ++++++++++++++++++ tembo-operator/tests/integration_tests.rs | 180 ++++ tembo-operator/yaml/sample-vectordb.yaml | 15 +- 11 files changed, 1286 insertions(+), 13 deletions(-) create mode 100644 tembo-operator/src/prometheus/mod.rs create mode 100644 tembo-operator/src/prometheus/podmonitor_crd.rs diff --git a/charts/tembo-operator/Chart.yaml b/charts/tembo-operator/Chart.yaml index 8f9bef23f..5806aea06 100644 --- a/charts/tembo-operator/Chart.yaml +++ b/charts/tembo-operator/Chart.yaml @@ -3,7 +3,7 @@ name: tembo-operator description: "Helm chart to deploy the tembo-operator" type: application icon: https://cloud.tembo.io/images/TemboElephant.png -version: 0.5.3 +version: 0.6.0 home: https://tembo.io sources: - https://github.com/tembo-io/tembo diff --git a/charts/tembo-operator/templates/crd.yaml b/charts/tembo-operator/templates/crd.yaml index 2cd4a29bf..9ba6ad65d 100644 --- a/charts/tembo-operator/templates/crd.yaml +++ b/charts/tembo-operator/templates/crd.yaml @@ -653,6 +653,22 @@ spec: image: description: Defines the container image to use for the appService. type: string + metrics: + description: Defines the metrics endpoints to be scraped by Prometheus. This implements a subset of features available by PodMonitorPodMetricsEndpoints. + nullable: true + properties: + path: + description: path to scrape metrics + type: string + port: + description: port must be also exposed in one of AppService.routing[] + format: uint16 + minimum: 0.0 + type: integer + required: + - path + - port + type: object middlewares: description: Defines the ingress middeware configuration for the appService. This is specifically configured for the ingress controller Traefik. items: diff --git a/tembo-operator/Cargo.lock b/tembo-operator/Cargo.lock index 8f64e4640..334dd8337 100644 --- a/tembo-operator/Cargo.lock +++ b/tembo-operator/Cargo.lock @@ -494,7 +494,7 @@ dependencies = [ [[package]] name = "controller" -version = "0.47.3" +version = "0.48.0" dependencies = [ "actix-web", "anyhow", diff --git a/tembo-operator/Cargo.toml b/tembo-operator/Cargo.toml index f8417c4ec..e7c45a46f 100644 --- a/tembo-operator/Cargo.toml +++ b/tembo-operator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "controller" description = "Tembo Operator for Postgres" -version = "0.47.3" +version = "0.48.0" edition = "2021" default-run = "controller" license = "Apache-2.0" diff --git a/tembo-operator/src/app_service/manager.rs b/tembo-operator/src/app_service/manager.rs index 7253183d8..ade048f66 100644 --- a/tembo-operator/src/app_service/manager.rs +++ b/tembo-operator/src/app_service/manager.rs @@ -38,6 +38,8 @@ use super::{ use crate::{app_service::types::IngressType, secret::fetch_all_decoded_data_from_secret}; +const APP_CONTAINER_PORT_PREFIX: &str = "app-"; + // private wrapper to hold the AppService Resources #[derive(Clone, Debug)] struct AppServiceResources { @@ -48,6 +50,7 @@ struct AppServiceResources { ingress_tcp_routes: Option>, entry_points: Option>, entry_points_tcp: Option>, + podmonitor: Option, } // generates Kubernetes Deployment and Service templates for a AppService @@ -76,11 +79,13 @@ fn generate_resource( coredb_name, &resource_name, namespace, - oref, + oref.clone(), annotations, - placement, + placement.clone(), ); + let maybe_podmonitor = generate_podmonitor(appsvc, &resource_name, namespace, annotations); + // If DATA_PLANE_BASEDOMAIN is not set, don't generate IngressRoutes, IngressRouteTCPs, or EntryPoints if domain.is_none() { return AppServiceResources { @@ -91,6 +96,7 @@ fn generate_resource( ingress_tcp_routes: None, entry_points: None, entry_points_tcp: None, + podmonitor: maybe_podmonitor, }; } // It's safe to unwrap domain here because we've already checked if it's None @@ -158,6 +164,7 @@ fn generate_resource( ingress_tcp_routes, entry_points, entry_points_tcp, + podmonitor: maybe_podmonitor, } } @@ -194,7 +201,7 @@ fn generate_service( port: p as i32, // there can be more than one ServicePort per Service // these must be unique, so we'll use the port number - name: Some(format!("http-{}", p)), + name: Some(format!("{APP_CONTAINER_PORT_PREFIX}{p}")), target_port: None, ..ServicePort::default() }) @@ -279,6 +286,7 @@ fn generate_deployment( let container_ports: Vec = distinct_ports .into_iter() .map(|p| ContainerPort { + name: Some(format!("{APP_CONTAINER_PORT_PREFIX}{p}")), container_port: p as i32, protocol: Some("TCP".to_string()), ..ContainerPort::default() @@ -585,7 +593,6 @@ pub fn to_delete(desired: Vec, actual: Vec) -> Option, client: &Client, ns: &str) -> bool { let deployment_api: Api = Api::namespaced(client.clone(), ns); - let service_api: Api = Api::namespaced(client.clone(), ns); let ps = PatchParams::apply("cntrlr").force(); let mut has_errors: bool = false; @@ -612,6 +619,8 @@ async fn apply_resources(resources: Vec, client: &Client, n if res.service.is_none() { continue; } + + let service_api: Api = Api::namespaced(client.clone(), ns); match service_api .patch(&res.name, &ps, &Patch::Apply(&res.service)) .await @@ -629,6 +638,50 @@ async fn apply_resources(resources: Vec, client: &Client, n ); } } + + let podmon_api: Api = Api::namespaced(client.clone(), ns); + if let Some(mut pmon) = res.podmonitor { + // assign ownership of the PodMonitor to the Service + // if Service is deleted, so is the PodMonitor + let meta = service_api.get(&res.name).await; + if let Ok(svc) = meta { + let uid = svc.metadata.uid.unwrap_or_default(); + let oref = OwnerReference { + api_version: "v1".to_string(), + kind: "Service".to_string(), + name: res.name.clone(), + uid, + controller: Some(true), + block_owner_deletion: Some(true), + }; + pmon.metadata.owner_references = Some(vec![oref]); + } + match podmon_api + .patch(&res.name, &ps, &Patch::Apply(&pmon)) + .await + .map_err(Error::KubeError) + { + Ok(_) => { + debug!("ns: {}, applied PodMonitor: {}", ns, res.name); + } + Err(e) => { + has_errors = true; + error!( + "ns: {}, failed to apply PodMonitor for AppService: {}, error: {}", + ns, res.name, e + ); + } + } + } else { + match podmon_api.delete(&res.name, &Default::default()).await.ok() { + Some(_) => { + debug!("ns: {}, deleted PodMonitor: {}", ns, res.name); + } + None => { + debug!("ns: {}, PodMonitor does not exist: {}", ns, res.name); + } + } + } } has_errors } @@ -947,6 +1000,51 @@ pub async fn prepare_apps_connection_secret(client: Client, cdb: &CoreDB) -> Res Ok(()) } +use crate::prometheus::podmonitor_crd as podmon; + +fn generate_podmonitor( + appsvc: &AppService, + resource_name: &str, + namespace: &str, + annotations: &BTreeMap, +) -> Option { + let metrics = appsvc.metrics.clone()?; + + let mut selector_labels: BTreeMap = BTreeMap::new(); + selector_labels.insert("app".to_owned(), resource_name.to_string()); + + let mut labels = selector_labels.clone(); + labels.insert("component".to_owned(), COMPONENT_NAME.to_owned()); + labels.insert("coredb.io/name".to_owned(), namespace.to_owned()); + + let podmon_metadata = ObjectMeta { + name: Some(resource_name.to_string()), + namespace: Some(namespace.to_owned()), + labels: Some(labels.clone()), + annotations: Some(annotations.clone()), + ..ObjectMeta::default() + }; + + let metrics_endpoint = podmon::PodMonitorPodMetricsEndpoints { + path: Some(metrics.path), + port: Some(format!("{APP_CONTAINER_PORT_PREFIX}{}", metrics.port)), + ..podmon::PodMonitorPodMetricsEndpoints::default() + }; + + let pmonspec = podmon::PodMonitorSpec { + pod_metrics_endpoints: Some(vec![metrics_endpoint]), + selector: podmon::PodMonitorSelector { + match_labels: Some(selector_labels.clone()), + ..podmon::PodMonitorSelector::default() + }, + ..podmon::PodMonitorSpec::default() + }; + Some(podmon::PodMonitor { + metadata: podmon_metadata, + spec: pmonspec, + }) +} + #[cfg(test)] mod tests { use crate::{apis::coredb_types::CoreDB, app_service::manager::generate_appsvc_annotations}; diff --git a/tembo-operator/src/app_service/types.rs b/tembo-operator/src/app_service/types.rs index 92b16c156..3773036f4 100644 --- a/tembo-operator/src/app_service/types.rs +++ b/tembo-operator/src/app_service/types.rs @@ -120,6 +120,10 @@ pub struct AppService { /// See the [Kubernetes docs](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/). pub probes: Option, + /// Defines the metrics endpoints to be scraped by Prometheus. + /// This implements a subset of features available by PodMonitorPodMetricsEndpoints. + pub metrics: Option, + /// Defines the ingress middeware configuration for the appService. /// This is specifically configured for the ingress controller Traefik. pub middlewares: Option>, @@ -146,6 +150,14 @@ pub fn default_resources() -> ResourceRequirements { } } +#[derive(Clone, Debug, Serialize, Deserialize, ToSchema, JsonSchema, PartialEq)] +pub struct AppMetrics { + /// port must be also exposed in one of AppService.routing[] + pub port: u16, + /// path to scrape metrics + pub path: String, +} + // Secrets are injected into the container as environment variables // ths allows users to map these secrets to environment variable of their choice #[derive(Clone, Debug, Serialize, Deserialize, ToSchema, JsonSchema, PartialEq)] diff --git a/tembo-operator/src/lib.rs b/tembo-operator/src/lib.rs index 021ebbb3a..2355cc543 100644 --- a/tembo-operator/src/lib.rs +++ b/tembo-operator/src/lib.rs @@ -17,6 +17,7 @@ pub use metrics::Metrics; mod config; pub mod defaults; pub mod errors; +pub mod prometheus; pub mod cloudnativepg; mod deployment_postgres_exporter; diff --git a/tembo-operator/src/prometheus/mod.rs b/tembo-operator/src/prometheus/mod.rs new file mode 100644 index 000000000..970993642 --- /dev/null +++ b/tembo-operator/src/prometheus/mod.rs @@ -0,0 +1 @@ +pub mod podmonitor_crd; diff --git a/tembo-operator/src/prometheus/podmonitor_crd.rs b/tembo-operator/src/prometheus/podmonitor_crd.rs new file mode 100644 index 000000000..0bafacb2a --- /dev/null +++ b/tembo-operator/src/prometheus/podmonitor_crd.rs @@ -0,0 +1,962 @@ +// WARNING: generated by kopium - manual changes will be overwritten +// kopium command: kopium -D Default podmonitors.monitoring.coreos.com -A +// kopium version: 0.20.0 + +#[allow(unused_imports)] +mod prelude { + pub use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; + pub use kube::CustomResource; + pub use schemars::JsonSchema; + pub use serde::{Deserialize, Serialize}; + pub use std::collections::BTreeMap; +} +use self::prelude::*; + +/// Specification of desired Pod selection for target discovery by Prometheus. +#[derive(CustomResource, Serialize, Default, Deserialize, Clone, Debug, JsonSchema)] +#[kube( + group = "monitoring.coreos.com", + version = "v1", + kind = "PodMonitor", + plural = "podmonitors" +)] +#[kube(namespaced)] +#[kube(derive = "Default")] +pub struct PodMonitorSpec { + /// `attachMetadata` defines additional metadata which is added to the + /// discovered targets. + /// + /// + /// It requires Prometheus >= v2.37.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "attachMetadata" + )] + pub attach_metadata: Option, + /// When defined, bodySizeLimit specifies a job level limit on the size + /// of uncompressed response body that will be accepted by Prometheus. + /// + /// + /// It requires Prometheus >= v2.28.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "bodySizeLimit" + )] + pub body_size_limit: Option, + /// The label to use to retrieve the job name from. + /// `jobLabel` selects the label from the associated Kubernetes `Pod` + /// object which will be used as the `job` label for all metrics. + /// + /// + /// For example if `jobLabel` is set to `foo` and the Kubernetes `Pod` + /// object is labeled with `foo: bar`, then Prometheus adds the `job="bar"` + /// label to all ingested metrics. + /// + /// + /// If the value of this field is empty, the `job` label of the metrics + /// defaults to the namespace and name of the PodMonitor object (e.g. `/`). + #[serde(default, skip_serializing_if = "Option::is_none", rename = "jobLabel")] + pub job_label: Option, + /// Per-scrape limit on the number of targets dropped by relabeling + /// that will be kept in memory. 0 means no limit. + /// + /// + /// It requires Prometheus >= v2.47.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "keepDroppedTargets" + )] + pub keep_dropped_targets: Option, + /// Per-scrape limit on number of labels that will be accepted for a sample. + /// + /// + /// It requires Prometheus >= v2.27.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "labelLimit" + )] + pub label_limit: Option, + /// Per-scrape limit on length of labels name that will be accepted for a sample. + /// + /// + /// It requires Prometheus >= v2.27.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "labelNameLengthLimit" + )] + pub label_name_length_limit: Option, + /// Per-scrape limit on length of labels value that will be accepted for a sample. + /// + /// + /// It requires Prometheus >= v2.27.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "labelValueLengthLimit" + )] + pub label_value_length_limit: Option, + /// Selector to select which namespaces the Kubernetes `Pods` objects + /// are discovered from. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "namespaceSelector" + )] + pub namespace_selector: Option, + /// List of endpoints part of this PodMonitor. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "podMetricsEndpoints" + )] + pub pod_metrics_endpoints: Option>, + /// `podTargetLabels` defines the labels which are transferred from the + /// associated Kubernetes `Pod` object onto the ingested metrics. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "podTargetLabels" + )] + pub pod_target_labels: Option>, + /// `sampleLimit` defines a per-scrape limit on the number of scraped samples + /// that will be accepted. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "sampleLimit" + )] + pub sample_limit: Option, + /// The scrape class to apply. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "scrapeClass" + )] + pub scrape_class: Option, + /// `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + /// protocols supported by Prometheus in order of preference (from most to least preferred). + /// + /// + /// If unset, Prometheus uses its default value. + /// + /// + /// It requires Prometheus >= v2.49.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "scrapeProtocols" + )] + pub scrape_protocols: Option>, + /// Label selector to select the Kubernetes `Pod` objects. + pub selector: PodMonitorSelector, + /// `targetLimit` defines a limit on the number of scraped targets that will + /// be accepted. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "targetLimit" + )] + pub target_limit: Option, +} + +/// `attachMetadata` defines additional metadata which is added to the +/// discovered targets. +/// +/// +/// It requires Prometheus >= v2.37.0. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorAttachMetadata { + /// When set to true, Prometheus must have the `get` permission on the + /// `Nodes` objects. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub node: Option, +} + +/// Selector to select which namespaces the Kubernetes `Pods` objects +/// are discovered from. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorNamespaceSelector { + /// Boolean describing whether all namespaces are selected in contrast to a + /// list restricting them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub any: Option, + /// List of namespace names to select from. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "matchNames" + )] + pub match_names: Option>, +} + +/// PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by +/// Prometheus. +#[derive(Serialize, Default, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpoints { + /// `authorization` configures the Authorization header credentials to use when + /// scraping the target. + /// + /// + /// Cannot be set at the same time as `basicAuth`, or `oauth2`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authorization: Option, + /// `basicAuth` configures the Basic Authentication credentials to use when + /// scraping the target. + /// + /// + /// Cannot be set at the same time as `authorization`, or `oauth2`. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "basicAuth")] + pub basic_auth: Option, + /// `bearerTokenSecret` specifies a key of a Secret containing the bearer + /// token for scraping targets. The secret needs to be in the same namespace + /// as the PodMonitor object and readable by the Prometheus Operator. + /// + /// + /// Deprecated: use `authorization` instead. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "bearerTokenSecret" + )] + pub bearer_token_secret: Option, + /// `enableHttp2` can be used to disable HTTP2 when scraping the target. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "enableHttp2" + )] + pub enable_http2: Option, + /// When true, the pods which are not running (e.g. either in Failed or + /// Succeeded state) are dropped during the target discovery. + /// + /// + /// If unset, the filtering is enabled. + /// + /// + /// More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "filterRunning" + )] + pub filter_running: Option, + /// `followRedirects` defines whether the scrape requests should follow HTTP + /// 3xx redirects. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "followRedirects" + )] + pub follow_redirects: Option, + /// When true, `honorLabels` preserves the metric's labels when they collide + /// with the target's labels. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "honorLabels" + )] + pub honor_labels: Option, + /// `honorTimestamps` controls whether Prometheus preserves the timestamps + /// when exposed by the target. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "honorTimestamps" + )] + pub honor_timestamps: Option, + /// Interval at which Prometheus scrapes the metrics from the target. + /// + /// + /// If empty, Prometheus uses the global scrape interval. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub interval: Option, + /// `metricRelabelings` configures the relabeling rules to apply to the + /// samples before ingestion. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "metricRelabelings" + )] + pub metric_relabelings: Option>, + /// `oauth2` configures the OAuth2 settings to use when scraping the target. + /// + /// + /// It requires Prometheus >= 2.27.0. + /// + /// + /// Cannot be set at the same time as `authorization`, or `basicAuth`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oauth2: Option, + /// `params` define optional HTTP URL parameters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub params: Option>, + /// HTTP path from which to scrape for metrics. + /// + /// + /// If empty, Prometheus uses the default value (e.g. `/metrics`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + /// Name of the Pod port which this endpoint refers to. + /// + /// + /// It takes precedence over `targetPort`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port: Option, + /// `proxyURL` configures the HTTP Proxy URL (e.g. + /// "http://proxyserver:2195") to go through when scraping the target. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "proxyUrl")] + pub proxy_url: Option, + /// `relabelings` configures the relabeling rules to apply the target's + /// metadata labels. + /// + /// + /// The Operator automatically adds relabelings for a few standard Kubernetes fields. + /// + /// + /// The original scrape job's name is available via the `__tmp_prometheus_job_name` label. + /// + /// + /// More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relabelings: Option>, + /// HTTP scheme to use for scraping. + /// + /// + /// `http` and `https` are the expected values unless you rewrite the + /// `__scheme__` label via relabeling. + /// + /// + /// If empty, Prometheus uses the default value `http`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheme: Option, + /// Timeout after which Prometheus considers the scrape to be failed. + /// + /// + /// If empty, Prometheus uses the global scrape timeout unless it is less + /// than the target's scrape interval value in which the latter is used. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "scrapeTimeout" + )] + pub scrape_timeout: Option, + /// Name or number of the target port of the `Pod` object behind the Service, the + /// port must be specified with container port property. + /// + /// + /// Deprecated: use 'port' instead. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "targetPort" + )] + pub target_port: Option, + /// TLS configuration to use when scraping the target. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "tlsConfig")] + pub tls_config: Option, + /// `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + /// the metrics that have an explicit timestamp present in scraped data. + /// Has no effect if `honorTimestamps` is false. + /// + /// + /// It requires Prometheus >= v2.48.0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "trackTimestampsStaleness" + )] + pub track_timestamps_staleness: Option, +} + +/// `authorization` configures the Authorization header credentials to use when +/// scraping the target. +/// +/// +/// Cannot be set at the same time as `basicAuth`, or `oauth2`. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsAuthorization { + /// Selects a key of a Secret in the namespace that contains the credentials for authentication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credentials: Option, + /// Defines the authentication type. The value is case-insensitive. + /// + /// + /// "Basic" is not a supported value. + /// + /// + /// Default: "Bearer" + #[serde(default, skip_serializing_if = "Option::is_none", rename = "type")] + pub r#type: Option, +} + +/// Selects a key of a Secret in the namespace that contains the credentials for authentication. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsAuthorizationCredentials { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// `basicAuth` configures the Basic Authentication credentials to use when +/// scraping the target. +/// +/// +/// Cannot be set at the same time as `authorization`, or `oauth2`. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsBasicAuth { + /// `password` specifies a key of a Secret containing the password for + /// authentication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// `username` specifies a key of a Secret containing the username for + /// authentication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, +} + +/// `password` specifies a key of a Secret containing the password for +/// authentication. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsBasicAuthPassword { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// `username` specifies a key of a Secret containing the username for +/// authentication. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsBasicAuthUsername { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// `bearerTokenSecret` specifies a key of a Secret containing the bearer +/// token for scraping targets. The secret needs to be in the same namespace +/// as the PodMonitor object and readable by the Prometheus Operator. +/// +/// +/// Deprecated: use `authorization` instead. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsBearerTokenSecret { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// RelabelConfig allows dynamic rewriting of the label set for targets, alerts, +/// scraped samples and remote write samples. +/// +/// +/// More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsMetricRelabelings { + /// Action to perform based on the regex matching. + /// + /// + /// `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + /// `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + /// + /// + /// Default: "Replace" + #[serde(default, skip_serializing_if = "Option::is_none")] + pub action: Option, + /// Modulus to take of the hash of the source label values. + /// + /// + /// Only applicable when the action is `HashMod`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modulus: Option, + /// Regular expression against which the extracted value is matched. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regex: Option, + /// Replacement value against which a Replace action is performed if the + /// regular expression matches. + /// + /// + /// Regex capture groups are available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replacement: Option, + /// Separator is the string between concatenated SourceLabels. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub separator: Option, + /// The source labels select values from existing labels. Their content is + /// concatenated using the configured Separator and matched against the + /// configured regular expression. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "sourceLabels" + )] + pub source_labels: Option>, + /// Label to which the resulting string is written in a replacement. + /// + /// + /// It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + /// `KeepEqual` and `DropEqual` actions. + /// + /// + /// Regex capture groups are available. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "targetLabel" + )] + pub target_label: Option, +} + +/// RelabelConfig allows dynamic rewriting of the label set for targets, alerts, +/// scraped samples and remote write samples. +/// +/// +/// More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config +#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)] +pub enum PodMonitorPodMetricsEndpointsMetricRelabelingsAction { + #[default] + #[serde(rename = "replace")] + Replace, + #[serde(rename = "Replace")] + ReplaceX, + #[serde(rename = "keep")] + Keep, + #[serde(rename = "Keep")] + KeepX, + #[serde(rename = "drop")] + Drop, + #[serde(rename = "Drop")] + DropX, + #[serde(rename = "hashmod")] + Hashmod, + HashMod, + #[serde(rename = "labelmap")] + Labelmap, + LabelMap, + #[serde(rename = "labeldrop")] + Labeldrop, + LabelDrop, + #[serde(rename = "labelkeep")] + Labelkeep, + LabelKeep, + #[serde(rename = "lowercase")] + Lowercase, + #[serde(rename = "Lowercase")] + LowercaseX, + #[serde(rename = "uppercase")] + Uppercase, + #[serde(rename = "Uppercase")] + UppercaseX, + #[serde(rename = "keepequal")] + Keepequal, + KeepEqual, + #[serde(rename = "dropequal")] + Dropequal, + DropEqual, +} + +/// `oauth2` configures the OAuth2 settings to use when scraping the target. +/// +/// +/// It requires Prometheus >= 2.27.0. +/// +/// +/// Cannot be set at the same time as `authorization`, or `basicAuth`. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsOauth2 { + /// `clientId` specifies a key of a Secret or ConfigMap containing the + /// OAuth2 client's ID. + #[serde(rename = "clientId")] + pub client_id: PodMonitorPodMetricsEndpointsOauth2ClientId, + /// `clientSecret` specifies a key of a Secret containing the OAuth2 + /// client's secret. + #[serde(rename = "clientSecret")] + pub client_secret: PodMonitorPodMetricsEndpointsOauth2ClientSecret, + /// `endpointParams` configures the HTTP parameters to append to the token + /// URL. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "endpointParams" + )] + pub endpoint_params: Option>, + /// `scopes` defines the OAuth2 scopes used for the token request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scopes: Option>, + /// `tokenURL` configures the URL to fetch the token from. + #[serde(rename = "tokenUrl")] + pub token_url: String, +} + +/// `clientId` specifies a key of a Secret or ConfigMap containing the +/// OAuth2 client's ID. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsOauth2ClientId { + /// ConfigMap containing data to use for the targets. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "configMap")] + pub config_map: Option, + /// Secret containing data to use for the targets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret: Option, +} + +/// ConfigMap containing data to use for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsOauth2ClientIdConfigMap { + /// The key to select. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the ConfigMap or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// Secret containing data to use for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsOauth2ClientIdSecret { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// `clientSecret` specifies a key of a Secret containing the OAuth2 +/// client's secret. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsOauth2ClientSecret { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// RelabelConfig allows dynamic rewriting of the label set for targets, alerts, +/// scraped samples and remote write samples. +/// +/// +/// More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsRelabelings { + /// Action to perform based on the regex matching. + /// + /// + /// `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + /// `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + /// + /// + /// Default: "Replace" + #[serde(default, skip_serializing_if = "Option::is_none")] + pub action: Option, + /// Modulus to take of the hash of the source label values. + /// + /// + /// Only applicable when the action is `HashMod`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modulus: Option, + /// Regular expression against which the extracted value is matched. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regex: Option, + /// Replacement value against which a Replace action is performed if the + /// regular expression matches. + /// + /// + /// Regex capture groups are available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replacement: Option, + /// Separator is the string between concatenated SourceLabels. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub separator: Option, + /// The source labels select values from existing labels. Their content is + /// concatenated using the configured Separator and matched against the + /// configured regular expression. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "sourceLabels" + )] + pub source_labels: Option>, + /// Label to which the resulting string is written in a replacement. + /// + /// + /// It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + /// `KeepEqual` and `DropEqual` actions. + /// + /// + /// Regex capture groups are available. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "targetLabel" + )] + pub target_label: Option, +} + +/// RelabelConfig allows dynamic rewriting of the label set for targets, alerts, +/// scraped samples and remote write samples. +/// +/// +/// More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config +#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)] +pub enum PodMonitorPodMetricsEndpointsRelabelingsAction { + #[default] + #[serde(rename = "replace")] + Replace, + #[serde(rename = "Replace")] + ReplaceX, + #[serde(rename = "keep")] + Keep, + #[serde(rename = "Keep")] + KeepX, + #[serde(rename = "drop")] + Drop, + #[serde(rename = "Drop")] + DropX, + #[serde(rename = "hashmod")] + Hashmod, + HashMod, + #[serde(rename = "labelmap")] + Labelmap, + LabelMap, + #[serde(rename = "labeldrop")] + Labeldrop, + LabelDrop, + #[serde(rename = "labelkeep")] + Labelkeep, + LabelKeep, + #[serde(rename = "lowercase")] + Lowercase, + #[serde(rename = "Lowercase")] + LowercaseX, + #[serde(rename = "uppercase")] + Uppercase, + #[serde(rename = "Uppercase")] + UppercaseX, + #[serde(rename = "keepequal")] + Keepequal, + KeepEqual, + #[serde(rename = "dropequal")] + Dropequal, + DropEqual, +} + +/// PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by +/// Prometheus. +#[derive(Serialize, Deserialize, Clone, Debug, Default, JsonSchema)] +pub enum PodMonitorPodMetricsEndpointsScheme { + #[serde(rename = "http")] + Http, + #[default] + #[serde(rename = "https")] + Https, +} + +/// TLS configuration to use when scraping the target. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfig { + /// Certificate authority used when verifying server certificates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca: Option, + /// Client certificate to present when doing client-authentication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cert: Option, + /// Disable target certificate validation. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "insecureSkipVerify" + )] + pub insecure_skip_verify: Option, + /// Secret containing the client key file for the targets. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "keySecret")] + pub key_secret: Option, + /// Used to verify the hostname for the targets. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "serverName" + )] + pub server_name: Option, +} + +/// Certificate authority used when verifying server certificates. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigCa { + /// ConfigMap containing data to use for the targets. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "configMap")] + pub config_map: Option, + /// Secret containing data to use for the targets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret: Option, +} + +/// ConfigMap containing data to use for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigCaConfigMap { + /// The key to select. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the ConfigMap or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// Secret containing data to use for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigCaSecret { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// Client certificate to present when doing client-authentication. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigCert { + /// ConfigMap containing data to use for the targets. + #[serde(default, skip_serializing_if = "Option::is_none", rename = "configMap")] + pub config_map: Option, + /// Secret containing data to use for the targets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub secret: Option, +} + +/// ConfigMap containing data to use for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigCertConfigMap { + /// The key to select. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the ConfigMap or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// Secret containing data to use for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigCertSecret { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// Secret containing the client key file for the targets. +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorPodMetricsEndpointsTlsConfigKeySecret { + /// The key of the secret to select from. Must be a valid secret key. + pub key: String, + /// Name of the referent. + /// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + /// TODO: Add other useful fields. apiVersion, kind, uid? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Specify whether the Secret or its key must be defined + #[serde(default, skip_serializing_if = "Option::is_none")] + pub optional: Option, +} + +/// Label selector to select the Kubernetes `Pod` objects. +#[derive(Serialize, Default, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorSelector { + /// matchExpressions is a list of label selector requirements. The requirements are ANDed. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "matchExpressions" + )] + pub match_expressions: Option>, + /// matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + /// map is equivalent to an element of matchExpressions, whose key field is "key", the + /// operator is "In", and the values array contains only "value". The requirements are ANDed. + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "matchLabels" + )] + pub match_labels: Option>, +} + +/// A label selector requirement is a selector that contains values, a key, and an operator that +/// relates the key and values. +#[derive(Serialize, Default, Deserialize, Clone, Debug, JsonSchema)] +pub struct PodMonitorSelectorMatchExpressions { + /// key is the label key that the selector applies to. + pub key: String, + /// operator represents a key's relationship to a set of values. + /// Valid operators are In, NotIn, Exists and DoesNotExist. + pub operator: String, + /// values is an array of string values. If the operator is In or NotIn, + /// the values array must be non-empty. If the operator is Exists or DoesNotExist, + /// the values array must be empty. This array is replaced during a strategic + /// merge patch. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub values: Option>, +} diff --git a/tembo-operator/tests/integration_tests.rs b/tembo-operator/tests/integration_tests.rs index 076de40f4..47a4be5ce 100644 --- a/tembo-operator/tests/integration_tests.rs +++ b/tembo-operator/tests/integration_tests.rs @@ -587,6 +587,44 @@ mod test { use k8s_openapi::NamespaceResourceScope; use serde::{de::DeserializeOwned, Deserialize}; + async fn get_resource( + client: Client, + namespace: &str, + name: &str, + retries: usize, + // is resource expected to exist? + expected: bool, + ) -> Result + where + R: kube::api::Resource + + std::fmt::Debug + + 'static + + Clone + + DeserializeOwned + + for<'de> serde::Deserialize<'de>, + R::DynamicType: Default, + { + let api: Api = Api::namespaced(client, namespace); + for _ in 0..retries { + let resource = api.get(name).await; + if expected { + if resource.is_ok() { + return resource; + } else { + println!("Failed to get resource: {}. Retrying...", name); + thread::sleep(Duration::from_millis(2000)); + } + } else { + if resource.is_err() { + return resource; + } else { + println!("Resource {} should not exist. Retrying...", name); + thread::sleep(Duration::from_millis(2000)); + } + } + } + panic!("Timed out getting resource, {}", name); + } // helper function retrieve all instances of a resource in namespace // used repeatedly in appService tests // handles retries @@ -5540,4 +5578,146 @@ CREATE EVENT TRIGGER pgrst_watch test.teardown().await; } + + #[tokio::test] + #[ignore] + async fn functional_test_app_service_podmonitor() { + // Validates PodMonitor resource created and destroyed properly + // PodMonitor created/destroyed when adding/removing `metrics` from CoredBSpec + // PodMonitor destroyed when the corresponding AppService is deleted + + // Initialize the Kubernetes client + let client = kube_client().await; + let mut rng = rand::thread_rng(); + let suffix = rng.gen_range(0..100000); + let cdb_name = &format!("test-app-service-podmon-{}", suffix); + let namespace = match create_namespace(client.clone(), cdb_name).await { + Ok(namespace) => namespace, + Err(e) => { + eprintln!("Error creating namespace: {}", e); + std::process::exit(1); + } + }; + + let kind = "CoreDB"; + + // Apply a basic configuration of CoreDB + println!("Creating CoreDB resource {}", cdb_name); + let coredbs: Api = Api::namespaced(client.clone(), &namespace); + // generate an instance w/ 2 appServices + let full_app = serde_json::json!({ + "apiVersion": API_VERSION, + "kind": kind, + "metadata": { + "name": cdb_name + }, + "spec": { + "appServices": [ + { + "name": "dummy-exporter", + "image": "prom/blackbox-exporter", + "routing": [ + { + "port": 9115, + "ingressPath": "/metrics", + "ingressType": "http" + } + ], + "metrics": { + "path": "/metrics", + "port": 9115 + } + }, + ] + } + }); + let params = PatchParams::apply("tembo-integration-test"); + let patch = Patch::Apply(&full_app); + let _coredb_resource = coredbs.patch(cdb_name, ¶ms, &patch).await.unwrap(); + + // pause for resource creation + use controller::prometheus::podmonitor_crd::PodMonitor; + let pmon_name = format!("{}-dummy-exporter", cdb_name); + let podmon = get_resource::(client.clone(), &namespace, &pmon_name, 50, true) + .await + .unwrap(); + let pmon_spec = podmon.spec.pod_metrics_endpoints.unwrap(); + assert_eq!(pmon_spec.len(), 1); + + // disable metrics + let no_metrics_app = serde_json::json!({ + "apiVersion": API_VERSION, + "kind": kind, + "metadata": { + "name": cdb_name + }, + "spec": { + "appServices": [ + { + "name": "dummy-exporter", + "image": "prom/blackbox-exporter", + "routing": [ + { + "port": 9115, + "ingressPath": "/metrics", + "ingressType": "http" + } + ], + }, + ] + } + }); + let params = PatchParams::apply("tembo-integration-test"); + let patch = Patch::Apply(&no_metrics_app); + let _coredb_resource = coredbs.patch(cdb_name, ¶ms, &patch).await.unwrap(); + let podmon = + get_resource::(client.clone(), &namespace, &pmon_name, 50, false).await; + assert!(podmon.is_err()); + + // renable it, assert it exists, then delete the app and assert PodMonitor is gone + let patch = Patch::Apply(&full_app); + let _coredb_resource = coredbs.patch(cdb_name, ¶ms, &patch).await.unwrap(); + let podmon = get_resource::(client.clone(), &namespace, &pmon_name, 50, true) + .await + .unwrap(); + + let pmon_spec = podmon.spec.pod_metrics_endpoints.unwrap(); + assert_eq!(pmon_spec.len(), 1); + // delete the app + let no_app = serde_json::json!({ + "apiVersion": API_VERSION, + "kind": kind, + "metadata": { + "name": cdb_name + }, + "spec": { + "appServices": [] + } + }); + let patch = Patch::Apply(&no_app); + let _coredb_resource = coredbs.patch(cdb_name, ¶ms, &patch).await.unwrap(); + let podmon = + get_resource::(client.clone(), &namespace, &pmon_name, 50, false).await; + assert!(podmon.is_err()); + + // CLEANUP TEST + // Cleanup CoreDB + coredbs.delete(cdb_name, &Default::default()).await.unwrap(); + println!("Waiting for CoreDB to be deleted: {}", &cdb_name); + let _assert_coredb_deleted = tokio::time::timeout( + Duration::from_secs(TIMEOUT_SECONDS_COREDB_DELETED), + await_condition(coredbs.clone(), cdb_name, conditions::is_deleted("")), + ) + .await + .unwrap_or_else(|_| { + panic!( + "CoreDB {} was not deleted after waiting {} seconds", + cdb_name, TIMEOUT_SECONDS_COREDB_DELETED + ) + }); + println!("CoreDB resource deleted {}", cdb_name); + + // Delete namespace + let _ = delete_namespace(client.clone(), &namespace).await; + } } diff --git a/tembo-operator/yaml/sample-vectordb.yaml b/tembo-operator/yaml/sample-vectordb.yaml index 7cdc083a6..ecdc9cdd3 100644 --- a/tembo-operator/yaml/sample-vectordb.yaml +++ b/tembo-operator/yaml/sample-vectordb.yaml @@ -5,8 +5,11 @@ metadata: spec: image: "quay.io/tembo/standard-cnpg:15-a0a5ab5" appServices: - - image: quay.io/tembo/vector-serve:7343bf4 + - image: quay.io/tembo/vector-serve:5caa95d name: embeddings + metrics: + path: /metrics + port: 3000 env: - name: TMPDIR value: /models @@ -57,11 +60,11 @@ spec: name: hf-data-vol trunk_installs: - name: pgmq - version: 1.1.1 + version: 1.2.1 - name: vectorize - version: 0.4.0 + version: 0.15.0 - name: pgvector - version: 0.5.1 + version: 0.7.0 - name: pg_stat_statements version: 1.10.0 extensions: @@ -79,12 +82,12 @@ spec: locations: - database: postgres enabled: true - version: 1.1.1 + version: 1.2.1 - name: vectorize locations: - database: postgres enabled: true - version: 0.4.0 + version: 0.45.0 - name: pg_stat_statements locations: - database: postgres