feat: enable additional scheduling options for wasmCloud host pods

This refactors the WasmCloudHostConfig CRD so that it has a single field
(`schedulingOptions`) for configuring how the underlying Pods are
scheduled in Kubernetes. This includes:
* Relocating the `daemonset` option to this new field
* Relocating the `resources` option to this new field
* Adding a new `pod_template_additions` field that allows you to set any
  valid option in a `PodSpec`

Doing so allows cluster operators to do things like set node affinity
and node selector rules, along with any other valid PodSpec option. The
only thing that cannot be done is adding additional containers to the
pod, since that is all handled by the controller. We could look at
exposing that option if users want to be able to add additional
sidecars.

Signed-off-by: Dan Norris <protochron@users.noreply.github.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "wasmcloud-operator"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 
 [[bin]]

Dockerfile.local

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM rust:1.75-bookworm as builder
+FROM rust:1.77-bookworm as builder
 
 WORKDIR /app
 COPY . .

crates/types/src/v1alpha1/wasmcloud_host_config.rs

@@ -1,8 +1,8 @@
-use k8s_openapi::api::core::v1::ResourceRequirements;
+use k8s_openapi::api::core::v1::{PodSpec, ResourceRequirements};
 use kube::CustomResource;
-use schemars::JsonSchema;
+use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{BTreeSet, HashMap};
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
 #[cfg_attr(test, derive(Default))]
@@ -36,10 +36,6 @@ pub struct WasmCloudHostConfigSpec {
     pub enable_structured_logging: Option<bool>,
     /// Name of a secret containing the registry credentials
     pub registry_credentials_secret: Option<String>,
-    /// Kubernetes resources to allocate for the host. See
-    /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid
-    /// values to use here.
-    pub resources: Option<WasmCloudHostConfigResources>,
     /// The control topic prefix to use for the host.
     pub control_topic_prefix: Option<String>,
     /// The leaf node domain to use for the NATS sidecar. Defaults to "leaf".
@@ -57,9 +53,39 @@ pub struct WasmCloudHostConfigSpec {
     /// The log level to use for the host. Defaults to "INFO".
     #[serde(default = "default_log_level")]
     pub log_level: String,
+    /// Kubernetes scheduling options for the wasmCloud host.
+    pub scheduling_options: Option<KubernetesSchedulingOptions>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)]
+pub struct KubernetesSchedulingOptions {
     /// Run hosts as a DaemonSet instead of a Deployment.
     #[serde(default)]
     pub daemonset: bool,
+    /// Kubernetes resources to allocate for the host. See
+    /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid
+    /// values to use here.
+    pub resources: Option<WasmCloudHostConfigResources>,
+    #[schemars(schema_with = "pod_schema")]
+    /// Any other pod template spec options to set for the underlying wasmCloud host pods.
+    pub pod_template_additions: Option<PodSpec>,
+}
+
+/// This is a workaround for the fact that we can't override the PodSpec schema to make containers
+/// an optional field. It generates the OpenAPI schema for the PodSpec type the same way that
+/// kube.rs does while dropping any required fields.
+fn pod_schema(_gen: &mut SchemaGenerator) -> Schema {
+    let gen = schemars::gen::SchemaSettings::openapi3()
+        .with(|s| {
+            s.inline_subschemas = true;
+            s.meta_schema = None;
+        })
+        .with_visitor(kube::core::schema::StructuralSchemaRewriter)
+        .into_generator();
+    let mut val = gen.into_root_schema_for::<PodSpec>();
+    // Drop `containers` as a required field, along with any others.
+    val.schema.object.as_mut().unwrap().required = BTreeSet::new();
+    val.schema.into()
 }
 
 fn default_host_replicas() -> u32 {

sample.yaml

@@ -18,5 +18,23 @@ spec:
   secretName: cluster-secrets
   logLevel: INFO
   natsAddress: nats://nats-cluster.default.svc.cluster.local
-  # Enable the following to run the wasmCloud hosts as a DaemonSet
-  #daemonset: true
+  # Additional options to control how the underlying wasmCloud hosts are scheduled in Kubernetes.
+  # This includes setting resource requirements for the nats and wasmCloud host
+  # containers along with any additional pot template settings.
+  #schedulingOptions:
+    # Enable the following to run the wasmCloud hosts as a DaemonSet
+    #daemonset: true
+    # Set the resource requirements for the nats and wasmCloud host containers.
+    #resources:
+    #  nats:
+    #    requests:
+    #      cpu: 100m
+    # wasmCloudHost:
+    #    requests:
+    #      cpu: 100m
+    # Any additional pod template settings to apply to the wasmCloud host pods.
+    # See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#podspec-v1-core for all valid options.
+    # Note that you *cannot* set the `containers` field here as it is managed by the controller.
+    #pod_template_additions:
+    #  nodeSelector:
+    #    kubernetes.io/os: linux

src/controller.rs

@@ -315,9 +315,11 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplate
 
     let mut nats_resources: Option<k8s_openapi::api::core::v1::ResourceRequirements> = None;
     let mut wasmcloud_resources: Option<k8s_openapi::api::core::v1::ResourceRequirements> = None;
-    if let Some(resources) = &config.spec.resources {
-        nats_resources = resources.nats.clone();
-        wasmcloud_resources = resources.wasmcloud.clone();
+    if let Some(scheduling_options) = &config.spec.scheduling_options {
+        if let Some(resources) = &scheduling_options.resources {
+            nats_resources = resources.nats.clone();
+            wasmcloud_resources = resources.wasmcloud.clone();
+        }
     }
 
     let containers = vec![
@@ -371,14 +373,34 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplate
             ..Default::default()
         },
     ];
-    PodTemplateSpec {
+
+    let mut volumes = vec![
+        Volume {
+            name: "nats-config".to_string(),
+            config_map: Some(ConfigMapVolumeSource {
+                name: Some(config.name_any()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        },
+        Volume {
+            name: "nats-creds".to_string(),
+            secret: Some(SecretVolumeSource {
+                secret_name: Some(config.spec.secret_name.clone()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        },
+    ];
+    let service_account = config.name_any();
+    let mut template = PodTemplateSpec {
         metadata: Some(ObjectMeta {
             labels: Some(labels),
             ..Default::default()
         }),
         spec: Some(PodSpec {
             service_account: Some(config.name_any()),
-            containers,
+            containers: containers.clone(),
             volumes: Some(vec![
                 Volume {
                     name: "nats-config".to_string(),
@@ -399,7 +421,21 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc<Context>) -> PodTemplate
             ]),
             ..Default::default()
         }),
-    }
+    };
+
+    if let Some(scheduling_options) = &config.spec.scheduling_options {
+        if let Some(pod_overrides) = &scheduling_options.pod_template_additions {
+            let mut overrides = pod_overrides.clone();
+            overrides.service_account_name = Some(service_account);
+            overrides.containers = containers.clone();
+            if let Some(vols) = overrides.volumes {
+                volumes.extend(vols);
+            }
+            overrides.volumes = Some(volumes);
+            template.spec = Some(overrides);
+        }
+    };
+    template
 }
 
 fn deployment_spec(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> DeploymentSpec {
@@ -504,31 +540,34 @@ async fn configure_hosts(config: &WasmCloudHostConfig, ctx: Arc<Context>) -> Res
         ];
     }
 
-    if config.spec.daemonset {
-        let mut spec = daemonset_spec(config, ctx.clone());
-        spec.template.spec.as_mut().unwrap().containers[1]
-            .env
-            .as_mut()
-            .unwrap()
-            .append(&mut env_vars);
-        let ds = DaemonSet {
-            metadata: ObjectMeta {
-                name: Some(config.name_any()),
-                namespace: Some(config.namespace().unwrap()),
-                owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+    if let Some(scheduling_options) = &config.spec.scheduling_options {
+        if scheduling_options.daemonset {
+            let mut spec = daemonset_spec(config, ctx.clone());
+            spec.template.spec.as_mut().unwrap().containers[1]
+                .env
+                .as_mut()
+                .unwrap()
+                .append(&mut env_vars);
+            let ds = DaemonSet {
+                metadata: ObjectMeta {
+                    name: Some(config.name_any()),
+                    namespace: Some(config.namespace().unwrap()),
+                    owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]),
+                    ..Default::default()
+                },
+                spec: Some(spec),
                 ..Default::default()
-            },
-            spec: Some(spec),
-            ..Default::default()
-        };
-
-        let api = Api::<DaemonSet>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
-        api.patch(
-            &config.name_any(),
-            &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
-            &Patch::Apply(ds),
-        )
-        .await?;
+            };
+
+            let api =
+                Api::<DaemonSet>::namespaced(ctx.client.clone(), &config.namespace().unwrap());
+            api.patch(
+                &config.name_any(),
+                &PatchParams::apply(CLUSTER_CONFIG_FINALIZER),
+                &Patch::Apply(ds),
+            )
+            .await?;
+        }
     } else {
         let mut spec = deployment_spec(config, ctx.clone());
         spec.template.spec.as_mut().unwrap().containers[1]