Skip to content

Commit ccfe0eb

Browse files
authored
🤖 feat: remove provisioner bootstrap credentials (#79)
## Summary Remove `CoderProvisioner` bootstrap credential configuration (`spec.bootstrap.credentialsSecretRef`) and make provisioner key management rely exclusively on operator-managed access from the referenced `CoderControlPlane` status. ## Background `CoderProvisioner` accepted user-supplied bootstrap tokens while `CoderControlPlane` now manages an operator token lifecycle. This change aligns provisioner reconciliation with the operator-managed token source and removes duplicate credential wiring in v1alpha1. ## Implementation - API/schema - Removed `CoderProvisionerBootstrapSpec` - Removed `CoderProvisionerSpec.Bootstrap` - Replaced `CoderProvisionerConditionBootstrapSecretReady` with `CoderProvisionerConditionOperatorAccessReady` - Regenerated deepcopy and CRD artifacts - Controller behavior - Provisioner reconciliation now resolves session token only from referenced control plane status: - `status.operatorAccessReady == true` - `status.operatorTokenSecretRef` present and valid - Added explicit condition reasons/messages and requeue behavior for not-ready/missing/invalid token-ref states - Kept defensive assertion style and best-effort deletion semantics - Samples/docs - Updated provisioner sample to remove `spec.bootstrap` - Kept sample namespace as `coder` - Regenerated API reference docs for `CoderProvisioner` - Tests - Removed provisioner bootstrap secret setup assumptions - Added/updated coverage for: - operator access ready path - operator access not ready - missing/invalid `operatorTokenSecretRef` ## Validation - `make verify-vendor` - `make test` - `make build` - `make lint` - `make codegen` - `make manifests` - `KUBEBUILDER_ASSETS="$(GOFLAGS=-mod=vendor go run ./vendor/sigs.k8s.io/controller-runtime/tools/setup-envtest use 1.35.x --bin-dir $(pwd)/bin/envtest -p path)" GOFLAGS=-mod=vendor go test ./internal/controller/...` - `mkdocs build --strict` ## Risks - **Backward-incompatible API behavior for v1alpha1**: clients/manifests still setting `spec.bootstrap` on `CoderProvisioner` must be updated. - Runtime dependency is now explicitly on referenced `CoderControlPlane` operator access status/secret ref readiness. - `CoderWorkspaceProxy` bootstrap credential semantics were intentionally left unchanged. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `$MUX_MODEL_STRING` • Thinking: `$MUX_THINKING_LEVEL` • Cost: `$$MUX_COSTS_USD`_ <!-- mux-attribution: model=$MUX_MODEL_STRING thinking=$MUX_THINKING_LEVEL costs=$MUX_COSTS_USD -->
1 parent c82ff7c commit ccfe0eb

File tree

9 files changed

+271
-227
lines changed

9 files changed

+271
-227
lines changed

‎api/v1alpha1/coderprovisioner_types.go‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ const (
1313

1414
// CoderProvisionerConditionControlPlaneReady indicates whether the referenced control plane is reachable.
1515
CoderProvisionerConditionControlPlaneReady = "ControlPlaneReady"
16-
// CoderProvisionerConditionBootstrapSecretReady indicates whether the bootstrap credentials secret is available.
17-
CoderProvisionerConditionBootstrapSecretReady = "BootstrapSecretReady"
16+
// CoderProvisionerConditionOperatorAccessReady indicates whether operator-managed
17+
// access from the referenced control plane is ready for provisioner key management.
18+
CoderProvisionerConditionOperatorAccessReady = "OperatorAccessReady"
1819
// CoderProvisionerConditionProvisionerKeyReady indicates whether the provisioner key exists in coderd.
1920
CoderProvisionerConditionProvisionerKeyReady = "ProvisionerKeyReady"
2021
// CoderProvisionerConditionProvisionerKeySecretReady indicates whether the provisioner key secret is populated.
@@ -32,13 +33,6 @@ const (
3233
ProvisionerKeyCleanupFinalizer = "coder.com/provisioner-key-cleanup"
3334
)
3435

35-
// CoderProvisionerBootstrapSpec configures credentials for provisioner key management.
36-
type CoderProvisionerBootstrapSpec struct {
37-
// CredentialsSecretRef points to a Secret containing a Coder session token
38-
// with permission to manage provisioner keys.
39-
CredentialsSecretRef SecretKeySelector `json:"credentialsSecretRef"`
40-
}
41-
4236
// CoderProvisionerKeySpec configures provisioner key naming and storage.
4337
type CoderProvisionerKeySpec struct {
4438
// Name is the provisioner key name in coderd. Defaults to the CR name.
@@ -55,12 +49,12 @@ type CoderProvisionerKeySpec struct {
5549
// CoderProvisionerSpec defines the desired state of a CoderProvisioner.
5650
type CoderProvisionerSpec struct {
5751
// ControlPlaneRef identifies which CoderControlPlane instance to join.
52+
// Provisioner key management uses operator-managed access from this
53+
// control plane's status.operatorTokenSecretRef.
5854
ControlPlaneRef corev1.LocalObjectReference `json:"controlPlaneRef"`
5955
// OrganizationName is the Coder organization. Defaults to "default".
6056
// +kubebuilder:validation:MaxLength=128
6157
OrganizationName string `json:"organizationName,omitempty"`
62-
// Bootstrap configures credentials for provisioner key management.
63-
Bootstrap CoderProvisionerBootstrapSpec `json:"bootstrap"`
6458
// Key configures provisioner key naming and secret storage.
6559
Key CoderProvisionerKeySpec `json:"key,omitempty"`
6660
// Replicas is the desired number of provisioner pods.

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 0 additions & 18 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/coder.com_coderprovisioners.yaml‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -50,30 +50,11 @@ spec:
5050
spec:
5151
description: CoderProvisionerSpec defines the desired state of a CoderProvisioner.
5252
properties:
53-
bootstrap:
54-
description: Bootstrap configures credentials for provisioner key
55-
management.
56-
properties:
57-
credentialsSecretRef:
58-
description: |-
59-
CredentialsSecretRef points to a Secret containing a Coder session token
60-
with permission to manage provisioner keys.
61-
properties:
62-
key:
63-
description: Key is the key inside the Secret data map.
64-
type: string
65-
name:
66-
description: Name is the Kubernetes Secret name.
67-
type: string
68-
required:
69-
- name
70-
type: object
71-
required:
72-
- credentialsSecretRef
73-
type: object
7453
controlPlaneRef:
75-
description: ControlPlaneRef identifies which CoderControlPlane instance
76-
to join.
54+
description: |-
55+
ControlPlaneRef identifies which CoderControlPlane instance to join.
56+
Provisioner key management uses operator-managed access from this
57+
control plane's status.operatorTokenSecretRef.
7758
properties:
7859
name:
7960
default: ""
@@ -369,7 +350,6 @@ spec:
369350
format: int64
370351
type: integer
371352
required:
372-
- bootstrap
373353
- controlPlaneRef
374354
type: object
375355
status:

‎config/samples/coder_v1alpha1_codercontrolplane.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ apiVersion: coder.com/v1alpha1
22
kind: CoderControlPlane
33
metadata:
44
name: codercontrolplane-sample
5-
namespace: default
5+
namespace: coder
66
spec:
77
image: "ghcr.io/coder/coder:latest"
88
# Optional Enterprise license upload:

‎config/samples/coder_v1alpha1_coderprovisioner.yaml‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,16 @@
22
#
33
# Prerequisites:
44
# 1. A CoderControlPlane resource must exist in the same namespace.
5-
# 2. Create the bootstrap credentials Secret with a valid Coder session token:
6-
#
7-
# kubectl create secret generic coder-bootstrap-token \
8-
# --namespace=default \
9-
# --from-literal=token=<YOUR_SESSION_TOKEN>
10-
#
11-
# IMPORTANT: Never commit real tokens to source control.
5+
# 2. The referenced CoderControlPlane status must report operator access ready:
6+
# status.operatorAccessReady=true and status.operatorTokenSecretRef set.
127
apiVersion: coder.com/v1alpha1
138
kind: CoderProvisioner
149
metadata:
1510
name: coderprovisioner-sample
16-
namespace: default
11+
namespace: coder
1712
spec:
1813
controlPlaneRef:
1914
name: codercontrolplane-sample
20-
bootstrap:
21-
credentialsSecretRef:
22-
name: coder-bootstrap-token
23-
key: token
2415
# key:
2516
# name: my-provisioner-key # Provisioner key name in coderd (defaults to CR name)
2617
# secretName: my-key-secret # K8s Secret to store the key (defaults to "{name}-provisioner-key")

‎docs/reference/api/coderprovisioner.md‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,8 @@
1313

1414
| Field | Type | Description |
1515
| --- | --- | --- |
16-
| `controlPlaneRef` | [LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#localobjectreference-v1-core) | ControlPlaneRef identifies which CoderControlPlane instance to join. |
16+
| `controlPlaneRef` | [LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#localobjectreference-v1-core) | ControlPlaneRef identifies which CoderControlPlane instance to join. Provisioner key management uses operator-managed access from this control plane's status.operatorTokenSecretRef. |
1717
| `organizationName` | string | OrganizationName is the Coder organization. Defaults to "default". |
18-
| `bootstrap` | [CoderProvisionerBootstrapSpec](#coderprovisionerbootstrapspec) | Bootstrap configures credentials for provisioner key management. |
1918
| `key` | [CoderProvisionerKeySpec](#coderprovisionerkeyspec) | Key configures provisioner key naming and secret storage. |
2019
| `replicas` | integer | Replicas is the desired number of provisioner pods. |
2120
| `tags` | object (keys:string, values:string) | Tags are attached to the provisioner key for job routing. |
@@ -45,14 +44,6 @@
4544

4645
## Referenced types
4746

48-
### CoderProvisionerBootstrapSpec
49-
50-
CoderProvisionerBootstrapSpec configures credentials for provisioner key management.
51-
52-
| Field | Type | Description |
53-
| --- | --- | --- |
54-
| `credentialsSecretRef` | [SecretKeySelector](#secretkeyselector) | CredentialsSecretRef points to a Secret containing a Coder session token with permission to manage provisioner keys. |
55-
5647
### CoderProvisionerKeySpec
5748

5849
CoderProvisionerKeySpec configures provisioner key naming and storage.

‎docs/tutorials/getting-started.md‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ Install CRDs into your cluster:
3434
kubectl apply -f config/crd/bases/
3535
```
3636

37+
Create the sample namespace used by shipped manifests:
38+
39+
```bash
40+
kubectl create namespace coder
41+
```
42+
3743
## 2) Run the controller locally
3844

3945
Start controller mode (terminal A):
@@ -58,19 +64,19 @@ Check resource status:
5864

5965
```bash
6066
kubectl get codercontrolplanes -A
61-
kubectl describe codercontrolplane codercontrolplane-sample -n default
67+
kubectl describe codercontrolplane codercontrolplane-sample -n coder
6268
```
6369

6470
The controller creates a Deployment + Service named after the control plane (`codercontrolplane-sample`) in the same namespace.
6571

6672
```bash
67-
kubectl get deploy,svc -n default
73+
kubectl get deploy,svc -n coder
6874
```
6975

7076
## 5) Clean up (optional)
7177

7278
```bash
73-
kubectl delete codercontrolplane codercontrolplane-sample -n default
79+
kubectl delete codercontrolplane codercontrolplane-sample -n coder
7480
```
7581

7682
If you used `kind-dev-up`, you can remove the cluster with:

‎internal/controller/coderprovisioner_controller.go‎

Lines changed: 82 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ const (
5050
provisionerRateLimitJitterRatio = 0.2
5151

5252
externalProvisionerEntitlementRetryInterval = 2 * time.Minute
53+
provisionerOperatorAccessRetryInterval = 15 * time.Second
5354

5455
// #nosec G101 -- this is a field index key, not a credential.
5556
provisionerControlPlaneRefNameFieldIndex = ".spec.controlPlaneRef.name"
@@ -171,24 +172,27 @@ func (r *CoderProvisionerReconciler) Reconcile(ctx context.Context, req ctrl.Req
171172
organizationName := provisionerOrganizationName(provisioner.Spec.OrganizationName)
172173
keyName, keySecretName, keySecretKey := provisionerKeyConfig(provisioner)
173174

174-
sessionToken, err := r.readBootstrapSessionToken(ctx, provisioner)
175+
sessionToken, operatorAccessResult, operatorAccessReason, operatorAccessMessage, err := r.readOperatorSessionToken(ctx, provisioner, controlPlane)
175176
if err != nil {
177+
return ctrl.Result{}, err
178+
}
179+
if operatorAccessResult.RequeueAfter > 0 {
176180
setCondition(
177181
provisioner,
178-
coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady,
182+
coderv1alpha1.CoderProvisionerConditionOperatorAccessReady,
179183
metav1.ConditionFalse,
180-
"BootstrapSecretUnavailable",
181-
fmt.Sprintf("Failed to read bootstrap credentials: %v", err),
184+
operatorAccessReason,
185+
operatorAccessMessage,
182186
)
183187
_ = r.Status().Update(ctx, provisioner)
184-
return ctrl.Result{}, err
188+
return operatorAccessResult, nil
185189
}
186190
setCondition(
187191
provisioner,
188-
coderv1alpha1.CoderProvisionerConditionBootstrapSecretReady,
192+
coderv1alpha1.CoderProvisionerConditionOperatorAccessReady,
189193
metav1.ConditionTrue,
190-
"BootstrapSecretAvailable",
191-
"Bootstrap credentials secret is available",
194+
operatorAccessReason,
195+
operatorAccessMessage,
192196
)
193197

194198
entitlementResult, entitlementErr := r.reconcileExternalProvisionerEntitlement(ctx, provisioner, controlPlane, sessionToken)
@@ -643,28 +647,40 @@ func (r *CoderProvisionerReconciler) reconcileDeletion(ctx context.Context, prov
643647
}
644648

645649
// Best-effort remote key cleanup: if the referenced control plane,
646-
// its URL, bootstrap credentials, or any other prerequisite is
650+
// its URL, operator access token, or any other prerequisite is
647651
// unavailable, log a warning and proceed to finalizer removal so the
648652
// CR does not get stuck in Terminating. This is common during
649653
// namespace teardown, when the control plane was never ready, or
650-
// when credentials were misconfigured.
654+
// when operator access bootstrap has not completed.
651655
controlPlaneURL := provisioner.Status.ControlPlaneURL
652-
if controlPlaneURL == "" {
653-
controlPlane, err := r.fetchControlPlane(ctx, provisioner)
654-
if err != nil {
655-
log.Info("unable to reach referenced CoderControlPlane during deletion, skipping remote key cleanup",
656-
"controlPlaneRef", provisioner.Spec.ControlPlaneRef.Name, "error", err)
657-
} else {
658-
controlPlaneURL = controlPlane.Status.URL
656+
controlPlaneName := strings.TrimSpace(provisioner.Spec.ControlPlaneRef.Name)
657+
var controlPlane *coderv1alpha1.CoderControlPlane
658+
if controlPlaneName != "" {
659+
controlPlane = &coderv1alpha1.CoderControlPlane{}
660+
namespacedName := types.NamespacedName{Name: controlPlaneName, Namespace: provisioner.Namespace}
661+
if err := r.Get(ctx, namespacedName, controlPlane); err != nil {
662+
log.Info("unable to read referenced CoderControlPlane during deletion, skipping remote key cleanup",
663+
"controlPlaneRef", controlPlaneName, "error", err)
664+
controlPlane = nil
665+
} else if controlPlane.Name != controlPlaneName || controlPlane.Namespace != provisioner.Namespace {
666+
return ctrl.Result{}, fmt.Errorf("assertion failed: fetched control plane %s/%s does not match expected %s/%s",
667+
controlPlane.Namespace, controlPlane.Name, provisioner.Namespace, controlPlaneName)
659668
}
660669
}
670+
if controlPlaneURL == "" && controlPlane != nil {
671+
controlPlaneURL = strings.TrimSpace(controlPlane.Status.URL)
672+
}
661673

662-
if controlPlaneURL != "" {
663-
sessionToken, tokenErr := r.readBootstrapSessionToken(ctx, provisioner)
664-
if tokenErr != nil {
665-
log.Info("unable to read bootstrap credentials during deletion, skipping remote key cleanup",
666-
"credentialsSecretRef", provisioner.Spec.Bootstrap.CredentialsSecretRef.Name, "error", tokenErr)
667-
} else {
674+
if controlPlaneURL != "" && controlPlane != nil {
675+
sessionToken, tokenResult, tokenReason, tokenMessage, tokenErr := r.readOperatorSessionToken(ctx, provisioner, controlPlane)
676+
switch {
677+
case tokenErr != nil:
678+
log.Info("unable to resolve operator access token during deletion, skipping remote key cleanup",
679+
"error", tokenErr)
680+
case tokenResult.RequeueAfter > 0:
681+
log.Info("operator access is not ready during deletion, skipping remote key cleanup",
682+
"reason", tokenReason, "message", tokenMessage)
683+
default:
668684
if deleteErr := r.BootstrapClient.DeleteProvisionerKey(
669685
ctx,
670686
controlPlaneURL,
@@ -726,23 +742,54 @@ func (r *CoderProvisionerReconciler) fetchControlPlane(ctx context.Context, prov
726742
return controlPlane, nil
727743
}
728744

729-
func (r *CoderProvisionerReconciler) readBootstrapSessionToken(ctx context.Context, provisioner *coderv1alpha1.CoderProvisioner) (string, error) {
730-
credentialsRef := provisioner.Spec.Bootstrap.CredentialsSecretRef
731-
if credentialsRef.Name == "" {
732-
return "", fmt.Errorf("coderprovisioner %s/%s spec.bootstrap.credentialsSecretRef.name is required", provisioner.Namespace, provisioner.Name)
745+
func (r *CoderProvisionerReconciler) readOperatorSessionToken(
746+
ctx context.Context,
747+
provisioner *coderv1alpha1.CoderProvisioner,
748+
controlPlane *coderv1alpha1.CoderControlPlane,
749+
) (string, ctrl.Result, string, string, error) {
750+
if provisioner == nil {
751+
return "", ctrl.Result{}, "", "", fmt.Errorf("assertion failed: coder provisioner must not be nil")
752+
}
753+
if controlPlane == nil {
754+
return "", ctrl.Result{}, "", "", fmt.Errorf("assertion failed: coder control plane must not be nil")
733755
}
734756

735-
credentialsKey := credentialsRef.Key
736-
if credentialsKey == "" {
737-
credentialsKey = coderv1alpha1.DefaultTokenSecretKey
757+
if !controlPlane.Status.OperatorAccessReady {
758+
return "", ctrl.Result{RequeueAfter: provisionerOperatorAccessRetryInterval},
759+
"OperatorAccessNotReady",
760+
fmt.Sprintf("Waiting for CoderControlPlane %s/%s status.operatorAccessReady=true before reconciling provisioner keys.", controlPlane.Namespace, controlPlane.Name),
761+
nil
762+
}
763+
764+
operatorTokenRef := controlPlane.Status.OperatorTokenSecretRef
765+
if operatorTokenRef == nil {
766+
return "", ctrl.Result{RequeueAfter: provisionerOperatorAccessRetryInterval},
767+
"OperatorTokenSecretRefMissing",
768+
fmt.Sprintf("Waiting for CoderControlPlane %s/%s status.operatorTokenSecretRef to be set by operator access bootstrap.", controlPlane.Namespace, controlPlane.Name),
769+
nil
770+
}
771+
772+
operatorTokenSecretName := strings.TrimSpace(operatorTokenRef.Name)
773+
if operatorTokenSecretName == "" {
774+
return "", ctrl.Result{RequeueAfter: provisionerOperatorAccessRetryInterval},
775+
"OperatorTokenSecretRefInvalid",
776+
fmt.Sprintf("Waiting for CoderControlPlane %s/%s status.operatorTokenSecretRef.name to be non-empty.", controlPlane.Namespace, controlPlane.Name),
777+
nil
778+
}
779+
operatorTokenSecretKey := strings.TrimSpace(operatorTokenRef.Key)
780+
if operatorTokenSecretKey == "" {
781+
operatorTokenSecretKey = coderv1alpha1.DefaultTokenSecretKey
738782
}
739783

740-
token, err := r.readSecretValue(ctx, provisioner.Namespace, credentialsRef.Name, credentialsKey)
784+
token, err := r.readSecretValue(ctx, provisioner.Namespace, operatorTokenSecretName, operatorTokenSecretKey)
741785
if err != nil {
742-
return "", fmt.Errorf("read bootstrap credentials secret %q/%q key %q: %w", provisioner.Namespace, credentialsRef.Name, credentialsKey, err)
786+
return "", ctrl.Result{RequeueAfter: provisionerOperatorAccessRetryInterval},
787+
"OperatorTokenSecretUnavailable",
788+
fmt.Sprintf("Failed to read operator token Secret %q/%q key %q from referenced control plane status: %v", provisioner.Namespace, operatorTokenSecretName, operatorTokenSecretKey, err),
789+
nil
743790
}
744791

745-
return token, nil
792+
return token, ctrl.Result{}, "OperatorAccessReady", "Operator-managed access token from referenced control plane is available", nil
746793
}
747794

748795
func (r *CoderProvisionerReconciler) reconcileExternalProvisionerEntitlement(
@@ -761,7 +808,7 @@ func (r *CoderProvisionerReconciler) reconcileExternalProvisionerEntitlement(
761808
return ctrl.Result{}, fmt.Errorf("assertion failed: coder control plane URL must not be empty")
762809
}
763810
if sessionToken == "" {
764-
return ctrl.Result{}, fmt.Errorf("assertion failed: bootstrap session token must not be empty")
811+
return ctrl.Result{}, fmt.Errorf("assertion failed: operator session token must not be empty")
765812
}
766813

767814
if controlPlane.Status.EntitlementsLastChecked != nil {
@@ -806,7 +853,7 @@ func (r *CoderProvisionerReconciler) reconcileExternalProvisionerEntitlement(
806853
message = "Coder deployment does not expose /api/v2/entitlements; cannot verify license."
807854
case http.StatusUnauthorized, http.StatusForbidden:
808855
reason = "Forbidden"
809-
message = "Bootstrap token is not authorized to read entitlements; retrying."
856+
message = "Operator access token is not authorized to read entitlements; retrying."
810857
}
811858
}
812859

0 commit comments

Comments
 (0)