Skip to content

Commit ee572bf

Browse files
KuromesiKuromesi
authored
read certs from secrets to support external certs (openkruise#1677)
Signed-off-by: Kuromesi <blackfacepan@163.com>
1 parent b19c4d8 commit ee572bf

File tree

5 files changed

+98
-32
lines changed

5 files changed

+98
-32
lines changed

‎pkg/webhook/util/configuration/configuration.go

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package configuration
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"encoding/json"
2223
"fmt"
@@ -52,15 +53,30 @@ func Ensure(kubeClient clientset.Interface, handlers map[string]types.HandlerGet
5253
if utilfeature.DefaultFeatureGate.Enabled(features.EnableExternalCerts) {
5354
// if using external certs, only check the caBundle of webhook
5455
for _, wh := range mutatingConfig.Webhooks {
55-
if wh.ClientConfig.CABundle == nil {
56-
return fmt.Errorf("caBundle of MutatingWebhookConfiguration %s is nil", mutatingWebhookConfigurationName)
57-
56+
path, err := getPath(&wh.ClientConfig)
57+
if err != nil {
58+
return err
59+
}
60+
if _, ok := handlers[path]; !ok {
61+
klog.Warningf("Ignore webhook for %s in configuration", path)
62+
continue
63+
}
64+
if !bytes.Equal(wh.ClientConfig.CABundle, caBundle) {
65+
return fmt.Errorf("caBundle of MutatingWebhookConfiguration %s does not match the external caBundle", mutatingWebhookConfigurationName)
5866
}
5967
}
6068

6169
for _, wh := range validatingConfig.Webhooks {
62-
if wh.ClientConfig.CABundle == nil {
63-
return fmt.Errorf("caBundle of ValidatingWebhookConfiguration %s is nil", mutatingWebhookConfigurationName)
70+
path, err := getPath(&wh.ClientConfig)
71+
if err != nil {
72+
return err
73+
}
74+
if _, ok := handlers[path]; !ok {
75+
klog.Warningf("Ignore webhook for %s in configuration", path)
76+
continue
77+
}
78+
if !bytes.Equal(wh.ClientConfig.CABundle, caBundle) {
79+
return fmt.Errorf("caBundle of ValidatingWebhookConfiguration %s does not match the external caBundle", validatingWebhookConfigurationName)
6480
}
6581
}
6682
return nil

‎pkg/webhook/util/controller/webhook_controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ func (c *Controller) sync() error {
238238
if utilfeature.DefaultFeatureGate.Enabled(features.EnableExternalCerts) {
239239
certWriter, err = writer.NewExternalCertWriter(writer.ExternalCertWriterOptions{
240240
Clientset: c.kubeClient,
241+
Secret: &types.NamespacedName{Namespace: webhookutil.GetNamespace(), Name: webhookutil.GetSecretName()},
241242
})
242243
} else if certWriterType == writer.FsCertWriter || (len(certWriterType) == 0 && len(webhookutil.GetHost()) != 0) {
243244
certWriter, err = writer.NewFSCertWriter(writer.FSCertWriterOptions{

‎pkg/webhook/util/crd/crd.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package crd
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"fmt"
2223
"reflect"
@@ -60,9 +61,13 @@ func Ensure(client apiextensionsclientset.Interface, lister apiextensionslisters
6061
continue
6162
}
6263

63-
if crd.Spec.Conversion.Webhook == nil || crd.Spec.Conversion.Webhook.ClientConfig == nil || crd.Spec.Conversion.Webhook.ClientConfig.CABundle == nil {
64+
if crd.Spec.Conversion.Webhook == nil || crd.Spec.Conversion.Webhook.ClientConfig == nil {
6465
return fmt.Errorf("bad conversion configuration of CRD %s", crd.Name)
6566
}
67+
68+
if !bytes.Equal(crd.Spec.Conversion.Webhook.ClientConfig.CABundle, caBundle) {
69+
return fmt.Errorf("caBundle of CRD %s does not match external caBundle", crd.Name)
70+
}
6671
}
6772
return nil
6873
}

‎pkg/webhook/util/writer/external.go

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,29 +20,38 @@ import (
2020
"bytes"
2121
"context"
2222
"errors"
23-
"fmt"
2423

24+
corev1 "k8s.io/api/core/v1"
25+
apierrors "k8s.io/apimachinery/pkg/api/errors"
2526
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/types"
2628
clientset "k8s.io/client-go/kubernetes"
2729
"k8s.io/klog/v2"
2830

2931
"github.com/openkruise/kruise/pkg/webhook/util/generator"
3032
)
3133

3234
const (
33-
mutatingWebhookConfigurationName = "kruise-mutating-webhook-configuration"
35+
ExternalCertWriter = "external"
36+
ExternalCACert = "ca.crt"
37+
ExternalCAKey = "ca.key"
38+
ExternalServerCert = "tls.crt"
39+
ExternalServerKey = "tls.key"
3440
)
3541

3642
var currentExternalCerts *generator.Artifacts
3743

38-
// externalCertWriter provisions the certificate by reading from the k8s mutating webhook configuration.
44+
// externalCertWriter provisions the certificate by reading from the k8s secrets.
3945
type externalCertWriter struct {
4046
*ExternalCertWriterOptions
4147
}
4248

4349
// ExternalCertWriterOptions is options for constructing a externalCertWriter.
4450
type ExternalCertWriterOptions struct {
51+
// client talks to a kubernetes cluster for creating the secret.
4552
Clientset clientset.Interface
53+
// secret points the secret that contains certificates that written by the CertWriter.
54+
Secret *types.NamespacedName
4655
}
4756

4857
var _ CertWriter = &externalCertWriter{}
@@ -51,9 +60,13 @@ func (ops *ExternalCertWriterOptions) validate() error {
5160
if ops.Clientset == nil {
5261
return errors.New("client must be set in externalCertWriterOptions")
5362
}
63+
if ops.Secret == nil {
64+
return errors.New("secret must be set in externalCertWriterOptions")
65+
}
5466
return nil
5567
}
5668

69+
// NewExternalCertWriter constructs a CertWriter that persists the certificate in a k8s secret.
5770
func NewExternalCertWriter(ops ExternalCertWriterOptions) (CertWriter, error) {
5871
err := ops.validate()
5972
if err != nil {
@@ -62,12 +75,14 @@ func NewExternalCertWriter(ops ExternalCertWriterOptions) (CertWriter, error) {
6275
return &externalCertWriter{ExternalCertWriterOptions: &ops}, nil
6376
}
6477

78+
// EnsureCert read and validate certs from k8s secret.
6579
func (s *externalCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) {
66-
// Read certs from mutating webhook configuration generated by external
80+
// Read certs from secrets generated externally
6781
certs, err := s.read()
6882
if err != nil {
6983
return nil, false, err
7084
}
85+
7186
// check if the certs are updated since last read
7287
if currentExternalCerts != nil && compareCerts(certs, currentExternalCerts) {
7388
klog.Info("external certs are not updated")
@@ -89,16 +104,28 @@ func (s *externalCertWriter) overwrite(resourceVersion string) (*generator.Artif
89104
}
90105

91106
func (s *externalCertWriter) read() (*generator.Artifacts, error) {
92-
mutatingConfig, err := s.Clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookConfigurationName, metav1.GetOptions{})
93-
if err != nil {
94-
return nil, fmt.Errorf("not found MutatingWebhookConfiguration %s", mutatingWebhookConfigurationName)
107+
secret, err := s.Clientset.CoreV1().Secrets(s.Secret.Namespace).Get(context.TODO(), s.Secret.Name, metav1.GetOptions{})
108+
if apierrors.IsNotFound(err) {
109+
return nil, notFoundError{err}
95110
}
96-
if len(mutatingConfig.Webhooks) == 0 {
97-
return nil, fmt.Errorf("not found webhook in MutatingWebhookConfiguration %s", mutatingWebhookConfigurationName)
111+
if err != nil {
112+
return nil, err
98113
}
114+
certs := externalSecretToCerts(secret)
115+
return certs, nil
116+
}
99117

100-
caBundle := mutatingConfig.Webhooks[0].ClientConfig.CABundle
101-
return &generator.Artifacts{CACert: caBundle}, nil
118+
func externalSecretToCerts(secret *corev1.Secret) *generator.Artifacts {
119+
ret := &generator.Artifacts{
120+
ResourceVersion: secret.ResourceVersion,
121+
}
122+
if secret.Data != nil {
123+
ret.CACert = secret.Data[ExternalCACert]
124+
ret.CAKey = secret.Data[ExternalCAKey]
125+
ret.Cert = secret.Data[ExternalServerCert]
126+
ret.Key = secret.Data[ExternalServerKey]
127+
}
128+
return ret
102129
}
103130

104131
func compareCerts(certsA, certsB *generator.Artifacts) bool {

‎pkg/webhook/util/writer/external_test.go

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ import (
2121
"testing"
2222

2323
"github.com/onsi/gomega"
24+
webhookutil "github.com/openkruise/kruise/pkg/webhook/util"
2425
"github.com/openkruise/kruise/pkg/webhook/util/generator"
25-
v1 "k8s.io/api/admissionregistration/v1"
26+
corev1 "k8s.io/api/core/v1"
2627
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/apimachinery/pkg/types"
2729
"k8s.io/client-go/kubernetes/fake"
2830
)
2931

@@ -36,11 +38,13 @@ func TestEnsureCert(t *testing.T) {
3638
teatCases := []struct {
3739
name string
3840
needRefreshCertCache bool
41+
secret types.NamespacedName
3942
expect ExpectOut
4043
}{
4144
{
42-
name: "test get certs from mutating webhook configuration",
45+
name: "test get certs from secret",
4346
needRefreshCertCache: true,
47+
secret: types.NamespacedName{Namespace: webhookutil.GetNamespace(), Name: webhookutil.GetSecretName()},
4448
expect: ExpectOut{
4549
changed: true,
4650
errorHappens: false,
@@ -49,11 +53,21 @@ func TestEnsureCert(t *testing.T) {
4953
{
5054
name: "test get certs from cache",
5155
needRefreshCertCache: false,
56+
secret: types.NamespacedName{Namespace: webhookutil.GetNamespace(), Name: webhookutil.GetSecretName()},
5257
expect: ExpectOut{
5358
changed: false,
5459
errorHappens: false,
5560
},
5661
},
62+
{
63+
name: "test secret not found",
64+
needRefreshCertCache: true,
65+
secret: types.NamespacedName{Namespace: "default", Name: "not-existed-secret"},
66+
expect: ExpectOut{
67+
changed: false,
68+
errorHappens: true,
69+
},
70+
},
5771
}
5872
dnsName := "kruise-webhook-service.svc"
5973
client := fake.NewSimpleClientset()
@@ -63,20 +77,23 @@ func TestEnsureCert(t *testing.T) {
6377
// certs expire after 10 years
6478
certs, err := certGenerator.Generate(dnsName)
6579
g.Expect(err).NotTo(gomega.HaveOccurred())
66-
wh := &v1.MutatingWebhookConfiguration{
80+
secret := corev1.Secret{
81+
TypeMeta: metav1.TypeMeta{
82+
APIVersion: "v1",
83+
Kind: "Secret",
84+
},
6785
ObjectMeta: metav1.ObjectMeta{
68-
Name: mutatingWebhookConfigurationName,
86+
Namespace: webhookutil.GetNamespace(),
87+
Name: webhookutil.GetSecretName(),
6988
},
70-
Webhooks: []v1.MutatingWebhook{
71-
{
72-
ClientConfig: v1.WebhookClientConfig{
73-
CABundle: certs.CACert,
74-
},
75-
},
89+
Data: map[string][]byte{
90+
ExternalCAKey: certs.CAKey,
91+
ExternalCACert: certs.CACert,
92+
ExternalServerKey: certs.Key,
93+
ExternalServerCert: certs.Cert,
7694
},
7795
}
78-
79-
_, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), wh, metav1.CreateOptions{})
96+
_, err = client.CoreV1().Secrets(webhookutil.GetNamespace()).Create(context.TODO(), &secret, metav1.CreateOptions{})
8097
g.Expect(err).NotTo(gomega.HaveOccurred())
8198

8299
for _, tc := range teatCases {
@@ -89,10 +106,10 @@ func TestEnsureCert(t *testing.T) {
89106

90107
externalCertWriter, err := NewExternalCertWriter(ExternalCertWriterOptions{
91108
Clientset: client,
109+
Secret: &types.NamespacedName{Namespace: tc.secret.Namespace, Name: tc.secret.Name},
92110
})
93111
g.Expect(err).NotTo(gomega.HaveOccurred())
94-
externalCerts, changed, err := externalCertWriter.EnsureCert(dnsName)
95-
g.Expect(externalCerts.CACert).Should(gomega.Equal(certs.CACert))
112+
_, changed, err := externalCertWriter.EnsureCert(dnsName)
96113
g.Expect(changed).Should(gomega.Equal(tc.expect.changed))
97114
g.Expect(err != nil).Should(gomega.Equal(tc.expect.errorHappens))
98115
})
@@ -104,5 +121,5 @@ func RefreshCurrentExternalCertsCache() {
104121
}
105122

106123
func UpdateCurrentExternalCertsCache(externalCerts *generator.Artifacts) {
107-
currentExternalCerts.CACert = externalCerts.CACert
124+
currentExternalCerts = externalCerts
108125
}

0 commit comments

Comments
 (0)