Skip to content

Commit

Permalink
Merge pull request #6665 from Nuckal777/vpa-certs2
Browse files Browse the repository at this point in the history
Add ability to the VPA admission-controller to reload it's certificate v2
  • Loading branch information
k8s-ci-robot authored Jun 25, 2024
2 parents 6d54cb7 + 0bd7db4 commit 22f8c84
Show file tree
Hide file tree
Showing 30 changed files with 4,589 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
args: ["--v=4", "--stderrthreshold=info", "--reload-cert"]
volumeMounts:
- name: tls-certs
mountPath: "/etc/tls-certs"
Expand Down
56 changes: 56 additions & 0 deletions vertical-pod-autoscaler/e2e/v1/admission_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package autoscaling
import (
"context"
"fmt"
"io"
"strings"
"time"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -852,6 +854,60 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
gomega.Expect(err2.Error()).To(gomega.MatchRegexp(`.*admission webhook .*vpa.* denied the request: .*`))
})

ginkgo.It("reloads the webhook certificate", func(ctx ginkgo.SpecContext) {
ginkgo.By("Retrieving alternative certificate")
c := f.ClientSet
e2eCertsSecret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(ctx, "vpa-e2e-certs", metav1.GetOptions{})
gomega.Expect(err).To(gomega.Succeed(), "Failed to get vpa-e2e-certs secret")
actualCertsSecret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(ctx, "vpa-tls-certs", metav1.GetOptions{})
gomega.Expect(err).To(gomega.Succeed(), "Failed to get vpa-tls-certs secret")
actualCertsSecret.Data["serverKey.pem"] = e2eCertsSecret.Data["e2eKey.pem"]
actualCertsSecret.Data["serverCert.pem"] = e2eCertsSecret.Data["e2eCert.pem"]
_, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Update(ctx, actualCertsSecret, metav1.UpdateOptions{})
gomega.Expect(err).To(gomega.Succeed(), "Failed to update vpa-tls-certs secret with e2e rotation certs")

ginkgo.By("Waiting for certificate reload")
pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{})
gomega.Expect(err).To(gomega.Succeed())

var admissionController apiv1.Pod
for _, p := range pods.Items {
if strings.HasPrefix(p.Name, "vpa-admission-controller") {
admissionController = p
}
}
gomega.Expect(admissionController.Name).ToNot(gomega.BeEmpty())

gomega.Eventually(func(g gomega.Gomega) string {
reader, err := c.CoreV1().Pods(metav1.NamespaceSystem).GetLogs(admissionController.Name, &apiv1.PodLogOptions{}).Stream(ctx)
g.Expect(err).To(gomega.Succeed())
logs, err := io.ReadAll(reader)
g.Expect(err).To(gomega.Succeed())
return string(logs)
}).Should(gomega.ContainSubstring("New certificate found, reloading"))

ginkgo.By("Setting up invalid VPA object")
// there is an invalid "requests" field.
invalidVPA := []byte(`{
"kind": "VerticalPodAutoscaler",
"apiVersion": "autoscaling.k8s.io/v1",
"metadata": {"name": "cert-vpa-invalid"},
"spec": {
"targetRef": {
"apiVersion": "apps/v1",
"kind": "Deployment",
"name":"hamster"
},
"resourcePolicy": {
"containerPolicies": [{"containerName": "*", "minAllowed":{"requests":{"cpu":"50m"}}}]
}
}
}`)
err = InstallRawVPA(f, invalidVPA)
gomega.Expect(err).To(gomega.HaveOccurred(), "Invalid VPA object accepted")
gomega.Expect(err.Error()).To(gomega.MatchRegexp(`.*admission webhook .*vpa.* denied the request: .*`), "Admission controller did not inspect the object")
})

})

func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment) *apiv1.PodList {
Expand Down
1 change: 1 addition & 0 deletions vertical-pod-autoscaler/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module k8s.io/autoscaler/vertical-pod-autoscaler
go 1.21

require (
github.com/fsnotify/fsnotify v1.7.0
github.com/golang/mock v1.6.0
github.com/prometheus/client_golang v1.17.0
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16
Expand Down
2 changes: 2 additions & 0 deletions vertical-pod-autoscaler/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKf
github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
Expand Down
2 changes: 1 addition & 1 deletion vertical-pod-autoscaler/hack/deploy-for-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ gcloud auth configure-docker -q

for i in ${COMPONENTS}; do
if [ $i == admission-controller ] ; then
(cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh || true)
(cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true)
fi
ALL_ARCHITECTURES=amd64 make --directory ${SCRIPT_ROOT}/pkg/${i} release
done
Expand Down
70 changes: 60 additions & 10 deletions vertical-pod-autoscaler/pkg/admission-controller/certs.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,18 @@ limitations under the License.
package main

import (
"crypto/tls"
"os"
"path"
"sync"

"github.com/fsnotify/fsnotify"
"k8s.io/klog/v2"
)

type certsContainer struct {
caCert, serverKey, serverCert []byte
}

type certsConfig struct {
clientCaFile, tlsCertFile, tlsPrivateKey *string
reload *bool
}

func readFile(filePath string) []byte {
Expand All @@ -41,10 +42,59 @@ func readFile(filePath string) []byte {
return res
}

func initCerts(config certsConfig) certsContainer {
res := certsContainer{}
res.caCert = readFile(*config.clientCaFile)
res.serverCert = readFile(*config.tlsCertFile)
res.serverKey = readFile(*config.tlsPrivateKey)
return res
type certReloader struct {
tlsCertPath string
tlsKeyPath string
cert *tls.Certificate
mu sync.RWMutex
}

func (cr *certReloader) start(stop <-chan struct{}) error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}

if err = watcher.Add(path.Dir(cr.tlsCertPath)); err != nil {
return err
}
if err = watcher.Add(path.Dir(cr.tlsKeyPath)); err != nil {
return err
}
go func() {
defer watcher.Close()
for {
select {
case event := <-watcher.Events:
if event.Has(fsnotify.Create) || event.Has(fsnotify.Write) {
klog.V(2).Info("New certificate found, reloading")
if err := cr.load(); err != nil {
klog.Errorf("Failed to reload certificate: %s", err)
}
}
case err := <-watcher.Errors:
klog.Warningf("Error watching certificate files: %s", err)
case <-stop:
return
}
}
}()
return nil
}

func (cr *certReloader) load() error {
cert, err := tls.LoadX509KeyPair(cr.tlsCertPath, cr.tlsKeyPath)
if err != nil {
return err
}
cr.mu.Lock()
defer cr.mu.Unlock()
cr.cert = &cert
return nil
}

func (cr *certReloader) getCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
cr.mu.RLock()
defer cr.mu.RUnlock()
return cr.cert, nil
}
150 changes: 150 additions & 0 deletions vertical-pod-autoscaler/pkg/admission-controller/certs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"bytes"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net"
"os"
"path"
"testing"
"time"
)

func generateCerts(t *testing.T, org string, caCert *x509.Certificate, caKey *rsa.PrivateKey) ([]byte, []byte) {
cert := &x509.Certificate{
SerialNumber: big.NewInt(0),
Subject: pkix.Name{
Organization: []string{org},
},
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(1, 0, 0),
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
certKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
t.Error(err)
}
certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &certKey.PublicKey, caKey)
if err != nil {
t.Error(err)
}

var certPem bytes.Buffer
err = pem.Encode(&certPem, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
})
if err != nil {
t.Error(err)
}

var certKeyPem bytes.Buffer
err = pem.Encode(&certKeyPem, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(certKey),
})
if err != nil {
t.Error(err)
}
return certPem.Bytes(), certKeyPem.Bytes()
}

func TestKeypairReloader(t *testing.T) {
tempDir := t.TempDir()
caCert := &x509.Certificate{
SerialNumber: big.NewInt(0),
Subject: pkix.Name{
Organization: []string{"ca"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(2, 0, 0),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
caKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
t.Error(err)
}
caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caKey.PublicKey, caKey)
if err != nil {
t.Error(err)
}
caPath := path.Join(tempDir, "ca.crt")
caFile, err := os.Create(caPath)
if err != nil {
t.Error(err)
}
err = pem.Encode(caFile, &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
if err != nil {
t.Error(err)
}

pub, privateKey := generateCerts(t, "first", caCert, caKey)
certPath := path.Join(tempDir, "cert.crt")
if err = os.WriteFile(certPath, pub, 0666); err != nil {
t.Error(err)
}
keyPath := path.Join(tempDir, "cert.key")
if err = os.WriteFile(keyPath, privateKey, 0666); err != nil {
t.Error(err)
}

reloader := certReloader{
tlsCertPath: certPath,
tlsKeyPath: keyPath,
}
stop := make(chan struct{})
defer close(stop)
if err = reloader.start(stop); err != nil {
t.Error(err)
}

pub, privateKey = generateCerts(t, "second", caCert, caKey)
if err = os.WriteFile(certPath, pub, 0666); err != nil {
t.Error(err)
}
if err = os.WriteFile(keyPath, privateKey, 0666); err != nil {
t.Error(err)
}
for {
tlsCert, err := reloader.getCertificate(nil)
if err != nil {
t.Error(err)
}
if tlsCert == nil {
continue
}
pubDER, _ := pem.Decode(pub)
if string(tlsCert.Certificate[0]) == string(pubDER.Bytes) {
return
}
}
}
29 changes: 22 additions & 7 deletions vertical-pod-autoscaler/pkg/admission-controller/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,10 @@ const (
webhookConfigName = "vpa-webhook-config"
)

func configTLS(serverCert, serverKey []byte, minTlsVersion, ciphers string) *tls.Config {
func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struct{}) *tls.Config {
var tlsVersion uint16
var ciphersuites []uint16
reverseCipherMap := make(map[string]uint16)
sCert, err := tls.X509KeyPair(serverCert, serverKey)
if err != nil {
klog.Fatal(err)
}

for _, c := range tls.CipherSuites() {
reverseCipherMap[c.Name] = c.ID
Expand All @@ -66,11 +62,30 @@ func configTLS(serverCert, serverKey []byte, minTlsVersion, ciphers string) *tls
klog.Fatal(fmt.Errorf("Unable to determine value for --min-tls-version (%s), must be either tls1_2 or tls1_3", minTlsVersion))
}

return &tls.Config{
config := &tls.Config{
MinVersion: tlsVersion,
Certificates: []tls.Certificate{sCert},
CipherSuites: ciphersuites,
}
if *cfg.reload {
cr := certReloader{
tlsCertPath: *cfg.tlsCertFile,
tlsKeyPath: *cfg.tlsPrivateKey,
}
if err := cr.load(); err != nil {
klog.Fatal(err)
}
if err := cr.start(stop); err != nil {
klog.Fatal(err)
}
config.GetCertificate = cr.getCertificate
} else {
cert, err := tls.LoadX509KeyPair(*cfg.tlsCertFile, *cfg.tlsPrivateKey)
if err != nil {
klog.Fatal(err)
}
config.Certificates = []tls.Certificate{cert}
}
return config
}

// register this webhook admission controller with the kube-apiserver
Expand Down
Loading

0 comments on commit 22f8c84

Please sign in to comment.