Skip to content

Commit bc35cba

Browse files
wangke19wangke19
committed
Reapply "OCPBUGS-64799: TLS 1.3 / Modern profile tests (#29611)"
This reverts commit 10dab7a.
1 parent d25eb09 commit bc35cba

File tree

1 file changed

+200
-26
lines changed

1 file changed

+200
-26
lines changed

‎test/extended/apiserver/tls.go‎

Lines changed: 200 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,238 @@
11
package apiserver
22

33
import (
4+
"context"
45
"crypto/tls"
6+
"fmt"
7+
"io"
8+
"math/rand"
9+
"os/exec"
10+
"strings"
11+
"time"
512

613
g "github.com/onsi/ginkgo/v2"
14+
o "github.com/onsi/gomega"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
e2e "k8s.io/kubernetes/test/e2e/framework"
717

18+
configv1 "github.com/openshift/api/config/v1"
819
"github.com/openshift/library-go/pkg/crypto"
20+
"github.com/openshift/origin/test/extended/networking"
921
exutil "github.com/openshift/origin/test/extended/util"
1022
)
1123

24+
const (
25+
namespace = "apiserver-tls-test"
26+
)
27+
28+
// This test only checks whether components are serving the proper TLS version based
29+
// on the expected version set in the TLS profile config. It is a part of the
30+
// openshift/conformance/parallel test suite, and it is expected that there are jobs
31+
// which run that entire conformance suite against clusters running any TLS profiles
32+
// that there is a desire to test.
1233
var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() {
1334
defer g.GinkgoRecover()
1435

15-
oc := exutil.NewCLI("apiserver")
36+
var oc = exutil.NewCLI(namespace)
37+
var ctx = context.Background()
1638

17-
g.It("TestTLSDefaults", func() {
18-
g.Skip("skipping because it was broken in master")
39+
g.BeforeEach(func() {
40+
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
41+
o.Expect(err).NotTo(o.HaveOccurred())
42+
43+
isHyperShift, err := exutil.IsHypershift(ctx, oc.AdminConfigClient())
44+
o.Expect(err).NotTo(o.HaveOccurred())
45+
46+
if isMicroShift || isHyperShift {
47+
g.Skip("TLS configuration for the apiserver resource is not applicable to MicroShift or HyperShift clusters - skipping")
48+
}
49+
50+
hasIPv4, _, err := networking.GetIPAddressFamily(oc)
51+
o.Expect(err).NotTo(o.HaveOccurred())
52+
if !hasIPv4 {
53+
g.Skip("TLS configuration is only tested on IPv4 clusters, skipping")
54+
}
55+
})
56+
57+
g.It("TestTLSMinimumVersions", func() {
58+
59+
g.By("Getting the APIServer configuration")
60+
config, err := oc.AdminConfigClient().ConfigV1().APIServers().Get(ctx, "cluster", metav1.GetOptions{})
61+
o.Expect(err).NotTo(o.HaveOccurred())
62+
63+
g.By("Determining expected TLS behavior based on the cluster's TLS profile")
64+
var tlsShouldWork, tlsShouldNotWork *tls.Config
65+
switch {
66+
case config.Spec.TLSSecurityProfile == nil,
67+
config.Spec.TLSSecurityProfile.Type == configv1.TLSProfileIntermediateType:
68+
tlsShouldWork = &tls.Config{MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS13, InsecureSkipVerify: true}
69+
tlsShouldNotWork = &tls.Config{MinVersion: tls.VersionTLS11, MaxVersion: tls.VersionTLS11, InsecureSkipVerify: true}
70+
g.By("Using intermediate TLS profile: connections with TLS ≥1.2 should work, <1.2 should fail")
71+
case config.Spec.TLSSecurityProfile.Type == configv1.TLSProfileModernType:
72+
tlsShouldWork = &tls.Config{MinVersion: tls.VersionTLS13, MaxVersion: tls.VersionTLS13, InsecureSkipVerify: true}
73+
tlsShouldNotWork = &tls.Config{MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12, InsecureSkipVerify: true}
74+
g.By("Using modern TLS profile: only TLS 1.3 connections should succeed")
75+
default:
76+
g.Skip("Only intermediate or modern profiles are tested")
77+
}
78+
79+
targets := []struct {
80+
name, namespace, port string
81+
}{
82+
{"apiserver", "openshift-kube-apiserver", "443"},
83+
{"oauth-openshift", "openshift-authentication", "443"},
84+
{"kube-controller-manager", "openshift-kube-controller-manager", "443"},
85+
{"scheduler", "openshift-kube-scheduler", "443"},
86+
{"api", "openshift-apiserver", "443"},
87+
{"api", "openshift-oauth-apiserver", "443"},
88+
{"machine-config-controller", "openshift-machine-config-operator", "9001"},
89+
}
90+
91+
g.By("Verifying TLS behavior for core control plane components")
92+
for _, target := range targets {
93+
g.By(fmt.Sprintf("Checking %s/%s on port %s", target.namespace, target.name, target.port))
94+
err = forwardPortAndExecute(target.name, target.namespace, target.port,
95+
func(port int) error { return checkTLSConnection(port, tlsShouldWork, tlsShouldNotWork) })
96+
o.Expect(err).NotTo(o.HaveOccurred())
97+
}
98+
99+
g.By("Checking etcd's TLS behavior")
100+
err = forwardPortAndExecute("etcd", "openshift-etcd", "2379", func(port int) error {
101+
conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork)
102+
if err != nil {
103+
if !strings.Contains(err.Error(), "remote error: tls: bad certificate") {
104+
return fmt.Errorf("should work: %w", err)
105+
}
106+
} else {
107+
err = conn.Close()
108+
if err != nil {
109+
return fmt.Errorf("failed to close connection: %w", err)
110+
}
111+
}
112+
conn, err = tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldNotWork)
113+
if err == nil {
114+
return fmt.Errorf("should not work: connection unexpectedly succeeded, closing conn status: %v", conn.Close())
115+
}
116+
return nil
117+
})
118+
o.Expect(err).NotTo(o.HaveOccurred())
119+
})
19120

121+
g.It("TestTLSDefaults", func() {
20122
t := g.GinkgoT()
21-
// Verify we fail with TLS versions less than the default, and work with TLS versions >= the default
123+
124+
_, err := e2e.LoadClientset(true)
125+
o.Expect(err).NotTo(o.HaveOccurred())
126+
127+
g.By("Getting the APIServer config")
128+
config, err := oc.AdminConfigClient().ConfigV1().APIServers().Get(ctx, "cluster", metav1.GetOptions{})
129+
o.Expect(err).NotTo(o.HaveOccurred())
130+
131+
if config.Spec.TLSSecurityProfile != nil &&
132+
config.Spec.TLSSecurityProfile.Type != configv1.TLSProfileIntermediateType {
133+
g.Skip("Cluster TLS profile is not default (intermediate), skipping cipher defaults check")
134+
}
135+
136+
g.By("Verifying TLS version behavior")
22137
for _, tlsVersionName := range crypto.ValidTLSVersions() {
23138
tlsVersion := crypto.TLSVersionOrDie(tlsVersionName)
24139
expectSuccess := tlsVersion >= crypto.DefaultTLSVersion()
25-
config := &tls.Config{MinVersion: tlsVersion, MaxVersion: tlsVersion, InsecureSkipVerify: true}
140+
cfg := &tls.Config{MinVersion: tlsVersion, MaxVersion: tlsVersion, InsecureSkipVerify: true}
141+
host := strings.TrimPrefix(oc.AdminConfig().Host, "https://")
26142

27-
{
28-
conn, err := tls.Dial("tcp4", oc.AdminConfig().Host, config)
29-
if err == nil {
30-
conn.Close()
31-
}
32-
if success := err == nil; success != expectSuccess {
33-
t.Errorf("Expected success %v, got %v with TLS version %s dialing master", expectSuccess, success, tlsVersionName)
143+
conn, err := tls.Dial("tcp", host, cfg)
144+
if err == nil {
145+
err := conn.Close()
146+
if err != nil {
147+
t.Errorf("Failed to close connection: %v", err)
34148
}
35149
}
150+
if success := err == nil; success != expectSuccess {
151+
t.Errorf("Expected success %v, got %v with TLS version %s dialing master", expectSuccess, success, tlsVersionName)
152+
}
36153
}
37154

38-
// Verify the only ciphers we work with are in the default set.
39-
// Not all default ciphers will succeed because they depend on the serving cert type.
155+
g.By("Verifying cipher suites")
40156
defaultCiphers := map[uint16]bool{}
41-
for _, defaultCipher := range crypto.DefaultCiphers() {
42-
defaultCiphers[defaultCipher] = true
157+
for _, c := range crypto.DefaultCiphers() {
158+
defaultCiphers[c] = true
43159
}
160+
44161
for _, cipherName := range crypto.ValidCipherSuites() {
45162
cipher, err := crypto.CipherSuite(cipherName)
46163
if err != nil {
47164
t.Fatal(err)
48165
}
49166
expectFailure := !defaultCiphers[cipher]
50-
config := &tls.Config{CipherSuites: []uint16{cipher}, InsecureSkipVerify: true}
51-
52-
{
53-
conn, err := tls.Dial("tcp4", oc.AdminConfig().Host, config)
54-
if err == nil {
55-
conn.Close()
56-
if expectFailure {
57-
t.Errorf("Expected failure on cipher %s, got success dialing master", cipherName)
58-
}
167+
cfg := &tls.Config{CipherSuites: []uint16{cipher}, InsecureSkipVerify: true}
168+
169+
conn, err := tls.Dial("tcp", oc.AdminConfig().Host, cfg)
170+
if err == nil {
171+
if expectFailure {
172+
t.Errorf("Expected failure on cipher %s, got success dialing master. Closing conn: %v", cipherName, conn.Close())
59173
}
60174
}
61175
}
62-
63176
})
64177
})
178+
179+
func forwardPortAndExecute(serviceName, namespace, remotePort string, toExecute func(localPort int) error) error {
180+
var err error
181+
for i := 0; i < 3; i++ {
182+
if err = func() error {
183+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
184+
defer cancel()
185+
localPort := rand.Intn(65534-1025) + 1025
186+
args := []string{"port-forward", fmt.Sprintf("svc/%s", serviceName), fmt.Sprintf("%d:%s", localPort, remotePort), "-n", namespace}
187+
188+
cmd := exec.CommandContext(ctx, "oc", args...)
189+
stdout, stderr, err := e2e.StartCmdAndStreamOutput(cmd)
190+
if err != nil {
191+
return err
192+
}
193+
defer stdout.Close()
194+
defer stderr.Close()
195+
defer e2e.TryKill(cmd)
196+
197+
e2e.Logf("oc port-forward output: %s", readPartialFrom(stdout, 1024))
198+
return toExecute(localPort)
199+
}(); err == nil {
200+
return nil
201+
} else {
202+
e2e.Logf("failed to start oc port-forward command or test: %v", err)
203+
time.Sleep(2 * time.Second)
204+
}
205+
}
206+
return err
207+
}
208+
209+
func readPartialFrom(r io.Reader, maxBytes int) string {
210+
buf := make([]byte, maxBytes)
211+
n, err := r.Read(buf)
212+
if err != nil && err != io.EOF {
213+
return fmt.Sprintf("error reading: %v", err)
214+
}
215+
return string(buf[:n])
216+
}
217+
218+
func checkTLSConnection(port int, tlsShouldWork, tlsShouldNotWork *tls.Config) error {
219+
conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldWork)
220+
if err != nil {
221+
return fmt.Errorf("should work: %w", err)
222+
}
223+
err = conn.Close()
224+
if err != nil {
225+
return fmt.Errorf("failed to close connection: %w", err)
226+
}
227+
228+
conn, err = tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), tlsShouldNotWork)
229+
if err == nil {
230+
return fmt.Errorf("should not work: connection unexpectedly succeeded, closing conn status: %v", conn.Close())
231+
}
232+
if !strings.Contains(err.Error(), "protocol version") &&
233+
!strings.Contains(err.Error(), "no supported versions satisfy") &&
234+
!strings.Contains(err.Error(), "handshake failure") {
235+
return fmt.Errorf("should not work: got error, but not a TLS version mismatch: %w", err)
236+
}
237+
return nil
238+
}

0 commit comments

Comments
 (0)