Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add HostNetwork enabled e2e test case #450

Merged
merged 1 commit into from
Feb 11, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions test/e2e/specs/specs.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ const (
EnableFileCacheAndMetricsPrefix = "gcsfuse-csi-enable-file-cache-and-metrics"
EnableFileCacheWithLargeCapacityPrefix = "gcsfuse-csi-enable-file-cache-large-capacity"
EnableMetadataPrefetchPrefix = "gcsfuse-csi-enable-metadata-prefetch"
EnableHostNetworkPrefix = "gcsfuse-csi-enable-hostnetwork"
EnableCustomReadAhead = "gcsfuse-csi-enable-custom-read-ahead"
EnableMetadataPrefetchAndFakeVolumePrefix = "gcsfuse-csi-enable-metadata-prefetch-and-fake-volume"
EnableMetadataPrefetchPrefixForceNewBucketPrefix = "gcsfuse-csi-enable-metadata-prefetch-and-force-new-bucket"
Expand Down Expand Up @@ -177,6 +178,10 @@ func (t *TestPod) GetPodName() string {
return t.pod.Name
}

func (t *TestPod) GetPodVols() []corev1.Volume {
return t.pod.Spec.Volumes
}

// VerifyExecInPodSucceed verifies shell cmd in target pod succeed.
func (t *TestPod) VerifyExecInPodSucceed(f *framework.Framework, containerName, shExec string) {
_ = t.VerifyExecInPodSucceedWithOutput(f, containerName, shExec)
Expand Down Expand Up @@ -486,6 +491,10 @@ func (t *TestPod) SetImage(image string) {
t.pod.Spec.Containers[0].Image = image
}

func (t *TestPod) EnableHostNetwork() {
t.pod.Spec.HostNetwork = true
}

func (t *TestPod) SetResource(cpuLimit, memoryLimit, storageLimit string) {
cpu, _ := resource.ParseQuantity(cpuLimit)
mem, _ := resource.ParseQuantity(memoryLimit)
Expand Down
24 changes: 21 additions & 3 deletions test/e2e/testsuites/failed_mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,18 @@ func (t *gcsFuseCSIFailedMountTestSuite) SkipUnsupportedTests(_ storageframework
}

func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
envVar := os.Getenv(utils.TestWithNativeSidecarEnvVar)
supportsNativeSidecar, err := strconv.ParseBool(envVar)
nativeSidecarEnvVar := os.Getenv(utils.TestWithNativeSidecarEnvVar)
supportsNativeSidecar, err := strconv.ParseBool(nativeSidecarEnvVar)
if err != nil {
klog.Fatalf(`env variable "%s" could not be converted to boolean`, envVar)
klog.Fatalf(`env variable "%s" could not be converted to boolean`, nativeSidecarEnvVar)
}

saVolInjectEnvVar := os.Getenv(utils.TestWithSAVolumeInjectionEnvVar)
supportSAVolInjection, err := strconv.ParseBool(saVolInjectEnvVar)
if err != nil {
klog.Fatalf(`env variable "%s" could not be converted to boolean`, saVolInjectEnvVar)
}

type local struct {
config *storageframework.PerTestConfig
volumeResource *storageframework.VolumeResource
Expand Down Expand Up @@ -211,6 +218,9 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes

ginkgo.By("Configuring the pod")
tPod := specs.NewTestPod(f.ClientSet, f.Namespace)
if configPrefix == specs.EnableHostNetworkPrefix {
tPod.EnableHostNetwork()
}
tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false)

ginkgo.By("Deploying a Kubernetes service account without access to the GCS bucket")
Expand Down Expand Up @@ -254,6 +264,14 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes
testCaseSAInsufficientAccess(specs.SkipCSIBucketAccessCheckPrefix)
})

ginkgo.It("[hostnetwork] should fail when the specified service account does not have access to the GCS bucket", func() {
if supportSAVolInjection {
testCaseSAInsufficientAccess(specs.EnableHostNetworkPrefix)
} else {
ginkgo.By("Skipping the hostnetwork test for cluster version < 1.33.0")
}
})

ginkgo.It("should respond to service account permission changes", func() {
init()
defer cleanup()
Expand Down
63 changes: 63 additions & 0 deletions test/e2e/testsuites/mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@ package testsuites
import (
"context"
"fmt"
"os"
"strconv"

"github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/webhook"
"github.com/googlecloudplatform/gcs-fuse-csi-driver/test/e2e/specs"
"github.com/googlecloudplatform/gcs-fuse-csi-driver/test/e2e/utils"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"
"k8s.io/kubernetes/test/e2e/framework"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
Expand Down Expand Up @@ -57,6 +62,11 @@ func (t *gcsFuseCSIMountTestSuite) SkipUnsupportedTests(_ storageframework.TestD
}

func (t *gcsFuseCSIMountTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
envVar := os.Getenv(utils.TestWithSAVolumeInjectionEnvVar)
supportSAVolInjection, err := strconv.ParseBool(envVar)
if err != nil {
klog.Fatalf(`env variable "%s" could not be converted to boolean`, envVar)
}
type local struct {
config *storageframework.PerTestConfig
volumeResource *storageframework.VolumeResource
Expand Down Expand Up @@ -111,10 +121,63 @@ func (t *gcsFuseCSIMountTestSuite) DefineTests(driver storageframework.TestDrive
tPod1.Cleanup(ctx)
}

testCaseHostNetworkEnabled := func(configPrefix ...string) {
init(configPrefix...)
defer cleanup()

ginkgo.By("Configuring hostnetwork enabled pod")
tPod := specs.NewTestPod(f.ClientSet, f.Namespace)
tPod.EnableHostNetwork()
tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false)

ginkgo.By("Deploying hostnetwork enabled pod")
tPod.Create(ctx)

ginkgo.By("Checking pod is running")
tPod.WaitForRunning(ctx)

ginkgo.By("Checking that the pod command exits with no error")
tPod.VerifyExecInPodSucceedWithOutput(f, specs.TesterContainerName, fmt.Sprintf(`mountpoint -d "%s"`, mountPath))

ginkgo.By("Checking that the pod can access bucket")
// Create a new file B using gcsfuse.
testFile := "testfile"
tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("touch %v/%v", mountPath, testFile))

// Check mounted volumes on pod
projectedSAVolMounted := false
for _, vol := range tPod.GetPodVols() {
if vol.Name == webhook.SidecarContainerSATokenVolumeName {
projectedSAVolMounted = true

break
}
}
gomega.Expect(projectedSAVolMounted).To(gomega.BeTrue())

// Check the volume content.
volumeContents := tPod.VerifyExecInPodSucceedWithOutput(f, specs.TesterContainerName, fmt.Sprintf("ls %v", mountPath))
gomega.Expect(volumeContents).To(gomega.Equal(testFile))

ginkgo.By("Deleting pod")
tPod.Cleanup(ctx)
}

ginkgo.It("[read ahead config] should update read ahead config knobs", func() {
if pattern.VolType == storageframework.DynamicPV {
e2eskipper.Skipf("skip for volume type %v", storageframework.DynamicPV)
}
testCaseStoreAndRetainData(specs.EnableCustomReadAhead)
})

ginkgo.It("should successfully mount for hostnetwork enabled pods", func() {
siyanshen marked this conversation as resolved.
Show resolved Hide resolved
if pattern.VolType == storageframework.DynamicPV {
e2eskipper.Skipf("skip for volume type %v", storageframework.DynamicPV)
}
if supportSAVolInjection {
testCaseHostNetworkEnabled()
} else {
ginkgo.By("Skipping the hostnetwork test for cluster version < 1.33.0")
}
})
}
11 changes: 7 additions & 4 deletions test/e2e/utils/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ import (
"k8s.io/klog/v2"
)

var nativeSidecarMinimumVersion = version.MustParseGeneric("1.29.0")
var (
nativeSidecarMinimumVersion = version.MustParseGeneric("1.29.0")
saTokenVolInjectionMinimumVersion = version.MustParseGeneric("1.33.0")
)

func clusterDownGKE(testParams *TestParameters) error {
//nolint:gosec
Expand Down Expand Up @@ -133,22 +136,22 @@ func clusterUpGKE(testParams *TestParameters) error {
return nil
}

func ClusterSupportsNativeSidecar(clusterVersion, nodeVersion string) (bool, error) {
func ClusterAtLeastMinVersion(clusterVersion, nodeVersion string, minVersion *version.Version) (bool, error) {
supportsNativeSidecar := false
if clusterVersion != "" {
parsedClusterVersion, err := version.ParseGeneric(clusterVersion)
if err != nil {
return false, err
}
if parsedClusterVersion.AtLeast(nativeSidecarMinimumVersion) {
if parsedClusterVersion.AtLeast(minVersion) {
supportsNativeSidecar = true

if nodeVersion != "" {
parsedNodeVersion, err := version.ParseGeneric(nodeVersion)
if err != nil {
return false, err
}
if !parsedNodeVersion.AtLeast(nativeSidecarMinimumVersion) {
if !parsedNodeVersion.AtLeast(minVersion) {
supportsNativeSidecar = false
}
}
Expand Down
18 changes: 16 additions & 2 deletions test/e2e/utils/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,15 @@ type TestParameters struct {
GinkgoSkipGcpSaTest bool

SupportsNativeSidecar bool
SupportSAVolInjection bool
IstioVersion string
GcsfuseClientProtocol string
}

const TestWithNativeSidecarEnvVar = "TEST_WITH_NATIVE_SIDECAR"
const (
TestWithNativeSidecarEnvVar = "TEST_WITH_NATIVE_SIDECAR"
TestWithSAVolumeInjectionEnvVar = "TEST_WITH_SA_VOL_INJECTION"
)

func Handle(testParams *TestParameters) error {
oldMask := syscall.Umask(0o000)
Expand Down Expand Up @@ -165,7 +169,7 @@ func Handle(testParams *TestParameters) error {
testFocusStr = fmt.Sprintf(".*%s.*", testFocusStr)
}

supportsNativeSidecar, err := ClusterSupportsNativeSidecar(testParams.GkeClusterVersion, testParams.GkeNodeVersion)
supportsNativeSidecar, err := ClusterAtLeastMinVersion(testParams.GkeClusterVersion, testParams.GkeNodeVersion, nativeSidecarMinimumVersion)
if err != nil {
klog.Fatalf(`native sidecar support could not be determined: %v`, err)
}
Expand All @@ -175,6 +179,16 @@ func Handle(testParams *TestParameters) error {
klog.Fatalf(`env variable "%s" could not be set: %v`, TestWithNativeSidecarEnvVar, err)
}

supportSAVolInjection, err := ClusterAtLeastMinVersion(testParams.GkeClusterVersion, testParams.GkeNodeVersion, saTokenVolInjectionMinimumVersion)
if err != nil {
klog.Fatalf(`SA Vol Injection support could not be determined: %v`, err)
}
testParams.SupportSAVolInjection = supportSAVolInjection

if err = os.Setenv(TestWithSAVolumeInjectionEnvVar, strconv.FormatBool(supportSAVolInjection)); err != nil {
klog.Fatalf(`env variable "%s" could not be set: %v`, TestWithSAVolumeInjectionEnvVar, err)
}

testSkipStr := generateTestSkip(testParams)
if !strings.Contains(testSkipStr, "istio") && (len(testFocusStr) == 0 || strings.Contains(testFocusStr, "istio")) {
installIstio(testParams.IstioVersion)
Expand Down