From 28e25f900908cd773ac12155b24e65d5504adccb Mon Sep 17 00:00:00 2001 From: zoetrope Date: Mon, 3 Jun 2024 10:44:31 +0000 Subject: [PATCH] Add test code --- Dockerfile | 4 +- Makefile | 37 +++++----------- config/rbac/role.yaml | 4 +- internal/controller/metrics.go | 2 +- internal/controller/pdb_controller.go | 21 ++------- internal/tty-exporter/metrics.go | 2 +- test/e2e/e2e_test.go | 38 ++++------------ test/testdata/statefulset.yaml | 64 +++++++++++++++++++++++++++ 8 files changed, 96 insertions(+), 76 deletions(-) create mode 100644 test/testdata/statefulset.yaml diff --git a/Dockerfile b/Dockerfile index 4564434..431cb78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,9 @@ RUN go mod download # Copy the go source COPY cmd/ cmd/ -COPY internal/controller/ internal/controller/ +COPY internal/ internal/ -RUN CGO_ENABLED=0 go install -ldflags="-w -s" ./... +RUN CGO_ENABLED=0 go install -ldflags="-w -s" ./cmd/... # Build the tty-exporter binary FROM scratch AS tty-exporter diff --git a/Makefile b/Makefile index 5acc53a..f0fabd4 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ # Image URL to use all building/pushing image targets -IMG ?= cybozu-go/login-protector:dev +PROTECTOR_IMG ?= cybozu-go/login-protector:dev +EXPORTER_IMG ?= cybozu-go/tty-exporter:dev # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -55,7 +56,7 @@ vet: ## Run go vet against code. # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: +test-e2e: start-kind load-image deploy go test ./test/e2e/ -v -ginkgo.v .PHONY: lint @@ -81,38 +82,24 @@ run: manifests fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build -t ${PROTECTOR_IMG} . --target=login-protector + $(CONTAINER_TOOL) build -t ${EXPORTER_IMG} . --target=tty-exporter .PHONY: docker-push docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} + $(CONTAINER_TOOL) push ${PROTECTOR_IMG} + $(CONTAINER_TOOL) push ${EXPORTER_IMG} .PHONY: load-image load-image: docker-build setup - kind load docker-image ${IMG} - - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name project-v3-builder - $(CONTAINER_TOOL) buildx use project-v3-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm project-v3-builder - rm Dockerfile.cross + kind load docker-image ${PROTECTOR_IMG} + kind load docker-image ${EXPORTER_IMG} + .PHONY: build-installer build-installer: manifests setup ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist - cd config/manager && kustomize edit set image controller=${IMG} + cd config/manager && kustomize edit set image controller=${PROTECTOR_IMG} kustomize build config/default > dist/install.yaml ##@ Deployment @@ -131,7 +118,7 @@ stop-kind: setup .PHONY: deploy deploy: manifests setup ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && kustomize edit set image controller=${PROTECTOR_IMG} kustomize build config/default | kubectl apply -f - kubectl -n login-protector-system wait --for=condition=available --timeout=180s --all deployments diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8d05c0c..2117541 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -37,8 +37,10 @@ rules: - apiGroups: - policy resources: - - poddisruptionbudget + - poddisruptionbudgets verbs: - create - delete - get + - list + - watch diff --git a/internal/controller/metrics.go b/internal/controller/metrics.go index abe2762..789d13a 100644 --- a/internal/controller/metrics.go +++ b/internal/controller/metrics.go @@ -4,7 +4,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const metricsNamespace = "ttypdb_controller" +const metricsNamespace = "login_protector" var metricsPollingSkipsCounter = prometheus.NewCounter( prometheus.CounterOpts{ diff --git a/internal/controller/pdb_controller.go b/internal/controller/pdb_controller.go index 43f864c..ff07162 100644 --- a/internal/controller/pdb_controller.go +++ b/internal/controller/pdb_controller.go @@ -7,12 +7,10 @@ import ( "fmt" "io" "net/http" - "sync" "time" "github.com/cybozu-go/login-protector/internal/common" "github.com/go-logr/logr" - "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" @@ -38,29 +36,18 @@ func NewPDBController(client client.Client, logger logr.Logger, interval time.Du //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch -//+kubebuilder:rbac:groups=policy,resources=poddisruptionbudget,verbs=create;get;delete +//+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=create;get;list;watch;delete func (w *PDBController) Start(ctx context.Context) error { ticker := time.NewTicker(w.interval) defer ticker.Stop() - m := sync.Mutex{} - for { select { case <-ctx.Done(): - m.Lock() // wait for doPollPods completion return nil case <-ticker.C: - if m.TryLock() { - go func() { - defer m.Unlock() - w.pollPods(ctx) - }() - } else { - metricsPollingSkipsCounter.Inc() - w.logger.Info(fmt.Sprintf("The previous polling takes more than %d seconds. Skip polling this time.", w.interval)) - } + w.pollPods(ctx) } } } @@ -71,7 +58,7 @@ func (w *PDBController) pollPods(ctx context.Context) { defer func() { duration := time.Since(startTime) metricsPollingDurationSecondsHistogram.Observe(duration.Seconds()) - w.logger.Info("polling completed", zap.Duration("duration", duration.Round(time.Millisecond))) + w.logger.Info("polling completed", "duration", duration.Round(time.Millisecond)) }() var stsList appsv1.StatefulSetList @@ -111,7 +98,7 @@ func (w *PDBController) reconcilePDB(ctx context.Context, pod *corev1.Pod) { var container *corev1.Container for _, c := range pod.Spec.Containers { - if c.Name == "ttypdb-sidecar" { + if c.Name == "tty-exporter" { container = &c break } diff --git a/internal/tty-exporter/metrics.go b/internal/tty-exporter/metrics.go index 00598b0..2b5c8d4 100644 --- a/internal/tty-exporter/metrics.go +++ b/internal/tty-exporter/metrics.go @@ -7,7 +7,7 @@ import ( "go.uber.org/zap" ) -const metricsNamespace = "tty-exporter" +const metricsNamespace = "tty_exporter" func InitMetrics(logger *zap.Logger) { prometheus.MustRegister(prometheus.NewGaugeFunc( diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 51e9368..e13988f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -15,44 +15,17 @@ const namespace = "login-protector-system" var _ = Describe("controller", Ordered, func() { BeforeAll(func() { - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) }) AfterAll(func() { - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) }) Context("Operator", func() { It("should run successfully", func() { - var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "cybozu-go/login-protector:dev" - - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { // Get pod name - - cmd = exec.Command("kubectl", "get", + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ @@ -67,7 +40,7 @@ var _ = Describe("controller", Ordered, func() { if len(podNames) != 1 { return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) } - controllerPodName = podNames[0] + controllerPodName := podNames[0] ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) // Validate pod status @@ -85,5 +58,12 @@ var _ = Describe("controller", Ordered, func() { EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) }) + + It("should deploy target statefulset", func() { + By("deploying target statefulset") + cmd := exec.Command("kubectl", "apply", "-f", "./test/testdata/statefulset.yaml") + _, err := utils.Run(cmd) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + }) }) }) diff --git a/test/testdata/statefulset.yaml b/test/testdata/statefulset.yaml new file mode 100644 index 0000000..7e62963 --- /dev/null +++ b/test/testdata/statefulset.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: teststs + labels: + login-protector.cybozu.io/update-strategy: "evict" +spec: + replicas: 2 + selector: + matchLabels: + name: teststs + serviceName: teststs + template: + metadata: + labels: + name: teststs + foo: bar + hoge: fuga + spec: + containers: + - name: main + image: ghcr.io/cybozu/ubuntu:22.04 + imagePullPolicy: IfNotPresent + command: [ "sleep", "infinity" ] + - name: tty-exporter + image: cybozu-go/tty-exporter:dev + imagePullPolicy: Never + ports: + - name: sidecar + containerPort: 8080 + shareProcessNamespace: true +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: teststs2 + labels: + login-protector.cybozu.io/update-strategy: "evict" +spec: + replicas: 1 + selector: + matchLabels: + name: teststs2 + serviceName: teststs2 + template: + metadata: + labels: + name: teststs2 + foo: baz + hoge: piyo + spec: + containers: + - name: main + image: ghcr.io/cybozu/ubuntu:22.04 + imagePullPolicy: IfNotPresent + command: [ "sleep", "infinity" ] + - name: tty-exporter + image: cybozu-go/tty-exporter:dev + imagePullPolicy: Never + ports: + - name: sidecar + containerPort: 8080 + shareProcessNamespace: true