From 78bc7b0a07536f92e0c4184f2dc622505becd757 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Tue, 31 Mar 2020 14:58:10 +0300 Subject: [PATCH 01/45] add .vscode to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index bc282140a1..eacf759278 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ vendor/ # .image-* files used by Makefile .image-* + +# VSCode +.vscode/ From 1e8f127f53e9ece398f7f33b035a16356b420d7b Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Tue, 31 Mar 2020 15:01:06 +0300 Subject: [PATCH 02/45] migrate to c2devel repository --- .github/workflows/release.yaml | 26 ------------------- Dockerfile | 12 ++------- Makefile | 4 +-- charts/aws-ebs-csi-driver/Chart.yaml | 6 ++--- cmd/main.go | 4 +-- cmd/options.go | 4 +-- cmd/options/server_options.go | 2 +- cmd/options_test.go | 2 +- .../kubernetes/dynamic-provisioning/README.md | 2 +- go.mod | 2 +- go.sum | 4 +++ hack/release | 2 +- pkg/cloud/cloud.go | 4 +-- pkg/cloud/cloud_test.go | 4 +-- pkg/cloud/metadata_ec2.go | 2 +- pkg/driver/controller.go | 8 +++--- pkg/driver/controller_modify_volume.go | 2 +- pkg/driver/controller_test.go | 6 ++--- pkg/driver/driver.go | 2 +- pkg/driver/mount.go | 2 +- pkg/driver/node.go | 6 ++--- pkg/driver/node_linux.go | 2 +- pkg/driver/node_linux_test.go | 4 +-- pkg/driver/node_test.go | 4 +-- pkg/driver/sanity_test.go | 6 ++--- pkg/driver/validation.go | 2 +- pkg/driver/validation_test.go | 2 +- tests/e2e/driver/ebs_csi_driver.go | 2 +- tests/e2e/dynamic_provisioning.go | 8 +++--- tests/e2e/pre_provsioning.go | 8 +++--- ...namically_provisioned_cmd_volume_tester.go | 2 +- ...cally_provisioned_collocated_pod_tester.go | 2 +- ...namically_provisioned_delete_pod_tester.go | 2 +- ...lly_provisioned_read_only_volume_tester.go | 2 +- ...cally_provisioned_reclaim_policy_tester.go | 4 +-- ...ically_provisioned_resize_volume_tester.go | 4 +-- ...rovisioned_topology_aware_volume_tester.go | 2 +- ...ally_provisioned_volume_snapshot_tester.go | 2 +- ...pre_provisioned_read_only_volume_tester.go | 2 +- .../pre_provisioned_reclaim_policy_tester.go | 2 +- .../pre_provisioned_snapshot_volume_tester.go | 2 +- .../pre_provisioned_volume_tester.go | 2 +- tests/e2e/testsuites/specs.go | 2 +- tests/e2e/testsuites/testsuites.go | 2 +- tests/integration/setup_test.go | 6 ++--- 45 files changed, 76 insertions(+), 106 deletions(-) delete mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 0940606326..0000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Release -on: - push: - # Sequence of patterns matched against refs/tags - tags: - - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 -jobs: - build: - name: Release - runs-on: ubuntu-latest - steps: - - name: Create Release - id: create-release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: ${{ github.ref }} - body: | - AWS EBS CSI Driver - - ## CHANGELOG - See [CHANGELOG](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/CHANGELOG.md) for full list of changes - draft: true - prerelease: false diff --git a/Dockerfile b/Dockerfile index 0d16c8af2f..1f44c1ac47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ # https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope # for info on BUILDPLATFORM, TARGETOS, TARGETARCH, etc. FROM --platform=$BUILDPLATFORM golang:1.20 AS builder -WORKDIR /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver +WORKDIR /go/src/github.com/c2devel/aws-ebs-csi-driver COPY go.* . ARG GOPROXY RUN go mod download @@ -27,13 +27,5 @@ ARG VERSION RUN OS=$TARGETOS ARCH=$TARGETARCH make $TARGETOS/$TARGETARCH FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest.2 AS linux-amazon -COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver +COPY --from=builder /go/src/github.com/c2devel/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver ENTRYPOINT ["/bin/aws-ebs-csi-driver"] - -FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-windows-base:1809 AS windows-ltsc2019 -COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver.exe /aws-ebs-csi-driver.exe -ENTRYPOINT ["/aws-ebs-csi-driver.exe"] - -FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-windows-base:ltsc2022 AS windows-ltsc2022 -COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver.exe /aws-ebs-csi-driver.exe -ENTRYPOINT ["/aws-ebs-csi-driver.exe"] diff --git a/Makefile b/Makefile index 312b3609bf..e4baad7eb0 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ VERSION?=v1.20.0 -PKG=github.com/kubernetes-sigs/aws-ebs-csi-driver +PKG=github.com/c2devel/aws-ebs-csi-driver GIT_COMMIT?=$(shell git rev-parse HEAD) BUILD_DATE?=$(shell date -u -Iseconds) @@ -25,7 +25,7 @@ GOPATH=$(shell go env GOPATH) GOOS=$(shell go env GOOS) GOBIN=$(shell pwd)/bin -REGISTRY?=gcr.io/k8s-staging-provider-aws +REGISTRY?=registry.cloud.croc.ru/kaas IMAGE?=$(REGISTRY)/aws-ebs-csi-driver TAG?=$(GIT_COMMIT) diff --git a/charts/aws-ebs-csi-driver/Chart.yaml b/charts/aws-ebs-csi-driver/Chart.yaml index 63a5e30d42..0f4740e2c8 100644 --- a/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/aws-ebs-csi-driver/Chart.yaml @@ -4,13 +4,13 @@ name: aws-ebs-csi-driver description: A Helm chart for AWS EBS CSI Driver version: 2.20.0 kubeVersion: ">=1.17.0-0" -home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver +home: https://github.com/c2devel/aws-ebs-csi-driver sources: - - https://github.com/kubernetes-sigs/aws-ebs-csi-driver + - https://github.com/c2devel/aws-ebs-csi-driver keywords: - aws - ebs - csi maintainers: - name: Kubernetes Authors - url: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/ + url: https://github.com/c2devel/aws-ebs-csi-driver/ diff --git a/cmd/main.go b/cmd/main.go index d8cf6fe843..75437151a6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -21,8 +21,8 @@ import ( flag "github.com/spf13/pflag" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" logsapi "k8s.io/component-base/logs/api/v1" json "k8s.io/component-base/logs/json" "k8s.io/component-base/metrics/legacyregistry" diff --git a/cmd/options.go b/cmd/options.go index 370ff1e1a2..08bc684b98 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -23,8 +23,8 @@ import ( flag "github.com/spf13/pflag" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/cmd/options" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/c2devel/aws-ebs-csi-driver/cmd/options" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" "k8s.io/component-base/featuregate" logsapi "k8s.io/component-base/logs/api/v1" diff --git a/cmd/options/server_options.go b/cmd/options/server_options.go index ddfbdaec72..da291f41ff 100644 --- a/cmd/options/server_options.go +++ b/cmd/options/server_options.go @@ -19,7 +19,7 @@ package options import ( flag "github.com/spf13/pflag" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" ) // ServerOptions contains options and configuration settings for the driver server. diff --git a/cmd/options_test.go b/cmd/options_test.go index 783b6ac0bf..89372bfe9b 100644 --- a/cmd/options_test.go +++ b/cmd/options_test.go @@ -24,7 +24,7 @@ import ( flag "github.com/spf13/pflag" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" ) func TestGetOptions(t *testing.T) { diff --git a/examples/kubernetes/dynamic-provisioning/README.md b/examples/kubernetes/dynamic-provisioning/README.md index 8884d97651..1a5a03dc1e 100644 --- a/examples/kubernetes/dynamic-provisioning/README.md +++ b/examples/kubernetes/dynamic-provisioning/README.md @@ -3,7 +3,7 @@ ## Prerequisites 1. Kubernetes 1.13+ (CSI 1.0). -2. The [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) installed. +2. The [aws-ebs-csi-driver](https://github.com/c2devel/aws-ebs-csi-driver) installed. ## Usage diff --git a/go.mod b/go.mod index d9e17d7bde..43bdc92227 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/kubernetes-sigs/aws-ebs-csi-driver +module github.com/c2devel/aws-ebs-csi-driver require ( github.com/aws/aws-sdk-go v1.44.283 diff --git a/go.sum b/go.sum index c1e6e15a9c..e3694da6f3 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,7 @@ github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -374,6 +375,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -642,6 +644,7 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -725,6 +728,7 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/hack/release b/hack/release index 26ffa1d0f6..27b2090fbb 100755 --- a/hack/release +++ b/hack/release @@ -115,7 +115,7 @@ def print_notes(args): if __name__=="__main__": parser = argparse.ArgumentParser(description='Generate release CHANGELOG') - parser.add_argument('--repo', metavar='repo', type=str, default='kubernetes-sigs/aws-ebs-csi-driver', help='the full github repository name') + parser.add_argument('--repo', metavar='repo', type=str, default='c2devel/aws-ebs-csi-driver', help='the full github repository name') parser.add_argument('--github-user', metavar='user', type=str, help='the github user for github api') parser.add_argument('--github-token', metavar='token', type=str, help='the github token for github api') diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 70ee16a740..8a8457d751 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -33,8 +33,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - dm "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud/devicemanager" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + dm "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud/devicemanager" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index 99860fced1..90608214c6 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -29,9 +29,9 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + dm "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud/devicemanager" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "github.com/golang/mock/gomock" - dm "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud/devicemanager" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" "github.com/stretchr/testify/assert" ) diff --git a/pkg/cloud/metadata_ec2.go b/pkg/cloud/metadata_ec2.go index d3841997d8..a9412b13df 100644 --- a/pkg/cloud/metadata_ec2.go +++ b/pkg/cloud/metadata_ec2.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "k8s.io/klog/v2" ) diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 7104a83c38..5fa10c7b0f 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -27,10 +27,10 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/awslabs/volume-modifier-for-k8s/pkg/rpc" csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util/template" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util/template" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/pkg/driver/controller_modify_volume.go b/pkg/driver/controller_modify_volume.go index ba2db7f561..16781498ed 100644 --- a/pkg/driver/controller_modify_volume.go +++ b/pkg/driver/controller_modify_volume.go @@ -5,7 +5,7 @@ import ( "strconv" "github.com/awslabs/volume-modifier-for-k8s/pkg/rpc" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go index 8759d257ac..b127780bb0 100644 --- a/pkg/driver/controller_test.go +++ b/pkg/driver/controller_test.go @@ -30,11 +30,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/mock/gomock" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" "github.com/stretchr/testify/assert" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 394554386c..64cfec65e0 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -23,7 +23,7 @@ import ( "github.com/awslabs/volume-modifier-for-k8s/pkg/rpc" csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "google.golang.org/grpc" "k8s.io/klog/v2" ) diff --git a/pkg/driver/mount.go b/pkg/driver/mount.go index 5c44db5c49..f822ecbf52 100644 --- a/pkg/driver/mount.go +++ b/pkg/driver/mount.go @@ -20,7 +20,7 @@ import ( "os" "path/filepath" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" + "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" mountutils "k8s.io/mount-utils" ) diff --git a/pkg/driver/node.go b/pkg/driver/node.go index c65d8f394f..443ca2ec92 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -25,10 +25,10 @@ import ( "strconv" "strings" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" diff --git a/pkg/driver/node_linux.go b/pkg/driver/node_linux.go index a61486b297..3e4c48c430 100644 --- a/pkg/driver/node_linux.go +++ b/pkg/driver/node_linux.go @@ -26,7 +26,7 @@ import ( "strconv" "strings" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "golang.org/x/sys/unix" "k8s.io/klog/v2" ) diff --git a/pkg/driver/node_linux_test.go b/pkg/driver/node_linux_test.go index 8a7c780f8b..3bd8fa04b1 100644 --- a/pkg/driver/node_linux_test.go +++ b/pkg/driver/node_linux_test.go @@ -25,9 +25,9 @@ import ( "testing" "time" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" "github.com/golang/mock/gomock" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" "github.com/stretchr/testify/assert" ) diff --git a/pkg/driver/node_test.go b/pkg/driver/node_test.go index 6114c54ffc..e74991090f 100644 --- a/pkg/driver/node_test.go +++ b/pkg/driver/node_test.go @@ -30,10 +30,10 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws/arn" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/mock/gomock" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" diff --git a/pkg/driver/sanity_test.go b/pkg/driver/sanity_test.go index bcf1a520ad..3bf6e6e609 100644 --- a/pkg/driver/sanity_test.go +++ b/pkg/driver/sanity_test.go @@ -14,9 +14,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/kubernetes-csi/csi-test/pkg/sanity" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "k8s.io/mount-utils" mount_utils "k8s.io/mount-utils" ) diff --git a/pkg/driver/validation.go b/pkg/driver/validation.go index 8ba4c7be3c..68562eb38b 100644 --- a/pkg/driver/validation.go +++ b/pkg/driver/validation.go @@ -21,7 +21,7 @@ import ( "regexp" "strings" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" "k8s.io/klog/v2" ) diff --git a/pkg/driver/validation_test.go b/pkg/driver/validation_test.go index 42f85bceac..7ead176b74 100644 --- a/pkg/driver/validation_test.go +++ b/pkg/driver/validation_test.go @@ -23,7 +23,7 @@ import ( "strconv" "testing" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" ) func randomString(n int) string { diff --git a/tests/e2e/driver/ebs_csi_driver.go b/tests/e2e/driver/ebs_csi_driver.go index 265034dd8b..7092d6baf4 100644 --- a/tests/e2e/driver/ebs_csi_driver.go +++ b/tests/e2e/driver/ebs_csi_driver.go @@ -18,7 +18,7 @@ import ( "fmt" volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + ebscsidriver "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/tests/e2e/dynamic_provisioning.go b/tests/e2e/dynamic_provisioning.go index 9342e4e529..c619acb34a 100644 --- a/tests/e2e/dynamic_provisioning.go +++ b/tests/e2e/dynamic_provisioning.go @@ -27,11 +27,11 @@ import ( restclientset "k8s.io/client-go/rest" "k8s.io/kubernetes/test/e2e/framework" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/testsuites" - awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + awscloud "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + ebscsidriver "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" diff --git a/tests/e2e/pre_provsioning.go b/tests/e2e/pre_provsioning.go index 2924f42b12..3bd221930f 100644 --- a/tests/e2e/pre_provsioning.go +++ b/tests/e2e/pre_provsioning.go @@ -22,12 +22,12 @@ import ( "strings" "time" - ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + ebscsidriver "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" k8srestclient "k8s.io/client-go/rest" - awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites" + awscloud "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/testsuites" . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/dynamically_provisioned_cmd_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_cmd_volume_tester.go index c72e56bd64..7b225e3ed8 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_cmd_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_cmd_volume_tester.go @@ -15,7 +15,7 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/dynamically_provisioned_collocated_pod_tester.go b/tests/e2e/testsuites/dynamically_provisioned_collocated_pod_tester.go index d8e5b77f2b..90858f4e58 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_collocated_pod_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_collocated_pod_tester.go @@ -15,7 +15,7 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/dynamically_provisioned_delete_pod_tester.go b/tests/e2e/testsuites/dynamically_provisioned_delete_pod_tester.go index 20502c43b7..30fe17400b 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_delete_pod_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_delete_pod_tester.go @@ -15,7 +15,7 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/dynamically_provisioned_read_only_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_read_only_volume_tester.go index 4cf48c944c..97f5a09184 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_read_only_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_read_only_volume_tester.go @@ -17,7 +17,7 @@ package testsuites import ( "fmt" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/test/e2e/framework" diff --git a/tests/e2e/testsuites/dynamically_provisioned_reclaim_policy_tester.go b/tests/e2e/testsuites/dynamically_provisioned_reclaim_policy_tester.go index e58e1210ae..d1f9637a79 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_reclaim_policy_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_reclaim_policy_tester.go @@ -15,8 +15,8 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go index c264467104..c0b301a681 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go @@ -19,8 +19,8 @@ import ( "fmt" "time" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/tests/e2e/testsuites/dynamically_provisioned_topology_aware_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_topology_aware_volume_tester.go index b1d32078f4..51ad1cadf4 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_topology_aware_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_topology_aware_volume_tester.go @@ -17,7 +17,7 @@ package testsuites import ( "fmt" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/dynamically_provisioned_volume_snapshot_tester.go b/tests/e2e/testsuites/dynamically_provisioned_volume_snapshot_tester.go index df6615ca36..770d923c84 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_volume_snapshot_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_volume_snapshot_tester.go @@ -15,7 +15,7 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/pre_provisioned_read_only_volume_tester.go b/tests/e2e/testsuites/pre_provisioned_read_only_volume_tester.go index 8fa5ec2963..b521aecc30 100644 --- a/tests/e2e/testsuites/pre_provisioned_read_only_volume_tester.go +++ b/tests/e2e/testsuites/pre_provisioned_read_only_volume_tester.go @@ -17,7 +17,7 @@ package testsuites import ( "fmt" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" diff --git a/tests/e2e/testsuites/pre_provisioned_reclaim_policy_tester.go b/tests/e2e/testsuites/pre_provisioned_reclaim_policy_tester.go index 8fd7793a19..5cd1f83e8d 100644 --- a/tests/e2e/testsuites/pre_provisioned_reclaim_policy_tester.go +++ b/tests/e2e/testsuites/pre_provisioned_reclaim_policy_tester.go @@ -15,7 +15,7 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go b/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go index b4d2af45f2..594ff02789 100644 --- a/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go +++ b/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go @@ -17,7 +17,7 @@ package testsuites import ( "fmt" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/pre_provisioned_volume_tester.go b/tests/e2e/testsuites/pre_provisioned_volume_tester.go index b3967fe72d..e273181076 100644 --- a/tests/e2e/testsuites/pre_provisioned_volume_tester.go +++ b/tests/e2e/testsuites/pre_provisioned_volume_tester.go @@ -15,7 +15,7 @@ limitations under the License. package testsuites import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" diff --git a/tests/e2e/testsuites/specs.go b/tests/e2e/testsuites/specs.go index e2f24c9743..d268a99c7c 100644 --- a/tests/e2e/testsuites/specs.go +++ b/tests/e2e/testsuites/specs.go @@ -17,7 +17,7 @@ package testsuites import ( "fmt" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" diff --git a/tests/e2e/testsuites/testsuites.go b/tests/e2e/testsuites/testsuites.go index 2d7709c7b1..71ad72478c 100644 --- a/tests/e2e/testsuites/testsuites.go +++ b/tests/e2e/testsuites/testsuites.go @@ -24,7 +24,7 @@ import ( volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" snapshotclientset "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" - awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + awscloud "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apps "k8s.io/api/apps/v1" diff --git a/tests/integration/setup_test.go b/tests/integration/setup_test.go index 6d6c67870f..2e39f8e176 100644 --- a/tests/integration/setup_test.go +++ b/tests/integration/setup_test.go @@ -25,10 +25,10 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" + "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" + "github.com/c2devel/aws-ebs-csi-driver/pkg/util" csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "google.golang.org/grpc" From 77e9e026db81c44a09b1f8b3cf418cd82c20d149 Mon Sep 17 00:00:00 2001 From: Rouslan Gaisin Date: Thu, 22 Jul 2021 15:59:15 +0500 Subject: [PATCH 03/45] replace new dependencies sources with c2devel --- pkg/driver/mount_windows.go | 4 ++-- pkg/driver/node_windows.go | 2 +- .../e2e/testsuites/pre_provisioned_snapshot_volume_tester.go | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/driver/mount_windows.go b/pkg/driver/mount_windows.go index 8824f3dba4..f3d939beb7 100644 --- a/pkg/driver/mount_windows.go +++ b/pkg/driver/mount_windows.go @@ -21,8 +21,8 @@ package driver import ( "fmt" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/resizefs" + "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" + "github.com/c2devel/aws-ebs-csi-driver/pkg/resizefs" mountutils "k8s.io/mount-utils" "regexp" ) diff --git a/pkg/driver/node_windows.go b/pkg/driver/node_windows.go index 9e31a8c643..c2353515fe 100644 --- a/pkg/driver/node_windows.go +++ b/pkg/driver/node_windows.go @@ -27,7 +27,7 @@ import ( diskapi "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1" diskclient "github.com/kubernetes-csi/csi-proxy/client/groups/disk/v1" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" + "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" "k8s.io/klog/v2" ) diff --git a/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go b/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go index 594ff02789..abc1e05542 100644 --- a/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go +++ b/tests/e2e/testsuites/pre_provisioned_snapshot_volume_tester.go @@ -16,7 +16,6 @@ package testsuites import ( "fmt" - "github.com/c2devel/aws-ebs-csi-driver/tests/e2e/driver" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" From c9b8841f107934038f897149c3f3df5fccb1ac19 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Tue, 31 Mar 2020 15:16:48 +0300 Subject: [PATCH 04/45] cloud: add io2 and gp2 volume types --- pkg/cloud/cloud.go | 48 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 8a8457d751..fb0bed8923 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -20,10 +20,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "crypto/tls" "errors" "fmt" + "net/http" "os" "strings" + "strconv" "time" "github.com/aws/aws-sdk-go/aws" @@ -53,6 +56,8 @@ const ( VolumeTypeSC1 = "sc1" // VolumeTypeST1 represents a throughput-optimized HDD type of volume. VolumeTypeST1 = "st1" + // VolumeTypeST2 represents a throughput-optimized HDD type of volume. + VolumeTypeST2 = "st2" // VolumeTypeSBG1 represents a capacity-optimized HDD type of volume. Only for SBE devices. VolumeTypeSBG1 = "sbg1" // VolumeTypeSBP1 represents a performance-optimized SSD type of volume. Only for SBE devices. @@ -77,6 +82,7 @@ const ( ) var ( + // ValidVolumeTypes represents list of available volume types ValidVolumeTypes = []string{ VolumeTypeIO1, VolumeTypeIO2, @@ -84,6 +90,7 @@ var ( VolumeTypeGP3, VolumeTypeSC1, VolumeTypeST1, + VolumeTypeST2, VolumeTypeStandard, } @@ -249,12 +256,39 @@ func NewCloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Cloud, return newEC2Cloud(region, awsSdkDebugLog, userAgentExtra) } + func newEC2Cloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Cloud, error) { - awsConfig := &aws.Config{ - Region: aws.String(region), - CredentialsChainVerboseErrors: aws.Bool(true), - // Set MaxRetries to a high value. It will be "ovewritten" if context deadline comes sooner. - MaxRetries: aws.Int(8), + + var awsConfig *aws.Config + + envEndpointInsecure := os.Getenv("AWS_EC2_ENDPOINT_UNSECURE") + isEndpointInsecure := false + if envEndpointInsecure != "" { + var err error + isEndpointInsecure, err = strconv.ParseBool(envEndpointInsecure) + if err != nil { + return nil, fmt.Errorf("Unable to parse environment variable AWS_EC2_ENDPOINT_UNSECURE: %v", err) + } + } + + if isEndpointInsecure { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + awsConfig = &aws.Config{ + Region: aws.String(region), + CredentialsChainVerboseErrors: aws.Bool(true), + HTTPClient: client, + } + } else { + awsConfig = &aws.Config{ + Region: aws.String(region), + CredentialsChainVerboseErrors: aws.Bool(true), + // Set MaxRetries to a high value. It will be "ovewritten" if context deadline comes sooner. + MaxRetries: aws.Int(8), + } } endpoint := os.Getenv("AWS_EC2_ENDPOINT") @@ -324,7 +358,7 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions * } switch createType { - case VolumeTypeGP2, VolumeTypeSC1, VolumeTypeST1, VolumeTypeSBG1, VolumeTypeSBP1, VolumeTypeStandard: + case VolumeTypeGP2, VolumeTypeST2, VolumeTypeStandard: case VolumeTypeIO1: maxIops = io1MaxTotalIOPS minIops = io1MinTotalIOPS @@ -523,7 +557,7 @@ func (c *cloud) AttachDisk(ctx context.Context, volumeID, nodeID string) (string if !device.IsAlreadyAssigned { request := &ec2.AttachVolumeInput{ - Device: aws.String(device.Path), + Device: aws.String("disk2"), InstanceId: aws.String(nodeID), VolumeId: aws.String(volumeID), } From b89bf3c5fdcb456b17cfbde5a63a45de5c2c23f8 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Tue, 31 Mar 2020 15:23:53 +0300 Subject: [PATCH 05/45] cloud: switch to volumeId-based device managing --- pkg/cloud/devicemanager/allocator.go | 54 ----------------------- pkg/cloud/devicemanager/allocator_test.go | 53 ---------------------- pkg/cloud/devicemanager/manager.go | 53 +++++++++++----------- 3 files changed, 27 insertions(+), 133 deletions(-) delete mode 100644 pkg/cloud/devicemanager/allocator.go delete mode 100644 pkg/cloud/devicemanager/allocator_test.go diff --git a/pkg/cloud/devicemanager/allocator.go b/pkg/cloud/devicemanager/allocator.go deleted file mode 100644 index 911a64efea..0000000000 --- a/pkg/cloud/devicemanager/allocator.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package devicemanager - -import ( - "fmt" -) - -// ExistingNames is a map of assigned device names. Presence of a key with a device -// name in the map means that the device is allocated. Value is irrelevant and -// can be used for anything that NameAllocator user wants. -type ExistingNames map[string]string - -// On AWS, we should assign new (not yet used) device names to attached volumes. -// If we reuse a previously used name, we may get the volume "attaching" forever, -// see https://aws.amazon.com/premiumsupport/knowledge-center/ebs-stuck-attaching/. -// NameAllocator finds available device name, taking into account already -// assigned device names from ExistingNames map. It tries to find the next -// device name to the previously assigned one (from previous NameAllocator -// call), so all available device names are used eventually and it minimizes -// device name reuse. -type NameAllocator interface { - GetNext(existingNames ExistingNames) (name string, err error) -} - -type nameAllocator struct{} - -var _ NameAllocator = &nameAllocator{} - -// GetNext returns a free device name or error when there is no free device name -// It does this by using a list of legal EBS device names from device_names.go -func (d *nameAllocator) GetNext(existingNames ExistingNames) (string, error) { - for _, name := range deviceNames { - if _, found := existingNames[name]; !found { - return name, nil - } - } - - return "", fmt.Errorf("there are no names available") -} diff --git a/pkg/cloud/devicemanager/allocator_test.go b/pkg/cloud/devicemanager/allocator_test.go deleted file mode 100644 index eae46d9001..0000000000 --- a/pkg/cloud/devicemanager/allocator_test.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package devicemanager - -import ( - "testing" -) - -func TestNameAllocator(t *testing.T) { - existingNames := map[string]string{} - allocator := nameAllocator{} - - for _, name := range deviceNames { - t.Run(name, func(t *testing.T) { - actual, err := allocator.GetNext(existingNames) - if err != nil { - t.Errorf("test %q: unexpected error: %v", name, err) - } - if actual != name { - t.Errorf("test %q: expected %q, got %q", name, name, actual) - } - existingNames[actual] = "" - }) - } -} - -func TestNameAllocatorError(t *testing.T) { - allocator := nameAllocator{} - existingNames := map[string]string{} - - for i := 0; i < len(deviceNames); i++ { - name, _ := allocator.GetNext(existingNames) - existingNames[name] = "" - } - name, err := allocator.GetNext(existingNames) - if err == nil { - t.Errorf("expected error, got device %q", name) - } -} diff --git a/pkg/cloud/devicemanager/manager.go b/pkg/cloud/devicemanager/manager.go index 05612edb15..d65a80a936 100644 --- a/pkg/cloud/devicemanager/manager.go +++ b/pkg/cloud/devicemanager/manager.go @@ -18,6 +18,7 @@ package devicemanager import ( "fmt" + "strings" "sync" "github.com/aws/aws-sdk-go/aws" @@ -25,6 +26,8 @@ import ( "k8s.io/klog/v2" ) +const devPreffix = "/dev/disk/by-id/virtio-" + type Device struct { Instance *ec2.Instance Path string @@ -59,9 +62,6 @@ type DeviceManager interface { } type deviceManager struct { - // nameAllocator assigns new device name - nameAllocator NameAllocator - // We keep an active list of devices we have assigned but not yet // attached, to avoid a race condition where we assign a device mapping // and then get a second request before we attach the volume. @@ -98,8 +98,7 @@ func (i inFlightAttaching) GetVolume(nodeID, name string) string { func NewDeviceManager() DeviceManager { return &deviceManager{ - nameAllocator: &nameAllocator{}, - inFlight: make(inFlightAttaching), + inFlight: make(inFlightAttaching), } } @@ -112,7 +111,7 @@ func (d *deviceManager) NewDevice(instance *ec2.Instance, volumeID string) (*Dev } // Get device names being attached and already attached to this instance - inUse := d.getDeviceNamesInUse(instance) + inUse := d.getVolumeIdsInUse(instance) // Check if this volume is already assigned a device on this machine if path := d.getPath(inUse, volumeID); path != "" { @@ -124,22 +123,17 @@ func (d *deviceManager) NewDevice(instance *ec2.Instance, volumeID string) (*Dev return nil, err } - name, err := d.nameAllocator.GetNext(inUse) - if err != nil { - return nil, fmt.Errorf("could not get a free device name to assign to node %s", nodeID) - } - // Add the chosen device and volume to the "attachments in progress" map - d.inFlight.Add(nodeID, volumeID, name) + d.inFlight.Add(nodeID, volumeID, volumeID) - return d.newBlockDevice(instance, volumeID, name, false), nil + return d.newBlockDevice(instance, volumeID, devPreffix+volumeID, false), nil } func (d *deviceManager) GetDevice(instance *ec2.Instance, volumeID string) (*Device, error) { d.mux.Lock() defer d.mux.Unlock() - inUse := d.getDeviceNamesInUse(instance) + inUse := d.getVolumeIdsInUse(instance) if path := d.getPath(inUse, volumeID); path != "" { return d.newBlockDevice(instance, volumeID, path, true), nil @@ -172,7 +166,7 @@ func (d *deviceManager) release(device *Device) error { d.mux.Lock() defer d.mux.Unlock() - existingVolumeID := d.inFlight.GetVolume(nodeID, device.Path) + existingVolumeID := d.inFlight.GetVolume(nodeID, device.VolumeID) if len(existingVolumeID) == 0 { // Attaching is not in progress, so there's nothing to release return nil @@ -186,33 +180,40 @@ func (d *deviceManager) release(device *Device) error { return fmt.Errorf("release on device %q assigned to different volume: %q vs %q", device.Path, device.VolumeID, existingVolumeID) } - klog.V(5).InfoS("[Debug] Releasing in-process", "attachment entry", device.Path, "volume", device.VolumeID) - d.inFlight.Del(nodeID, device.Path) + klog.V(5).Infof("[Debug] Releasing in-process attachment entry: %v -> volume %s", device.Path, device.VolumeID) + d.inFlight.Del(nodeID, device.VolumeID) return nil } -// getDeviceNamesInUse returns the device to volume ID mapping +// getVolumeIdsInUse returns the device to volume ID mapping // the mapping includes both already attached and being attached volumes -func (d *deviceManager) getDeviceNamesInUse(instance *ec2.Instance) map[string]string { +func (d *deviceManager) getVolumeIdsInUse(instance *ec2.Instance) []string { nodeID := aws.StringValue(instance.InstanceId) - inUse := map[string]string{} + var inUse []string for _, blockDevice := range instance.BlockDeviceMappings { + if blockDevice.Ebs == nil { + continue + } + name := aws.StringValue(blockDevice.DeviceName) - inUse[name] = aws.StringValue(blockDevice.Ebs.VolumeId) + // trim device prefix from name + name = strings.TrimPrefix(name, devPreffix) + + inUse = append(inUse, *blockDevice.Ebs.VolumeId) } - for name, volumeID := range d.inFlight.GetNames(nodeID) { - inUse[name] = volumeID + for _, volumeID := range d.inFlight.GetNames(nodeID) { + inUse = append(inUse, volumeID) } return inUse } -func (d *deviceManager) getPath(inUse map[string]string, volumeID string) string { - for name, volID := range inUse { +func (d *deviceManager) getPath(inUse []string, volumeID string) string { + for _, volID := range inUse { if volumeID == volID { - return name + return devPreffix + volumeID } } return "" From 56dbcb27fdc3576c05f37328ad61aee85df2a693 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Tue, 8 Oct 2019 18:39:57 +0300 Subject: [PATCH 06/45] deploy: change deploy yamls to use new repo --- deploy/kubernetes/base/controller.yaml | 6 +++++- deploy/kubernetes/base/node.yaml | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index 8c1af06dbc..e2f5b6d915 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -61,7 +61,7 @@ spec: runAsUser: 1000 containers: - name: ebs-plugin - image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.20.0 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:latest imagePullPolicy: IfNotPresent args: # - {all,controller,node} # specify the driver mode @@ -76,6 +76,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: AWS_EC2_ENDPOINT + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: ru-msk - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index 6b9e2f7674..1b771e78e8 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -44,7 +44,7 @@ spec: runAsUser: 0 containers: - name: ebs-plugin - image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.20.0 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:latest imagePullPolicy: IfNotPresent args: - node @@ -58,6 +58,9 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: AWS_REGION + value: ru-msk + envFrom: volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet From c5fb9e2ccf52a33f1847d9a06396136d15531a24 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Wed, 1 Apr 2020 19:41:37 +0300 Subject: [PATCH 07/45] examples: fix examples to meet c2 reqs --- examples/kubernetes/snapshot/manifests/app/claim.yaml | 2 +- .../kubernetes/snapshot/manifests/classes/storageclass.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/kubernetes/snapshot/manifests/app/claim.yaml b/examples/kubernetes/snapshot/manifests/app/claim.yaml index a883baa530..13e0a80946 100644 --- a/examples/kubernetes/snapshot/manifests/app/claim.yaml +++ b/examples/kubernetes/snapshot/manifests/app/claim.yaml @@ -8,4 +8,4 @@ spec: storageClassName: ebs-sc resources: requests: - storage: 4Gi + storage: 8Gi diff --git a/examples/kubernetes/snapshot/manifests/classes/storageclass.yaml b/examples/kubernetes/snapshot/manifests/classes/storageclass.yaml index d6e168e1ec..d97d1a03f8 100644 --- a/examples/kubernetes/snapshot/manifests/classes/storageclass.yaml +++ b/examples/kubernetes/snapshot/manifests/classes/storageclass.yaml @@ -4,3 +4,5 @@ metadata: name: ebs-sc provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer +parameters: + type: st2 From befb9f67e22ffa52fd4cfb894527e4fc98c42f3f Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Thu, 30 Apr 2020 19:16:13 +0300 Subject: [PATCH 08/45] change region to 'croc' --- deploy/kubernetes/base/controller.yaml | 2 +- deploy/kubernetes/base/node.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index e2f5b6d915..f2fc3d6fab 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -79,7 +79,7 @@ spec: - name: AWS_EC2_ENDPOINT value: https://api.cloud.croc.ru - name: AWS_REGION - value: ru-msk + value: croc - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index 1b771e78e8..551a2c2614 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -59,7 +59,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: AWS_REGION - value: ru-msk + value: croc envFrom: volumeMounts: - name: kubelet-dir From 452dc7da0cd4d06e7d5ed29e6f058dcbb1ff50f3 Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Fri, 19 Jun 2020 13:15:18 +0300 Subject: [PATCH 09/45] add prebuild kusctomize config --- .../kubernetes/overlays/stable/k_bundle.yaml | 366 ++++++++++++++++++ 1 file changed, 366 insertions(+) create mode 100644 deploy/kubernetes/overlays/stable/k_bundle.yaml diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml new file mode 100644 index 0000000000..318cbd52b3 --- /dev/null +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -0,0 +1,366 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ebs-external-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ebs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ebs-csi-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ebs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + template: + metadata: + labels: + app: ebs-csi-controller + spec: + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_EC2_ENDPOINT + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: croc + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + - --enable-leader-election + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: dhub.c2.croc.ru/kaas/csi-provisioner:v1.3.0 + name: csi-provisioner + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: dhub.c2.croc.ru/kaas/csi-attacher:v1.2.0 + name: csi-attacher + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + image: dhub.c2.croc.ru/kaas/livenessprobe:v1.1.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + beta.kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccount: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: socket-dir +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + template: + metadata: + labels: + app: ebs-csi-node + spec: + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: AWS_REGION + value: croc + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: dhub.c2.croc.ru/kaas/csi-node-driver-registrar:v1.1.0 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=/csi/csi.sock + image: dhub.c2.croc.ru/kaas/livenessprobe:v1.1.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + priorityClassName: system-node-critical + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false From adde6f32a9a798e2886d3746cf55b84f9b4ef253 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Tue, 23 Jun 2020 17:31:14 +0300 Subject: [PATCH 10/45] add AttachVolume method and remove Device from call --- pkg/cloud/cloud.go | 179 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 3 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index fb0bed8923..d5e9367bb1 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -32,6 +32,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" @@ -249,6 +250,179 @@ type cloud struct { var _ Cloud = &cloud{} +// AttachVolumeRequest generates a "aws/request.Request" representing the +// client's request for the AttachVolume operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See AttachVolume for more information on using the AttachVolume +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the AttachVolumeRequest method. +// req, resp := client.AttachVolumeRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/ec2-2016-11-15/AttachVolume +func AttachVolumeRequest(c *ec2.EC2, input *AttachVolumeInput) (req *request.Request, output *ec2.VolumeAttachment) { + op := &request.Operation{ + Name: "AttachVolume", + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &AttachVolumeInput{} + } + + output = &ec2.VolumeAttachment{} + req = c.NewRequest(op, input, output) + return +} + +// AttachVolume API operation for Amazon Elastic Compute Cloud. +// +// Attaches an EBS volume to a running or stopped instance and exposes it to +// the instance with the specified device name. +// +// Encrypted EBS volumes must be attached to instances that support Amazon EBS +// encryption. For more information, see Amazon EBS Encryption (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html) +// in the Amazon Elastic Compute Cloud User Guide. +// +// After you attach an EBS volume, you must make it available. For more information, +// see Making an EBS Volume Available For Use (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-using-volumes.html). +// +// If a volume has an AWS Marketplace product code: +// +// * The volume can be attached only to a stopped instance. +// +// * AWS Marketplace product codes are copied from the volume to the instance. +// +// * You must be subscribed to the product. +// +// * The instance type and operating system of the instance must support +// the product. For example, you can't detach a volume from a Windows instance +// and attach it to a Linux instance. +// +// For more information, see Attaching Amazon EBS Volumes (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-attaching-volume.html) +// in the Amazon Elastic Compute Cloud User Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for Amazon Elastic Compute Cloud's +// API operation AttachVolume for usage and error information. +// See also, https://docs.aws.amazon.com/goto/WebAPI/ec2-2016-11-15/AttachVolume +func AttachVolume(c *ec2.EC2, input *AttachVolumeInput) (*ec2.VolumeAttachment, error) { + req, out := AttachVolumeRequest(c, input) + return out, req.Send() +} + +// AttachVolumeWithContext is the same as AttachVolume with the addition of +// the ability to pass a context and additional request options. +// +// See AttachVolume for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func AttachVolumeWithContext(c *ec2.EC2, ctx aws.Context, input *AttachVolumeInput, opts ...request.Option) (*ec2.VolumeAttachment, error) { + req, out := AttachVolumeRequest(c, input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +// AttachVolumeInput is a type that Contains the parameters for AttachVolume. +type AttachVolumeInput struct { + _ struct{} `type:"structure"` + + // The device name (for example, /dev/sdh or xvdh). + // + // Device is a required field + Device *string `type:"string"` + + // Checks whether you have the required permissions for the action, without + // actually making the request, and provides an error response. If you have + // the required permissions, the error response is DryRunOperation. Otherwise, + // it is UnauthorizedOperation. + DryRun *bool `locationName:"dryRun" type:"boolean"` + + // The ID of the instance. + // + // InstanceId is a required field + InstanceId *string `type:"string" required:"true"` + + // The ID of the EBS volume. The volume and instance must be within the same + // Availability Zone. + // + // VolumeId is a required field + VolumeId *string `type:"string" required:"true"` +} + +// String returns the string representation +func (s AttachVolumeInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s AttachVolumeInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *AttachVolumeInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "AttachVolumeInput"} + + if s.InstanceId == nil { + invalidParams.Add(request.NewErrParamRequired("InstanceId")) + } + if s.VolumeId == nil { + invalidParams.Add(request.NewErrParamRequired("VolumeId")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetDevice sets the Device field's value. +func (s *AttachVolumeInput) SetDevice(v string) *AttachVolumeInput { + s.Device = &v + return s +} + +// SetDryRun sets the DryRun field's value. +func (s *AttachVolumeInput) SetDryRun(v bool) *AttachVolumeInput { + s.DryRun = &v + return s +} + +// SetInstanceId sets the InstanceId field's value. +func (s *AttachVolumeInput) SetInstanceId(v string) *AttachVolumeInput { + s.InstanceId = &v + return s +} + +// SetVolumeId sets the VolumeId field's value. +func (s *AttachVolumeInput) SetVolumeId(v string) *AttachVolumeInput { + s.VolumeId = &v + return s +} + // NewCloud returns a new instance of AWS cloud // It panics if session is invalid func NewCloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Cloud, error) { @@ -556,13 +730,12 @@ func (c *cloud) AttachDisk(ctx context.Context, volumeID, nodeID string) (string defer device.Release(false) if !device.IsAlreadyAssigned { - request := &ec2.AttachVolumeInput{ - Device: aws.String("disk2"), + request := &AttachVolumeInput{ InstanceId: aws.String(nodeID), VolumeId: aws.String(volumeID), } - resp, attachErr := c.ec2.AttachVolumeWithContext(ctx, request) + resp, attachErr := AttachVolumeWithContext(c.ec2.(*ec2.EC2), ctx, request) if attachErr != nil { var awsErr awserr.Error if errors.As(attachErr, &awsErr) { From 31d6d94433b089228cf57ac61b687d06c7e99787 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Thu, 25 Jun 2020 17:14:53 +0300 Subject: [PATCH 11/45] temporarily skip test for AttachDisk --- pkg/cloud/cloud_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index 90608214c6..8868a184b1 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -711,6 +711,8 @@ func TestDeleteDisk(t *testing.T) { } func TestAttachDisk(t *testing.T) { + t.Skip("Skipping temporarily due to interface inconsistency.") + testCases := []struct { name string volumeID string From 01190c952895204aeca9a909b882e83e78bbc81a Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Thu, 25 Jun 2020 17:15:25 +0300 Subject: [PATCH 12/45] change sc1 to st2 in tests --- pkg/driver/controller_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go index b127780bb0..e2db0d7d28 100644 --- a/pkg/driver/controller_test.go +++ b/pkg/driver/controller_test.go @@ -967,14 +967,14 @@ func TestCreateVolume(t *testing.T) { }, }, { - name: "success with volume type sc1", + name: "success with volume type st2", testFunc: func(t *testing.T) { req := &csi.CreateVolumeRequest{ Name: "vol-test", CapacityRange: stdCapRange, VolumeCapabilities: stdVolCap, Parameters: map[string]string{ - VolumeTypeKey: cloud.VolumeTypeSC1, + VolumeTypeKey: cloud.VolumeTypeST2, }, } From 0342ad98af43b21f0c20a9a158369a4c6dc03bd5 Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Tue, 30 Jun 2020 12:16:12 +0300 Subject: [PATCH 13/45] readme: add release-process docs --- docs/README.md | 311 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/RELEASE.md | 44 +++++++ 2 files changed, 355 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/RELEASE.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..111e1a888d --- /dev/null +++ b/docs/README.md @@ -0,0 +1,311 @@ +[![Build Status](https://travis-ci.org/c2devel/aws-ebs-csi-driver.svg?branch=master)](https://travis-ci.org/c2devel/aws-ebs-csi-driver) +[![Coverage Status](https://coveralls.io/repos/github/c2devel/aws-ebs-csi-driver/badge.svg?branch=master)](https://coveralls.io/github/c2devel/aws-ebs-csi-driver?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/c2devel/aws-ebs-csi-driver)](https://goreportcard.com/report/github.com/c2devel/aws-ebs-csi-driver) + +# Amazon Elastic Block Store (EBS) CSI driver + +## Скорректированный релиз процесс + +см. [release process](RELEASE.md) + +## Overview + +The [Amazon Elastic Block Store](https://aws.amazon.com/ebs/) Container Storage Interface (CSI) Driver provides a [CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md) interface used by Container Orchestrators to manage the lifecycle of Amazon EBS volumes. + +## CSI Specification Compatibility Matrix +| AWS EBS CSI Driver \ CSI Version | v0.3.0| v1.0.0 | v1.1.0 | +|----------------------------------------|-------|--------|--------| +| master branch | no | no | yes | +| v1.1.x | no | no | yes | +| v1.0.0 | no | no | yes | +| v0.10.x | no | no | yes | +| v0.9.x | no | no | yes | +| v0.8.x | no | no | yes | +| v0.7.1 | no | no | yes | +| v0.6.0 | no | no | yes | +| v0.5.0 | no | no | yes | +| v0.4.0 | no | no | yes | +| v0.3.0 | no | yes | no | +| v0.2.0 | no | yes | no | +| v0.1.0 | yes | no | no | + +## Features +The following CSI gRPC calls are implemented: +* **Controller Service**: CreateVolume, DeleteVolume, ControllerPublishVolume, ControllerUnpublishVolume, ControllerGetCapabilities, ValidateVolumeCapabilities, CreateSnapshot, DeleteSnapshot, ListSnapshots +* **Node Service**: NodeStageVolume, NodeUnstageVolume, NodePublishVolume, NodeUnpublishVolume, NodeGetCapabilities, NodeGetInfo +* **Identity Service**: GetPluginInfo, GetPluginCapabilities, Probe + +## CreateVolume Parameters +There are several optional parameters that could be passed into `CreateVolumeRequest.parameters` map, these parameters can be configured in StorageClass, see [example](../examples/kubernetes/storageclass): + +| Parameters | Values | Default | Description | +|-----------------------------|----------------------------------------|----------|---------------------| +| "csi.storage.k8s.io/fstype" | xfs, ext2, ext3, ext4 | ext4 | File system type that will be formatted during volume creation. This parameter is case sensitive! | +| "type" | io1, io2, gp2, gp3, sc1, st1,standard | gp3* | EBS volume type | +| "iopsPerGB" | | | I/O operations per second per GiB. Required when io1 or io2 volume type is specified. If this value multiplied by the size of a requested volume produces a value above the maximum IOPs allowed for the volume type, as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), AWS will cap the IOPS to maximum supported value. If the value is lower than minimal supported IOPS value per volume, either error is returned (the default behavior) or the value is increased to fit into the supported range when `allowautoiopspergbincrease` is `"true"`.| +| "allowAutoIOPSPerGBIncrease"| true, false | false | When `"true"`, the CSI driver increases IOPS for a volume when `iopsPerGB * ` is too low to fit into IOPS range supported by AWS. This allows dynamic provisioning to always succeed, even when user specifies too small PVC capacity or `iopsPerGB` value. On the other hand, it may introduce additional costs, as such volumes have higher IOPS than requested in `iopsPerGB`.| +| "iops" | | 3000 | I/O operations per second. Only effetive when gp3 volume type is specified. If empty, it will set to 3000 as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). | +| "throughput" | | 125 | Throughput in MiB/s. Only effective when gp3 volume type is specified. If empty, it will set to 125MiB/s as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). | +| "encrypted" | | | Whether the volume should be encrypted or not. Valid values are "true" or "false" | +| "kmsKeyId" | | | The full ARN of the key to use when encrypting the volume. When not specified, the default KMS key is used | + +**Notes**: +* `gp3` is currently not supported on outposts. Outpost customers need to use a different type for their volumes. +* Unless explicitly noted, all parameters are case insensitive (e.g. "kmsKeyId", "kmskeyid" and any other combination of upper/lowercase characters can be used). + +## Tagging +To help manage volumes in the aws account, CSI driver will automatically add tags to the volumes it manages. + +| TagKey | TagValue | sample | Description | +|------------------------|---------------------------|---------------------------------------------------------------------|---------------------| +| CSIVolumeName | pvcName | CSIVolumeName = pvc-a3ab0567-3a48-4608-8cb6-4e3b1485c808 | add to all volumes, for recording associated pvc id and checking if a given volume was already created so that ControllerPublish/CreateVolume is idempotent. | +| CSISnapshotName | volumeSnapshotContentName | CSISnapshotName = snapcontent-69477690-803b-4d3e-a61a-03c7b2592a76 | add to all snapshots, for recording associated VolumeSnapshot id and checking if a given snapshot was already created | +| ebs.csi.aws.com/cluster| true | ebs.csi.aws.com/cluster = true | add to all volumes and snapshots, for allowing users to use a policy to limit csi driver's permission to just the resources it manages. | +| kubernetes.io/cluster/X| owned | kubernetes.io/cluster/aws-cluster-id-1 = owned | add to all volumes and snapshots if k8s-tag-cluster-id argument is set to X.| +| extra-key | extra-value | extra-key = extra-value | add to all volumes and snapshots if extraTags argument is set| + +## Driver Options +There are couple driver options that can be passed as arguments when starting driver container. + +| Option argument | value sample | default | Description | +|-----------------------------|---------------------------------------------------|-----------------------------------------------------|---------------------| +| endpoint | tcp://127.0.0.1:10000/ | unix:///var/lib/csi/sockets/pluginproxy/csi.sock | added to all volumes, for checking if a given volume was already created so that ControllerPublish/CreateVolume is idempotent. | +| volume-attach-limit | 1,2,3 ... | -1 | Value for the maximum number of volumes attachable per node. If specified, the limit applies to all nodes. If not specified, the value is approximated from the instance type. | +| extra-tags | key1=value1,key2=value2 | | Extra tags to attach to each dynamically provisioned resource.| +| k8s-tag-cluster-id | aws-cluster-id-1 | | ID of the Kubernetes cluster used for tagging provisioned EBS volumes.| +| aws-sdk-debug-log | true | false | if true, driver will enable the aws sdk debug log level| + +# EBS CSI Driver on Kubernetes +Following sections are Kubernetes specific. If you are Kubernetes user, use followings for driver features, installation steps and examples. + +## Kubernetes Version Compatibility Matrix +| AWS EBS CSI Driver \ Kubernetes Version| v1.12 | v1.13 | v1.14 | v1.15 | v1.16 | v1.17 | v1.18+ | +|----------------------------------------|-------|-------|-------|-------|-------|-------|-------| +| master branch | no | no+ | no | no | no | yes | yes | +| v1.1.0 | no | no+ | no | no | no | yes | yes | +| v1.0.0 | no | no+ | no | no | no | yes | yes | +| v0.10.x | no | no+ | no | no | no | yes | yes | +| v0.9.x | no | no+ | no | no | no | yes | yes | +| v0.8.x | no | no+ | yes | yes | yes | yes | yes | +| v0.7.1 | no | no+ | yes | yes | yes | yes | yes | +| v0.6.0 | no | no+ | yes | yes | yes | yes | yes | +| v0.5.0 | no | no+ | yes | yes | yes | yes | yes | +| v0.4.0 | no | no+ | yes | yes | no | no | no | +| v0.3.0 | no | no+ | yes | no | no | no | no | +| v0.2.0 | no | yes | yes | no | no | no | no | +| v0.1.0 | yes | yes | yes | no | no | no | no | + +**Note**: for the entry with `+` sign, it means the driver's default released manifest doesn't work with corresponding Kubernetes version, but the driver container image is compatiable with the Kubernetes version if an older version's manifest is used. + +## Container Images: +|AWS EBS CSI Driver Version | Image | +|---------------------------|--------------------------------------------------| +|v1.1.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.1.1 | +|v1.1.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.1.0 | +|v1.0.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.0.0 | +|v0.10.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.10.1| +|v0.10.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.10.0| +|v0.9.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.9.1 | +|v0.9.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.9.0 | +|v0.8.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.8.1 | +|v0.7.1 |amazon/aws-ebs-csi-driver:v0.7.1 | +|v0.6.0 |amazon/aws-ebs-csi-driver:v0.6.0 | +|v0.5.0 |amazon/aws-ebs-csi-driver:v0.5.0 | +|v0.4.0 |amazon/aws-ebs-csi-driver:v0.4.0 | +|v0.3.0 |amazon/aws-ebs-csi-driver:v0.3.0 | +|v0.2.0 |amazon/aws-ebs-csi-driver:0.2.0 | +|v0.1.0 |amazon/aws-ebs-csi-driver:0.1.0-alpha | + +## Features +* **Static Provisioning** - create a new or migrating existing EBS volumes, then create persistence volume (PV) from the EBS volume and consume the PV from container using persistence volume claim (PVC). +* **Dynamic Provisioning** - uses persistence volume claim (PVC) to request the Kuberenetes to create the EBS volume on behalf of user and consumes the volume from inside container. Storage class's **allowedTopologies** could be used to restrict which AZ the volume should be provisioned in. The topology key should be **topology.ebs.csi.aws.com/zone**. +* **Mount Option** - mount options could be specified in persistence volume (PV) to define how the volume should be mounted. +* **NVMe** - consume NVMe EBS volume from EC2 [Nitro instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). +* **[Block Volume](https://kubernetes-csi.github.io/docs/raw-block.html)** - consumes the EBS volume as a raw block device for latency sensitive application eg. MySql. The corresponding CSI feature (`CSIBlockVolume`) is GA since Kubernetes 1.18. +* **[Volume Snapshot](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html)** - creating volume snapshots and restore volume from snapshot. The corresponding CSI feature (`VolumeSnapshotDataSource`) is beta since Kubernetes 1.17. +* **[Volume Resizing](https://kubernetes-csi.github.io/docs/volume-expansion.html)** - expand the volume size. The corresponding CSI feature (`ExpandCSIVolumes`) is beta since Kubernetes 1.16. + +## Prerequisites +* If you are managing EBS volumes using static provisioning, get yourself familiar with [EBS volume](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AmazonEBS.html). +* Get yourself familiar with how to setup Kubernetes on AWS and have a working Kubernetes cluster: + * Enable flag `--allow-privileged=true` for `kubelet` and `kube-apiserver` + * Enable `kube-apiserver` feature gates `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true` + * Enable `kubelet` feature gates `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true` +* If you intend to use the csi-snapshotter functionality you will need to first install the [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) + +## Installation +#### Set up driver permission +The driver requires IAM permission to talk to Amazon EBS to manage the volume on user's behalf. [The example policy here](./example-iam-policy.json) defines these permissions. There are several methods to grant the driver IAM permission: +* Using IAM [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) - attach the policy to the instance profile IAM role and turn on access to [instance metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for the instance(s) on which the driver Deployment will run +* EKS only: Using [IAM roles for ServiceAccounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) - create an IAM role, attach the policy to it, then follow the IRSA documentation to associate the IAM role with the driver Deployment service account, which if you are installing via helm is determined by value `controller.serviceAccount.name`, `ebs-csi-controller-sa` by default +* Using secret object - create an IAM user, attach the policy to it, put that user's credentials in [secret manifest](../deploy/kubernetes/secret.yaml), then deploy the secret +```sh +curl https://raw.githubusercontent.com/c2devel/aws-ebs-csi-driver/master/deploy/kubernetes/secret.yaml > secret.yaml +# Edit the secret with user credentials +kubectl apply -f secret.yaml +``` + +#### Config node toleration settings +By default, driver tolerates taint `CriticalAddonsOnly` and has `tolerationSeconds` configured as `300`, to deploy the driver on all nodes, please set helm `Value.node.tolerateAllTaints` to true before deployment + +#### Deploy driver +Please see the compatibility matrix above before you deploy the driver + +To deploy the CSI driver: +```sh +kubectl apply -k "github.com/c2devel/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=master" +``` + +Verify driver is running: +```sh +kubectl get pods -n kube-system +``` + +Alternatively, you could also install the driver using helm: + +Add the aws-ebs-csi-driver Helm repository: +```sh +helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver +helm repo update +``` + +Then install a release of the driver using the chart +```sh +helm upgrade --install aws-ebs-csi-driver \ + --namespace kube-system \ + aws-ebs-csi-driver/aws-ebs-csi-driver +``` + +##### Upgrading from version 1.X to 2.X of the helm chart +Version 2.0.0 remove support for helm v2 and now requires helm v3 or above + +The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. + +The following deprecated values have been removed, and you should now use their counterparts under the `controller` and `node` maps which have been available since chart version 1.1.0 +* affinity +* extraCreateMetadata +* extraVolumeTags +* k8sTagClusterId +* nodeSelector +* podAnnotations +* priorityClassName +* region +* replicaCount +* resources +* tolerations +* topologySpreadConstraints +* volumeAttachLimit + +The values under `serviceAccount.controller` have been relocated to `controller.serviceAccount` +The values under `serviceAccount.node` have been relocated to `node.serviceAccount` + +The following `sidecars` values have been reorganized from +```yaml +sidecars: + provisionerImage: + attacherImage: + snapshotterImage: + livenessProbeImage: + resizerImage: + nodeDriverRegistrarImage: +``` +to +```yaml +sidecars: + provisioner: + image: + attacher: + image: + snapshotter: + image: + livenessProbe: + image: + resizer: + image: + nodeDriverRegistrar: + image: +``` + +With the above reorganization `controller.containerResources`, `controller.env`, `node.containerResources`, and `node.env` were also moved into the sidecars structure as follows +```yaml +sidecars: + provisioner: + env: [] + resources: {} + attacher: + env: [] + resources: {} + snapshotter: + env: [] + resources: {} + livenessProbe: + resources: {} + resizer: + env: [] + resources: {} + nodeDriverRegistrar: + env: [] + resources: {} +``` + +#### Deploy driver with debug mode +To view driver debug logs, run the CSI driver with `-v=5` command line option + +To enable aws sdk debug logs, run the CSI driver with `--aws-sdk-debug-log=true` command line option. + +## Examples +Make sure you follow the [Prerequisites](README.md#Prerequisites) before the examples: +* [Dynamic Provisioning](../examples/kubernetes/dynamic-provisioning) +* [Block Volume](../examples/kubernetes/block-volume) +* [Volume Snapshot](../examples/kubernetes/snapshot) +* [Configure StorageClass](../examples/kubernetes/storageclass) +* [Volume Resizing](../examples/kubernetes/resizing) + + +## Migrating from in-tree EBS plugin +Starting from Kubernetes 1.17, CSI migration is supported as beta feature (alpha since 1.14). If you have persistent volumes that are created with in-tree `kubernetes.io/aws-ebs` plugin, you can migrate to use EBS CSI driver. To turn on the migration, set `CSIMigration` and `CSIMigrationAWS` feature gates to `true` for `kube-controller-manager`. Then drain Nodes and set the same feature gates to `true` for `kubelet`. + +To make sure dynamically provisioned EBS volumes have all tags that the in-tree volume plugin used: +* Run the external-provisioner sidecar with `--extra-create-metadata=true` cmdline option. The helm chart sets this option true by default. +* Run the CSI driver with `--k8s-tag-cluster-id=` command line option. + +To make sure that the CSI driver has permission to Attach, Detach, and Delete volumes that were dynamically provisioned and tagged by the in-tree plugin prior to migration being turned on, the IAM policy has to grant permission to operate on volumes with tag `kubernetes.io/cluster/": "owned"` like in [the example policy](./example-iam-policy.json#L85). + +**Warning**: +* kubelet *must* be drained of all pods with mounted EBS volumes ***before*** changing its CSI migration feature flags. Failure to do this will cause deleted pods to get stuck in `Terminating`, requiring a forced delete which can cause filesystem corruption. See [#679](../../../issues/679) for more details. + +## Development +Please go through [CSI Spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) and [General CSI driver development guideline](https://kubernetes-csi.github.io/docs/developing.html) to get some basic understanding of CSI driver before you start. + +### Requirements +* Golang 1.15.+ +* [Ginkgo](https://github.com/onsi/ginkgo) in your PATH for integration testing and end-to-end testing +* Docker 17.05+ for releasing + +### Dependency +Dependencies are managed through go module. To build the project, first turn on go mod using `export GO111MODULE=on`, then build the project using: `make` + +### Testing +* To execute all unit tests, run: `make test` +* To execute sanity test run: `make test-sanity` +* To execute integration tests, run: `make test-integration` +* To execute e2e tests, run: `make test-e2e-single-az` and `make test-e2e-multi-az` + +### Release Process +Please see [Release Process](./RELEASE.md). + +**Notes**: +* Sanity tests make sure the driver complies with the CSI specification +* EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk. See [Integration Testing](../tests/integration/README.md) for more details. +* E2E tests exercises various driver functionalities in Kubernetes cluster. See [E2E Testing](../tests/e2e/README.md) for more details. + +### Helm and manifests +The helm chart for this project is in the `charts/aws-ebs-csi-driver` directory. The manifests for this project are in the `deploy/kubernetes` directory. All of the manifests except kustomize patches are generated by running `helm template`. This keeps the helm chart and the manifests in sync. + +When updating the helm chart: +* Generate manifests: `make generate-kustomize` +* There are values files in `deploy/kubernetes/values` used for generating some of the manifests +* When adding a new resource template to the helm chart please update the `generate-kustomize` make target, the `deploy/kubernetes/values` files, and the appropriate kustomization.yaml file(s). + +## Milestone +[Milestones page](https://github.com/c2devel/aws-ebs-csi-driver/milestones) diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000000..f877417e66 --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,44 @@ +# Инструкция по релизу новой версии + +Инструкция протестирована на: +```sh +# uname -r +5.6.13-100.fc30.x86_64 +# cat /etc/os-release +NAME=Fedora +VERSION="30 (Thirty)" +ID=fedora +VERSION_ID=30 +VERSION_CODENAME="" +PLATFORM_ID="platform:f30" +PRETTY_NAME="Fedora 30 (Thirty)" +ANSI_COLOR="0;34" +LOGO=fedora-logo-icon +CPE_NAME="cpe:/o:fedoraproject:fedora:30" +HOME_URL="https://fedoraproject.org/" +DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/" +SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help" +BUG_REPORT_URL="https://bugzilla.redhat.com/" +REDHAT_BUGZILLA_PRODUCT="Fedora" +REDHAT_BUGZILLA_PRODUCT_VERSION=30 +REDHAT_SUPPORT_PRODUCT="Fedora" +REDHAT_SUPPORT_PRODUCT_VERSION=30 +PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy" +# docker --version +Docker version 19.03.12, build 48a66213fe +``` +## Версионирование + +Используется следующая схема версионирования - -CROC. Где X - инкрементируется с каждым новым релизом. Например при текущей версии апстрима v0.5.0 и текущей версии этой репы v0.5.0-CROC1 следующая версия будет v0.5.0-CROC2. При обновлении версии апстрима, например до v0.6.0, успешный ребейз на новый апстрим будет результирован в версию v0.6.0-CROC2. Предполагается суппорт только актуальных версий. + +Версии обозначаются гит тегами. Тегируется мастер ветка используя механизм релизов гитхаба. При создании нового релиза, описание релиза заполняется краткой сводкой изменений в новом релизе. После создания нового релиза (и тега), тег забирается на локалку (git pull upstream master --tags) и выполняется ручная сборка и публикация артефактов. + +## Артефакты + +Релизным артефактом этой репы является докер имадж. Для его создания необходимы установленный и настроенный докер демон - https://docs.docker.com/get-docker/ . Для сборки имаджа необходимо: +- находясь в руте репы выполнить: +```docker build -t aws-ebs-csi-driver``` +- после успешной сборки протегировать имадж: +```docker tag aws-ebs-csi-driver dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:``` +- запушить имадж в регистри (необходимы врайт права в регистри неймспейсе): +```docker push dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:``` From 821669a8eebf1e4b2fef07f5fa296808492974d2 Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 2 Jul 2020 14:09:20 +0300 Subject: [PATCH 14/45] deploy: change image tags to c2 dhub --- deploy/kubernetes/base/controller.yaml | 2 +- deploy/kubernetes/base/node.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index f2fc3d6fab..dc1da4087d 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -61,7 +61,7 @@ spec: runAsUser: 1000 containers: - name: ebs-plugin - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:latest + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.19.0 imagePullPolicy: IfNotPresent args: # - {all,controller,node} # specify the driver mode diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index 551a2c2614..95cc15e8b4 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -44,7 +44,7 @@ spec: runAsUser: 0 containers: - name: ebs-plugin - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:latest + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.19.0 imagePullPolicy: IfNotPresent args: - node From 1924669b0217893e47e9d558b846591b32928cbf Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 2 Jul 2020 14:09:44 +0300 Subject: [PATCH 15/45] deploy: regenerete deployment bundle --- deploy/kubernetes/overlays/stable/k_bundle.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index 318cbd52b3..b4906ef862 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -193,7 +193,7 @@ spec: key: access_key name: aws-secret optional: true - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0-CROC1 livenessProbe: failureThreshold: 5 httpGet: @@ -279,7 +279,7 @@ spec: value: unix:/csi/csi.sock - name: AWS_REGION value: croc - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0-CROC1 livenessProbe: failureThreshold: 5 httpGet: From 198e3965ebfa9ef3790397b82eef8af9dd20c18c Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 2 Jul 2020 14:11:01 +0300 Subject: [PATCH 16/45] docs: add info abot k_bundle file in release docs --- docs/RELEASE.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index f877417e66..d191714d41 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -26,6 +26,8 @@ REDHAT_SUPPORT_PRODUCT_VERSION=30 PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy" # docker --version Docker version 19.03.12, build 48a66213fe +# ./kustomize version +{Version:kustomize/v3.6.1 GitCommit:c97fa946d576eb6ed559f17f2ac43b3b5a8d5dbd BuildDate:2020-05-27T20:47:35Z GoOs:linux GoArch:amd64} ``` ## Версионирование @@ -35,7 +37,14 @@ Docker version 19.03.12, build 48a66213fe ## Артефакты -Релизным артефактом этой репы является докер имадж. Для его создания необходимы установленный и настроенный докер демон - https://docs.docker.com/get-docker/ . Для сборки имаджа необходимо: +Релизными артефактами этой репы является докер имадж и deployment конфиги для бубернетеса. При любом новом релизе необходимо обновлять kustomization.yaml и генерить бандл (например при релизе v0.5.0-CROC1): +- в файле deployment/kubernetes/stable/kustomization.yaml изменить ```newTag``` на новый актуальный (v0.5.0-CROC1) +- используя утилиту [kustomize](https://github.com/kubernetes-sigs/kustomize) собрать сингл-yaml-файл бандл для деплоймента: +``` +kustomize build ./deployment/kubernetes/stable/ > ./deployment/kubernetes/stable/k_bundle.yaml +``` + +Для создания докер имаджа необходимы установленный и настроенный докер демон - https://docs.docker.com/get-docker/ . Для сборки имаджа необходимо: - находясь в руте репы выполнить: ```docker build -t aws-ebs-csi-driver``` - после успешной сборки протегировать имадж: From 9f74f90df7e5885b30678900efda60556c26f475 Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 16 Jul 2020 23:43:35 +0300 Subject: [PATCH 17/45] cloud: remove deprecated volume types --- pkg/cloud/cloud.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index d5e9367bb1..5abf077495 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -85,14 +85,12 @@ const ( var ( // ValidVolumeTypes represents list of available volume types ValidVolumeTypes = []string{ - VolumeTypeIO1, VolumeTypeIO2, VolumeTypeGP2, VolumeTypeGP3, VolumeTypeSC1, VolumeTypeST1, VolumeTypeST2, - VolumeTypeStandard, } volumeModificationDuration = 1 * time.Second From 5aab3df942ed183e883cbf601448a0759100c9cd Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 16 Jul 2020 23:44:06 +0300 Subject: [PATCH 18/45] e2e driver: set valid minimal disk sizes --- tests/e2e/driver/ebs_csi_driver.go | 12 +++--------- tests/e2e/pre_provsioning.go | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/e2e/driver/ebs_csi_driver.go b/tests/e2e/driver/ebs_csi_driver.go index 7092d6baf4..b78e6d1ff3 100644 --- a/tests/e2e/driver/ebs_csi_driver.go +++ b/tests/e2e/driver/ebs_csi_driver.go @@ -126,16 +126,10 @@ func GetParameters(volumeType string, fsType string, encrypted bool) map[string] // MinimumSizeForVolumeType returns the minimum disk size for each volumeType func MinimumSizeForVolumeType(volumeType string) string { switch volumeType { - case "st1", "sc1": - return "500Gi" - case "gp2", "gp3": - return "1Gi" - case "io1", "io2": - return "4Gi" - case "standard": - return "10Gi" + case "st2": + return "32Gi" default: - return "1Gi" + return "8Gi" } } diff --git a/tests/e2e/pre_provsioning.go b/tests/e2e/pre_provsioning.go index 3bd221930f..61a8951305 100644 --- a/tests/e2e/pre_provsioning.go +++ b/tests/e2e/pre_provsioning.go @@ -36,8 +36,8 @@ import ( ) const ( - defaultDiskSize = 4 - defaultVoluemType = awscloud.VolumeTypeGP3 + defaultDiskSize = 8 + defaultVoluemType = awscloud.VolumeTypeGP2 awsAvailabilityZonesEnv = "AWS_AVAILABILITY_ZONES" From 07f6332a0f5bd8b618ed1d8cb2d7f92dc5cd37b1 Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 16 Jul 2020 23:44:36 +0300 Subject: [PATCH 19/45] e2e: change io1 to io2 volume type --- tests/e2e/dynamic_provisioning.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/dynamic_provisioning.go b/tests/e2e/dynamic_provisioning.go index c619acb34a..828c158a94 100644 --- a/tests/e2e/dynamic_provisioning.go +++ b/tests/e2e/dynamic_provisioning.go @@ -156,9 +156,9 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() { }, }, { - VolumeType: awscloud.VolumeTypeIO1, + VolumeType: awscloud.VolumeTypeIO2, FSType: ebscsidriver.FSTypeExt4, - ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO1), + ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO2), VolumeBindingMode: &volumeBindingMode, VolumeMount: testsuites.VolumeMountDetails{ NameGenerate: "test-volume-", @@ -195,9 +195,9 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() { Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", Volumes: []testsuites.VolumeDetails{ { - VolumeType: awscloud.VolumeTypeIO1, + VolumeType: awscloud.VolumeTypeIO2, FSType: ebscsidriver.FSTypeExt4, - ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO1), + ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO2), VolumeMount: testsuites.VolumeMountDetails{ NameGenerate: "test-volume-", MountPathGenerate: "/mnt/test-", @@ -245,9 +245,9 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() { Cmd: "dd if=/dev/zero of=/dev/xvda bs=1024k count=100 && echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", Volumes: []testsuites.VolumeDetails{ { - VolumeType: awscloud.VolumeTypeIO1, + VolumeType: awscloud.VolumeTypeIO2, FSType: ebscsidriver.FSTypeExt4, - ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO1), + ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO2), VolumeBindingMode: &volumeBindingMode, VolumeMount: testsuites.VolumeMountDetails{ NameGenerate: "test-volume-", @@ -296,9 +296,9 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() { Cmd: "while true; do echo $(date -u) >> /mnt/test-1/data; sleep 1; done", Volumes: []testsuites.VolumeDetails{ { - VolumeType: awscloud.VolumeTypeIO1, + VolumeType: awscloud.VolumeTypeIO2, FSType: ebscsidriver.FSTypeExt4, - ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO1), + ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeIO2), VolumeMount: testsuites.VolumeMountDetails{ NameGenerate: "test-volume-", MountPathGenerate: "/mnt/test-", From dde7593f1394369d502a29ebbaee72e8039929f2 Mon Sep 17 00:00:00 2001 From: Nikita Kretov Date: Thu, 16 Jul 2020 23:45:15 +0300 Subject: [PATCH 20/45] docs: add test docs --- docs/TEST.md | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 docs/TEST.md diff --git a/docs/TEST.md b/docs/TEST.md new file mode 100644 index 0000000000..7198d1201d --- /dev/null +++ b/docs/TEST.md @@ -0,0 +1,118 @@ +# Testing + +Типы тестов, которые есть в этой репе: +- unit +- e2e + +## unit + +Запускаются через ```make test```. Используют встроенную в го поддержку тестирования [вот](https://golang.org/doc/code.html#Testing). + +## e2e + +В апстриме используетс сложный агрегат под названием [aws-k8s-tester](https://github.com/aws/aws-k8s-tester). Этот инструкмент создает кластер, деплоит в него драйвер и запускает тесты. Инструкции по запуску тестов находятся в tester/*.yaml в секции test. Например в single-az-conifg.yaml инструкции следующие: + +``` +go get -u github.com/onsi/ginkgo/ginkgo +export KUBECONFIG=$HOME/.kube/config +export AWS_AVAILABILITY_ZONES=us-west-2a +$(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" tests/e2e -- -report-dir=$ARTIFACTS +``` +Как видно, это башовые командочки которые можно выполнить и без старшной штуки aws-k8s-tester. + +Структура e2e тестов: +``` +|-- e2e +| |-- driver +| | |-- driver.go +| | `-- ebs_csi_driver.go +| |-- dynamic_provisioning.go +| |-- pre_provsioning.go +| |-- README.md +| |-- reports +| | `-- junit_01.xml +| |-- suite_test.go +| `-- testsuites +| |-- dynamically_provisioned_cmd_volume_tester.go +| |-- dynamically_provisioned_collocated_pod_tester.go +| |-- dynamically_provisioned_delete_pod_tester.go +| |-- dynamically_provisioned_read_only_volume_tester.go +| |-- dynamically_provisioned_reclaim_policy_tester.go +| |-- dynamically_provisioned_topology_aware_volume_tester.go +| |-- dynamically_provisioned_volume_snapshot_tester.go +| |-- pre_provisioned_read_only_volume_tester.go +| |-- pre_provisioned_reclaim_policy_tester.go +| |-- pre_provisioned_volume_tester.go +| |-- specs.go +| `-- testsuites.go +|-- e2e-migration +| |-- e2e_test.go +| |-- go.mod +| |-- go.sum +| `-- README.md +`-- integration + |-- integration_test.go + |-- README.md + `-- setup_test.go +``` + +В директории tests есть e2e/e2e-migration/intergration субдиректории. Основная масса тестов находится в e2e. Тесты в е2е функциональные, в основном работют в api k8s и облака (лучшего облака - ц2 облака). + +Как запустить тесты +Для запуска тестов нам понадобится. +- [создать](https://docs.cloud.croc.ru/ru/services/kubernetes.html#creating) бубернетес кластер в ц2 кдауде +- попасть по ссш на мастер ноду и выполнить ```sudo -i``` +- проверить что в руте настроен kubectl - выполнить: ```kubectl get nodes``` +- установить голанг: +- - cd /tmp && curl -O https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz +- - tar -xzf go1.13.3.linux-amd64.tar.gz +- - mv go /usr/local +- - export GOROOT=/usr/local/go +- - export PATH=$GOROOT/bin:$PATH +- - cd - +- установить gcc (нужно для ginkgo): +- - yum install gcc +- склонить эту репу: +- - git clone https://github.com/c2devel/aws-ebs-csi-driver.git +- задать переменные окружения для подлкючения тестов к облаку: +- - export AWS_EC2_ENDPOINT="https://api.cloud.croc.ru" +- - export AWS_AVAILABILITY_ZONES="ru-msk-comp1p" +- - export AWS_SECRET_ACCESS_KEY="" +- - export AWS_ACCESS_KEY_ID="" +- установить ginkgo: +- - go get -u github.com/onsi/ginkgo/ginkgo +- задать переменные окружения для подлючения тестов к k8s: +- - export KUBECONFIG=$HOME/.kube/config +- запустить юнит тесты (проверить что код собирается) +- - cd +- - make test +- запустить e2e тесты: +- - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[single-az\]" /tests/e2e -- -report-dir=./reports/ + +Какие тесты есть: + +Красные: +- "should create a pod, write and read to it, take a volume snapshot, and create another pod from the snapshot" - снапшоттинг не реализован + +Пропущенные: +- все с тегами multi-az + +Зеленые: +- "should create a volume on demand with volume type %q and fs type %q", volumeType, fsType +- "should create a volume on demand with volumeType %q and encryption", volumeType +- "should create a volume on demand with provided mountOptions" +- "should create multiple PV objects, bind to PVCs and attach all to a single pod" +- "should create multiple PV objects, bind to PVCs and attach all to different pods" +- "should create a raw block volume on demand" +- "should create a raw block volume and a filesystem volume on demand and bind to the same pod" +- "should create multiple PV objects, bind to PVCs and attach all to different pods on the same node" +- "should create a volume on demand and mount it as readOnly in a pod" +- "should delete PV with reclaimPolicy %q", v1.PersistentVolumeReclaimDelete +- "[env] should retain PV with reclaimPolicy %q", v1.PersistentVolumeReclaimRetain +- "should create a deployment object, write and read to it, delete the pod and write and read to it again" +- "should allow for topology aware volume scheduling" +- "[env] should allow for topology aware volume with specified zone in allowedTopologies" +- "[env] should write and read to a pre-provisioned volume" +- "[env] should use a pre-provisioned volume and mount it as readOnly in a pod" +- "[env] should use a pre-provisioned volume and retain PV with reclaimPolicy %q", v1.PersistentVolumeReclaimRetain +- "[env] should use a pre-provisioned volume and delete PV with reclaimPolicy %q", v1.PersistentVolumeReclaimDelete" From cb04c19bc281590086d02ae8a33e46ba13635069 Mon Sep 17 00:00:00 2001 From: Aleksey Efimov Date: Wed, 1 Apr 2020 19:12:42 +0300 Subject: [PATCH 21/45] cloud: rewrite metadata aws region from env --- pkg/cloud/metadata_ec2.go | 7 ++++++- pkg/cloud/metadata_test.go | 40 +++++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pkg/cloud/metadata_ec2.go b/pkg/cloud/metadata_ec2.go index a9412b13df..ba5a0b3426 100644 --- a/pkg/cloud/metadata_ec2.go +++ b/pkg/cloud/metadata_ec2.go @@ -2,6 +2,7 @@ package cloud import ( "fmt" + "os" "strings" "github.com/aws/aws-sdk-go/aws" @@ -35,7 +36,7 @@ func EC2MetadataInstanceInfo(svc EC2Metadata, regionFromSession string) (*Metada return nil, fmt.Errorf("could not get valid EC2 instance type") } - if len(doc.Region) == 0 { + if len(doc.Region) == 0 && os.Getenv("AWS_REGION") == "" { if len(regionFromSession) != 0 && util.IsSBE(regionFromSession) { doc.Region = regionFromSession } else { @@ -51,6 +52,10 @@ func EC2MetadataInstanceInfo(svc EC2Metadata, regionFromSession string) (*Metada } } + if len(doc.Region) == 0 { + doc.Region = os.Getenv("AWS_REGION") + } + enis, err := svc.GetMetadata(enisEndpoint) if err != nil { return nil, fmt.Errorf("could not get number of attached ENIs: %w", err) diff --git a/pkg/cloud/metadata_test.go b/pkg/cloud/metadata_test.go index 16e4dccf37..ca82d322e5 100644 --- a/pkg/cloud/metadata_test.go +++ b/pkg/cloud/metadata_test.go @@ -41,6 +41,7 @@ const ( stdAvailabilityZone = "us-west-2b" snowRegion = "snow" snowAvailabilityZone = "snow" + envRegion = "instance-2" ) func TestNewMetadataService(t *testing.T) { @@ -66,6 +67,7 @@ func TestNewMetadataService(t *testing.T) { node v1.Node nodeNameEnvVar string regionFromSession string + isAwsRegionEnvSet bool }{ { name: "success: normal", @@ -76,8 +78,9 @@ func TestNewMetadataService(t *testing.T) { Region: stdRegion, AvailabilityZone: stdAvailabilityZone, }, - imdsENIOutput: "00:00:00:00:00:00", - expectedENIs: 1, + imdsENIOutput: "00:00:00:00:00:00", + expectedENIs: 1, + isAwsRegionEnvSet: false, }, { name: "success: outpost-arn is available", @@ -92,6 +95,7 @@ func TestNewMetadataService(t *testing.T) { expectedOutpostArn: validOutpostArn, imdsENIOutput: "00:00:00:00:00:00", expectedENIs: 1, + isAwsRegionEnvSet: false, }, { name: "success: outpost-arn is invalid", @@ -102,6 +106,7 @@ func TestNewMetadataService(t *testing.T) { Region: stdRegion, AvailabilityZone: stdAvailabilityZone, }, + isAwsRegionEnvSet: false, getMetadataValue: "foo", imdsENIOutput: "00:00:00:00:00:00", expectedENIs: 1, @@ -115,9 +120,10 @@ func TestNewMetadataService(t *testing.T) { Region: stdRegion, AvailabilityZone: stdAvailabilityZone, }, - getMetadataError: fmt.Errorf("404"), - imdsENIOutput: "00:00:00:00:00:00", - expectedENIs: 1, + getMetadataError: fmt.Errorf("404"), + imdsENIOutput: "00:00:00:00:00:00", + expectedENIs: 1, + isAwsRegionEnvSet: false, }, { name: "success: metadata not available, used k8s api", @@ -142,6 +148,7 @@ func TestNewMetadataService(t *testing.T) { }, expectedENIs: 1, nodeNameEnvVar: nodeName, + isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, k8s client error", @@ -153,6 +160,7 @@ func TestNewMetadataService(t *testing.T) { }, expectedErr: fmt.Errorf("error getting Node %s: client failure", nodeName), nodeNameEnvVar: nodeName, + isAwsRegionEnvSet: false, }, { @@ -160,6 +168,7 @@ func TestNewMetadataService(t *testing.T) { ec2metadataAvailable: false, expectedErr: fmt.Errorf("CSI_NODE_NAME env var not set"), nodeNameEnvVar: "", + isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, no provider ID", @@ -179,6 +188,7 @@ func TestNewMetadataService(t *testing.T) { Status: v1.NodeStatus{}, }, nodeNameEnvVar: nodeName, + isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, could not retrieve region", @@ -202,6 +212,7 @@ func TestNewMetadataService(t *testing.T) { Status: v1.NodeStatus{}, }, nodeNameEnvVar: nodeName, + isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, could not retrieve AZ", @@ -225,6 +236,7 @@ func TestNewMetadataService(t *testing.T) { Status: v1.NodeStatus{}, }, nodeNameEnvVar: nodeName, + isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, invalid instance id", @@ -249,12 +261,14 @@ func TestNewMetadataService(t *testing.T) { Status: v1.NodeStatus{}, }, nodeNameEnvVar: nodeName, + isAwsRegionEnvSet: false, }, { name: "fail: GetInstanceIdentityDocument returned error", ec2metadataAvailable: true, getInstanceIdentityDocumentError: fmt.Errorf("foo"), expectedErr: fmt.Errorf("could not get EC2 instance identity metadata: foo"), + isAwsRegionEnvSet: false, }, { name: "fail: GetInstanceIdentityDocument returned empty instance", @@ -267,6 +281,7 @@ func TestNewMetadataService(t *testing.T) { }, invalidInstanceIdentityDocument: true, expectedErr: fmt.Errorf("could not get valid EC2 instance ID"), + isAwsRegionEnvSet: false, }, { name: "fail: GetInstanceIdentityDocument returned empty region", @@ -279,6 +294,7 @@ func TestNewMetadataService(t *testing.T) { }, invalidInstanceIdentityDocument: true, expectedErr: fmt.Errorf("could not get valid EC2 region"), + isAwsRegionEnvSet: false, }, { name: "fail: GetInstanceIdentityDocument returned empty az", @@ -291,6 +307,7 @@ func TestNewMetadataService(t *testing.T) { }, invalidInstanceIdentityDocument: true, expectedErr: fmt.Errorf("could not get valid EC2 availability zone"), + isAwsRegionEnvSet: false, }, { name: "fail: outpost-arn failed", @@ -305,6 +322,7 @@ func TestNewMetadataService(t *testing.T) { expectedENIs: 1, getMetadataError: fmt.Errorf("405"), expectedErr: fmt.Errorf("something went wrong while getting EC2 outpost arn: 405"), + isAwsRegionEnvSet: false, }, { name: "success: GetMetadata() returns correct number of ENIs", @@ -359,6 +377,10 @@ func TestNewMetadataService(t *testing.T) { mockCtrl := gomock.NewController(t) mockEC2Metadata := NewMockEC2Metadata(mockCtrl) + if tc.isAwsRegionEnvSet { + os.Setenv("AWS_REGION", envRegion) + } + ec2MetadataClient := func() (EC2Metadata, error) { return mockEC2Metadata, nil } k8sAPIClient := func() (kubernetes.Interface, error) { clientsetInitialized = true; return clientset, nil } @@ -428,7 +450,15 @@ func TestNewMetadataService(t *testing.T) { if m.GetNumBlockDeviceMappings() != tc.expectedBlockDevices { t.Errorf("GetMetadata() failed for %s: got %v, expected %v", blockDevicesEndpoint, m.GetNumBlockDeviceMappings(), tc.expectedBlockDevices) } + if tc.isAwsRegionEnvSet && m.GetRegion() != envRegion { + t.Fatalf("GetRegion() failed: expected %v, got %v", envRegion, m.GetRegion()) + } + if !tc.isAwsRegionEnvSet && m.GetRegion() != stdRegion { + t.Fatalf("GetRegion() failed: expected %v, got %v", stdRegion, m.GetRegion()) + } } + + os.Unsetenv("AWS_REGION") mockCtrl.Finish() }) } From 1385876cc123b218a730e8ae268fa5ac6dae0875 Mon Sep 17 00:00:00 2001 From: Rouslan Gaisin Date: Fri, 10 Sep 2021 15:46:46 +0300 Subject: [PATCH 22/45] ignore VolumeIsNotAttached error while detaching disk --- pkg/cloud/cloud.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 5abf077495..683ce16061 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -799,6 +799,10 @@ func (c *cloud) DetachDisk(ctx context.Context, volumeID, nodeID string) error { _, err = c.ec2.DetachVolumeWithContext(ctx, request) if err != nil { + if isAWSErrorVolumeIsNotAttached(err) { + klog.Warningf("Volume %v was already detached from node %v, ignoring...", volumeID, nodeID) + return nil + } if isAWSErrorIncorrectState(err) || isAWSErrorInvalidAttachmentNotFound(err) || isAWSErrorVolumeNotFound(err) { @@ -1339,6 +1343,10 @@ func isAWSErrorIdempotentParameterMismatch(err error) bool { return isAWSError(err, "IdempotentParameterMismatch") } +func isAWSErrorVolumeIsNotAttached(err error) bool { + return isAWSError(err, "VolumeIsNotAttached") +} + // ResizeDisk resizes an EBS volume in GiB increments, rouding up to the next possible allocatable unit. // It returns the volume size after this call or an error if the size couldn't be determined. func (c *cloud) ResizeDisk(ctx context.Context, volumeID string, newSizeBytes int64) (int64, error) { From c555614be1bbff6cbc6689e20642626e92941331 Mon Sep 17 00:00:00 2001 From: Rouslan Gaisin Date: Fri, 10 Sep 2021 15:48:01 +0300 Subject: [PATCH 23/45] use VolumeID instead of Device --- pkg/cloud/cloud.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 683ce16061..46a99f7759 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -761,7 +761,9 @@ func (c *cloud) AttachDisk(ctx context.Context, volumeID, nodeID string) (string // Impossible? return "", fmt.Errorf("unexpected state: attachment nil after attached %q to %q", volumeID, nodeID) } - if device.Path != aws.StringValue(attachment.Device) { + + // using VolumeID instead of Device, because Device attribute can be changed + if device.VolumeID != aws.StringValue(attachment.VolumeId) { // Already checked in waitForAttachmentState(), but just to be sure... return "", fmt.Errorf("disk attachment of %q to %q failed: requested device %q but found %q", volumeID, nodeID, device.Path, aws.StringValue(attachment.Device)) } @@ -890,7 +892,9 @@ func (c *cloud) WaitForAttachmentState(ctx context.Context, volumeID, expectedSt // For example, we're waiting for a volume to be attached as /dev/xvdba, but AWS can tell us it's // attached as /dev/xvdbb, where it was attached before and it was already detached. // Retry couple of times, hoping AWS starts reporting the right status. - device := aws.StringValue(attachment.Device) + device := aws.StringValue(attachment.VolumeId) + const devPreffix = "/dev/disk/by-id/virtio-" + expectedDevice = strings.TrimPrefix(expectedDevice, devPreffix) if expectedDevice != "" && device != "" && device != expectedDevice { klog.InfoS("Expected device for volume not found", "expectedDevice", expectedDevice, "expectedState", expectedState, "volumeID", volumeID, "device", device, "attachmentState", attachmentState) return false, nil From ae156ed02131425939908d03b2314e0a8e65b9b0 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Wed, 29 Sep 2021 12:17:39 +0300 Subject: [PATCH 24/45] deploy: prepare kustomization for the release, del workflow --- .github/workflows/helm-chart-release.yaml | 27 ----------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/helm-chart-release.yaml diff --git a/.github/workflows/helm-chart-release.yaml b/.github/workflows/helm-chart-release.yaml deleted file mode 100644 index 579a6ad2ad..0000000000 --- a/.github/workflows/helm-chart-release.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Release Helm Charts - -on: - push: - branches: - - master - paths: - - "charts/**/Chart.yaml" - -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.2.1 - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - with: - config: .github/cr.yaml From 72c0c34f3b5a164c400d2d05458b4ecde33b1fc1 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Wed, 29 Sep 2021 12:26:58 +0300 Subject: [PATCH 25/45] deploy: update k_bundle --- .../kubernetes/overlays/stable/k_bundle.yaml | 303 ++++++++++++++++-- 1 file changed, 275 insertions(+), 28 deletions(-) diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index b4906ef862..cd031d1945 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -1,12 +1,24 @@ apiVersion: v1 kind: ServiceAccount metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-controller-sa namespace: kube-system --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-attacher-role rules: - apiGroups: @@ -18,6 +30,7 @@ rules: - list - watch - update + - patch - apiGroups: - "" resources: @@ -43,10 +56,19 @@ rules: - list - watch - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-external-provisioner-role rules: - apiGroups: @@ -127,10 +149,129 @@ rules: - delete - update - create +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-attacher-binding roleRef: apiGroup: rbac.authorization.k8s.io @@ -144,6 +285,8 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-provisioner-binding roleRef: apiGroup: rbac.authorization.k8s.io @@ -154,9 +297,41 @@ subjects: name: ebs-csi-controller-sa namespace: kube-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +--- apiVersion: apps/v1 kind: Deployment metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-controller namespace: kube-system spec: @@ -164,19 +339,25 @@ spec: selector: matchLabels: app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver template: metadata: labels: app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver spec: containers: - args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr - - --v=5 + - --v=2 env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: AWS_EC2_ENDPOINT value: https://api.cloud.croc.ru - name: AWS_REGION @@ -193,7 +374,8 @@ spec: key: access_key name: aws-secret optional: true - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0-CROC1 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC2 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 httpGet: @@ -207,79 +389,145 @@ spec: - containerPort: 9808 name: healthz protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - - --v=5 + - --v=2 - --feature-gates=Topology=true - - --enable-leader-election - - --leader-election-type=leases + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: dhub.c2.croc.ru/kaas/csi-provisioner:v1.3.0 + image: dhub.c2.croc.ru/kaas/csi-provisioner:v2.1.1 name: csi-provisioner volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - - --v=5 + - --v=2 - --leader-election=true - - --leader-election-type=leases env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: dhub.c2.croc.ru/kaas/csi-attacher:v1.2.0 + image: dhub.c2.croc.ru/kaas/csi-attacher:v3.1.0 name: csi-attacher volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: dhub.c2.croc.ru/kaas/csi-snapshotter:v3.0.3 + name: csi-snapshotter + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: dhub.c2.croc.ru/kaas/csi-resizer:v1.0.0 + imagePullPolicy: Always + name: csi-resizer + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir - args: - --csi-address=/csi/csi.sock - image: dhub.c2.croc.ru/kaas/livenessprobe:v1.1.0 + image: dhub.c2.croc.ru/kaas/livenessprobe:v2.2.0 name: liveness-probe volumeMounts: - mountPath: /csi name: socket-dir nodeSelector: - beta.kubernetes.io/os: linux + kubernetes.io/os: linux priorityClassName: system-cluster-critical - serviceAccount: ebs-csi-controller-sa + serviceAccountName: ebs-csi-controller-sa tolerations: - key: CriticalAddonsOnly operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 volumes: - emptyDir: {} name: socket-dir --- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver +--- apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs-csi-node namespace: kube-system spec: selector: matchLabels: app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver template: metadata: labels: app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate containers: - args: + - node - --endpoint=$(CSI_ENDPOINT) - --logtostderr - - --v=5 + - --v=2 env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: AWS_REGION value: croc - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v0.5.0-CROC1 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC2 livenessProbe: failureThreshold: 5 httpGet: @@ -306,20 +554,13 @@ spec: - args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=5 + - --v=2 env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - image: dhub.c2.croc.ru/kaas/csi-node-driver-registrar:v1.1.0 - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock + image: dhub.c2.croc.ru/kaas/csi-node-driver-registrar:v2.1.0 name: node-driver-registrar volumeMounts: - mountPath: /csi @@ -328,17 +569,21 @@ spec: name: registration-dir - args: - --csi-address=/csi/csi.sock - image: dhub.c2.croc.ru/kaas/livenessprobe:v1.1.0 + image: dhub.c2.croc.ru/kaas/livenessprobe:v2.2.0 name: liveness-probe volumeMounts: - mountPath: /csi name: plugin-dir - hostNetwork: true nodeSelector: - beta.kubernetes.io/os: linux + kubernetes.io/os: linux priorityClassName: system-node-critical + serviceAccountName: ebs-csi-node-sa tolerations: - - operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 volumes: - hostPath: path: /var/lib/kubelet @@ -357,9 +602,11 @@ spec: type: Directory name: device-dir --- -apiVersion: storage.k8s.io/v1beta1 +apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver name: ebs.csi.aws.com spec: attachRequired: true From 4c64190423a08fd9ea12afffd7dff6ba40d50bed Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Thu, 21 Oct 2021 14:57:44 +0300 Subject: [PATCH 26/45] tests: fix failures, fix vendoring --- go.sum | 4 ---- pkg/cloud/cloud_test.go | 20 ++++++++++---------- pkg/cloud/metadata_test.go | 33 ++++++++++++++++----------------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/go.sum b/go.sum index e3694da6f3..c1e6e15a9c 100644 --- a/go.sum +++ b/go.sum @@ -193,7 +193,6 @@ github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -375,7 +374,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -644,7 +642,6 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -728,7 +725,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index 8868a184b1..ef885e4816 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -711,7 +711,7 @@ func TestDeleteDisk(t *testing.T) { } func TestAttachDisk(t *testing.T) { - t.Skip("Skipping temporarily due to interface inconsistency.") + t.Skip("Skipping temporarily due to interface inconsistency.") testCases := []struct { name string @@ -1867,15 +1867,15 @@ func TestWaitForAttachmentState(t *testing.T) { alreadyAssigned: false, expectError: true, }, - { - name: "failure: unexpected device", - volumeID: "vol-test-1234", - expectedState: volumeAttachedState, - expectedInstance: "1234", - expectedDevice: "/dev/xvdc", - alreadyAssigned: false, - expectError: true, - }, + // { + // name: "failure: unexpected device", + // volumeID: "vol-test-1234", + // expectedState: volumeAttachedState, + // expectedInstance: "1234", + // expectedDevice: "/dev/xvdc", + // alreadyAssigned: false, + // expectError: true, + // }, { name: "failure: unexpected instance", volumeID: "vol-test-1234", diff --git a/pkg/cloud/metadata_test.go b/pkg/cloud/metadata_test.go index ca82d322e5..3b033e1212 100644 --- a/pkg/cloud/metadata_test.go +++ b/pkg/cloud/metadata_test.go @@ -107,9 +107,9 @@ func TestNewMetadataService(t *testing.T) { AvailabilityZone: stdAvailabilityZone, }, isAwsRegionEnvSet: false, - getMetadataValue: "foo", - imdsENIOutput: "00:00:00:00:00:00", - expectedENIs: 1, + getMetadataValue: "foo", + imdsENIOutput: "00:00:00:00:00:00", + expectedENIs: 1, }, { name: "success: outpost-arn is not found", @@ -148,7 +148,6 @@ func TestNewMetadataService(t *testing.T) { }, expectedENIs: 1, nodeNameEnvVar: nodeName, - isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, k8s client error", @@ -158,8 +157,8 @@ func TestNewMetadataService(t *testing.T) { return true, nil, fmt.Errorf("client failure") }) }, - expectedErr: fmt.Errorf("error getting Node %s: client failure", nodeName), - nodeNameEnvVar: nodeName, + expectedErr: fmt.Errorf("error getting Node %s: client failure", nodeName), + nodeNameEnvVar: nodeName, isAwsRegionEnvSet: false, }, @@ -168,7 +167,7 @@ func TestNewMetadataService(t *testing.T) { ec2metadataAvailable: false, expectedErr: fmt.Errorf("CSI_NODE_NAME env var not set"), nodeNameEnvVar: "", - isAwsRegionEnvSet: false, + isAwsRegionEnvSet: false, }, { name: "failure: metadata not available, no provider ID", @@ -187,7 +186,7 @@ func TestNewMetadataService(t *testing.T) { }, Status: v1.NodeStatus{}, }, - nodeNameEnvVar: nodeName, + nodeNameEnvVar: nodeName, isAwsRegionEnvSet: false, }, { @@ -211,7 +210,7 @@ func TestNewMetadataService(t *testing.T) { }, Status: v1.NodeStatus{}, }, - nodeNameEnvVar: nodeName, + nodeNameEnvVar: nodeName, isAwsRegionEnvSet: false, }, { @@ -235,7 +234,7 @@ func TestNewMetadataService(t *testing.T) { }, Status: v1.NodeStatus{}, }, - nodeNameEnvVar: nodeName, + nodeNameEnvVar: nodeName, isAwsRegionEnvSet: false, }, { @@ -260,7 +259,7 @@ func TestNewMetadataService(t *testing.T) { }, Status: v1.NodeStatus{}, }, - nodeNameEnvVar: nodeName, + nodeNameEnvVar: nodeName, isAwsRegionEnvSet: false, }, { @@ -294,7 +293,7 @@ func TestNewMetadataService(t *testing.T) { }, invalidInstanceIdentityDocument: true, expectedErr: fmt.Errorf("could not get valid EC2 region"), - isAwsRegionEnvSet: false, + isAwsRegionEnvSet: false, }, { name: "fail: GetInstanceIdentityDocument returned empty az", @@ -307,7 +306,7 @@ func TestNewMetadataService(t *testing.T) { }, invalidInstanceIdentityDocument: true, expectedErr: fmt.Errorf("could not get valid EC2 availability zone"), - isAwsRegionEnvSet: false, + isAwsRegionEnvSet: false, }, { name: "fail: outpost-arn failed", @@ -318,10 +317,10 @@ func TestNewMetadataService(t *testing.T) { Region: stdRegion, AvailabilityZone: stdAvailabilityZone, }, - imdsENIOutput: "00:00:00:00:00:00", - expectedENIs: 1, - getMetadataError: fmt.Errorf("405"), - expectedErr: fmt.Errorf("something went wrong while getting EC2 outpost arn: 405"), + imdsENIOutput: "00:00:00:00:00:00", + expectedENIs: 1, + getMetadataError: fmt.Errorf("405"), + expectedErr: fmt.Errorf("something went wrong while getting EC2 outpost arn: 405"), isAwsRegionEnvSet: false, }, { From 00c0992227e190c7cc909a2496a995b11d7f9462 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Thu, 21 Oct 2021 17:07:18 +0300 Subject: [PATCH 27/45] cloud: delete exceed volume types, fix io2 iops count --- pkg/cloud/cloud.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 46a99f7759..894a0be606 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -25,8 +25,8 @@ import ( "fmt" "net/http" "os" - "strings" "strconv" + "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -74,9 +74,9 @@ const ( io1MaxTotalIOPS = 64000 io1MaxIOPSPerGB = 50 io2MinTotalIOPS = 100 - io2MaxTotalIOPS = 64000 + io2MaxTotalIOPS = 50000 io2BlockExpressMaxTotalIOPS = 256000 - io2MaxIOPSPerGB = 500 + io2MaxIOPSPerGB = 50 gp3MaxTotalIOPS = 16000 gp3MinTotalIOPS = 3000 gp3MaxIOPSPerGB = 500 @@ -87,9 +87,6 @@ var ( ValidVolumeTypes = []string{ VolumeTypeIO2, VolumeTypeGP2, - VolumeTypeGP3, - VolumeTypeSC1, - VolumeTypeST1, VolumeTypeST2, } @@ -126,7 +123,9 @@ const ( // Defaults const ( // DefaultVolumeSize represents the default volume size. - DefaultVolumeSize int64 = 100 * util.GiB + DefaultVolumeSize int64 = 32 * util.GiB + // DefaultVolumeType specifies which storage to use for newly created Volumes. + DefaultVolumeType = VolumeTypeGP2 ) // Tags @@ -459,7 +458,7 @@ func newEC2Cloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Clo Region: aws.String(region), CredentialsChainVerboseErrors: aws.Bool(true), // Set MaxRetries to a high value. It will be "ovewritten" if context deadline comes sooner. - MaxRetries: aws.Int(8), + MaxRetries: aws.Int(8), } } From 371ff574078b940561714e213e25ad7ea9db8cf8 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Fri, 22 Oct 2021 11:38:32 +0300 Subject: [PATCH 28/45] specs: change the supported snapshot api version in tests --- tests/e2e/driver/driver.go | 2 +- tests/e2e/dynamic_provisioning.go | 2 +- tests/e2e/pre_provsioning.go | 2 +- tests/e2e/testsuites/specs.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/driver/driver.go b/tests/e2e/driver/driver.go index 7f1cbc0760..b4131904da 100644 --- a/tests/e2e/driver/driver.go +++ b/tests/e2e/driver/driver.go @@ -23,7 +23,7 @@ import ( const ( VolumeSnapshotClassKind = "VolumeSnapshotClass" - SnapshotAPIVersion = "snapshot.storage.k8s.io/v1" + SnapshotAPIVersion = "snapshot.storage.k8s.io/v1beta1" ) type PVTestDriver interface { diff --git a/tests/e2e/dynamic_provisioning.go b/tests/e2e/dynamic_provisioning.go index 828c158a94..9b1063a9eb 100644 --- a/tests/e2e/dynamic_provisioning.go +++ b/tests/e2e/dynamic_provisioning.go @@ -452,7 +452,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Snapshot", func() { BeforeEach(func() { cs = f.ClientSet var err error - snapshotrcs, err = restClient(testsuites.SnapshotAPIGroup, testsuites.APIVersionv1) + snapshotrcs, err = restClient(testsuites.SnapshotAPIGroup, testsuites.APIVersionv1beta1) if err != nil { Fail(fmt.Sprintf("could not get rest clientset: %v", err)) } diff --git a/tests/e2e/pre_provsioning.go b/tests/e2e/pre_provsioning.go index 61a8951305..ed805364ba 100644 --- a/tests/e2e/pre_provsioning.go +++ b/tests/e2e/pre_provsioning.go @@ -99,7 +99,7 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Pre-Provisioned", func() { } volumeID = disk.VolumeID diskSize = fmt.Sprintf("%dGi", defaultDiskSize) - snapshotrcs, err = restClient(testsuites.SnapshotAPIGroup, testsuites.APIVersionv1) + snapshotrcs, err = restClient(testsuites.SnapshotAPIGroup, testsuites.APIVersionv1beta1) if err != nil { Fail(fmt.Sprintf("could not get rest clientset: %v", err)) } diff --git a/tests/e2e/testsuites/specs.go b/tests/e2e/testsuites/specs.go index d268a99c7c..a25a2574ec 100644 --- a/tests/e2e/testsuites/specs.go +++ b/tests/e2e/testsuites/specs.go @@ -61,8 +61,8 @@ const ( const ( VolumeSnapshotKind = "VolumeSnapshot" VolumeSnapshotContentKind = "VolumeSnapshotContent" - SnapshotAPIVersion = "snapshot.storage.k8s.io/v1" - APIVersionv1 = "v1" + SnapshotAPIVersion = "snapshot.storage.k8s.io/v1beta1" + APIVersionv1beta1 = "v1beta1" ) var ( From 15b897758074ff55467cf098df8f8e7d4e9ad1a1 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Fri, 22 Oct 2021 17:51:29 +0300 Subject: [PATCH 29/45] doc: change the documentation to actual --- docs/TEST.md | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/TEST.md b/docs/TEST.md index 7198d1201d..a11d9bb713 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -58,6 +58,9 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t В директории tests есть e2e/e2e-migration/intergration субдиректории. Основная масса тестов находится в e2e. Тесты в е2е функциональные, в основном работют в api k8s и облака (лучшего облака - ц2 облака). +Для выполнения тестов потребуется: +- Для single-az 1 нода мастер и 1 воркер +- Для multi-az 3 ноды мастеров в разных аз и 3 воркера в разных аз Как запустить тесты Для запуска тестов нам понадобится. - [создать](https://docs.cloud.croc.ru/ru/services/kubernetes.html#creating) бубернетес кластер в ц2 кдауде @@ -86,33 +89,40 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t - запустить юнит тесты (проверить что код собирается) - - cd - - make test -- запустить e2e тесты: -- - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[single-az\]" /tests/e2e -- -report-dir=./reports/ - +- запустить e2e тесты для single az: +- - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[single-az\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config +- запустить e2e тесты для multi az: +- - export AWS_AVAILABILITY_ZONES="ru-msk-comp1p,ru-msk-vol51,ru-msk-vol52" +- - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[multi-az\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config Какие тесты есть: Красные: -- "should create a pod, write and read to it, take a volume snapshot, and create another pod from the snapshot" - снапшоттинг не реализован - -Пропущенные: -- все с тегами multi-az +- "should create a volume on demand and resize it" - не реализован метод describe-volumes-modifications Зеленые: -- "should create a volume on demand with volume type %q and fs type %q", volumeType, fsType -- "should create a volume on demand with volumeType %q and encryption", volumeType -- "should create a volume on demand with provided mountOptions" +- "should use a pre-defined snapshot and create pv from that" +- "should create a pod, write and read to it, take a volume snapshot, and create another pod from the snapshot" +- "should create a volume on demand with volumeType "gp2" and encryption" +- "should create a volume on demand with volumeType "st2" and encryption" +- "should create a volume on demand with volume type "gp2" and fs type "xfs"" +- "should create a volume on demand with volume type "st2" and fs type "xfs"" +- "should create a volume on demand with volume type "io2" and fs type "xfs"" +- "should create a volume on demand with volumeType "io2" and encryption" - "should create multiple PV objects, bind to PVCs and attach all to a single pod" - "should create multiple PV objects, bind to PVCs and attach all to different pods" - "should create a raw block volume on demand" - "should create a raw block volume and a filesystem volume on demand and bind to the same pod" - "should create multiple PV objects, bind to PVCs and attach all to different pods on the same node" - "should create a volume on demand and mount it as readOnly in a pod" -- "should delete PV with reclaimPolicy %q", v1.PersistentVolumeReclaimDelete -- "[env] should retain PV with reclaimPolicy %q", v1.PersistentVolumeReclaimRetain +- "should delete PV with reclaimPolicy "Delete"" +- "[env] should retain PV with reclaimPolicy "Retain"" - "should create a deployment object, write and read to it, delete the pod and write and read to it again" - "should allow for topology aware volume scheduling" - "[env] should allow for topology aware volume with specified zone in allowedTopologies" - "[env] should write and read to a pre-provisioned volume" - "[env] should use a pre-provisioned volume and mount it as readOnly in a pod" -- "[env] should use a pre-provisioned volume and retain PV with reclaimPolicy %q", v1.PersistentVolumeReclaimRetain -- "[env] should use a pre-provisioned volume and delete PV with reclaimPolicy %q", v1.PersistentVolumeReclaimDelete" +- "[env] should use a pre-provisioned volume and retain PV with reclaimPolicy "Retain"" +- "[env] should use a pre-provisioned volume and delete PV with reclaimPolicy "Delete"" +- "[env] should allow for topology aware volume with specified zone in allowedTopologies" +- "should allow for topology aware volume scheduling" +- "should create a volume on demand with provided mountOptions" \ No newline at end of file From 9ae158f0b230ee51a70de2d742b1bef61053fdd7 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Wed, 10 Nov 2021 10:01:27 +0300 Subject: [PATCH 30/45] doc: change the go version to actual --- docs/TEST.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/TEST.md b/docs/TEST.md index a11d9bb713..6e732c2412 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -67,8 +67,8 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t - попасть по ссш на мастер ноду и выполнить ```sudo -i``` - проверить что в руте настроен kubectl - выполнить: ```kubectl get nodes``` - установить голанг: -- - cd /tmp && curl -O https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz -- - tar -xzf go1.13.3.linux-amd64.tar.gz +- - cd /tmp && curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz +- - tar -xzf go1.16.10.linux-amd64.tar.gz - - mv go /usr/local - - export GOROOT=/usr/local/go - - export PATH=$GOROOT/bin:$PATH From 7cb687fdff2671e2ae5373f679d6de9648c9b994 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Wed, 10 Nov 2021 10:22:17 +0300 Subject: [PATCH 31/45] tests: fix unit io2 tests --- pkg/cloud/cloud_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index ef885e4816..6aec6e442e 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -111,18 +111,18 @@ func TestCreateDisk(t *testing.T) { name: "success: normal with io2 options", volumeName: "vol-test-name", diskOptions: &DiskOptions{ - CapacityBytes: util.GiBToBytes(1), + CapacityBytes: util.GiBToBytes(8), Tags: map[string]string{VolumeNameTagKey: "vol-test", AwsEbsDriverTagKey: "true"}, VolumeType: VolumeTypeIO2, - IOPSPerGB: 100, + IOPSPerGB: 50, }, expDisk: &Disk{ VolumeID: "vol-test", - CapacityGiB: 1, + CapacityGiB: 8, AvailabilityZone: defaultZone, }, expCreateVolumeInput: &ec2.CreateVolumeInput{ - Iops: aws.Int64(100), + Iops: aws.Int64(400), }, expErr: nil, }, @@ -479,7 +479,7 @@ func TestCreateDisk(t *testing.T) { AvailabilityZone: defaultZone, }, expCreateVolumeInput: &ec2.CreateVolumeInput{ - Iops: aws.Int64(2000), + Iops: aws.Int64(200), }, expErr: nil, }, @@ -498,7 +498,7 @@ func TestCreateDisk(t *testing.T) { AvailabilityZone: defaultZone, }, expCreateVolumeInput: &ec2.CreateVolumeInput{ - Iops: aws.Int64(64000), + Iops: aws.Int64(50000), }, expErr: nil, }, From e3eef4ca9e75da66e6e888ab68f5739bb01cf784 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Wed, 10 Nov 2021 10:45:51 +0300 Subject: [PATCH 32/45] driver: change max iops per Gb --- tests/e2e/driver/ebs_csi_driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/driver/ebs_csi_driver.go b/tests/e2e/driver/ebs_csi_driver.go index b78e6d1ff3..5fc32001d8 100644 --- a/tests/e2e/driver/ebs_csi_driver.go +++ b/tests/e2e/driver/ebs_csi_driver.go @@ -141,8 +141,8 @@ func IOPSPerGBForVolumeType(volumeType string) string { // Maximum IOPS/GB for io1 is 50 return "50" case "io2": - // Maximum IOPS/GB for io2 is 500 - return "500" + // Maximum IOPS/GB for io2 is 50 + return "50" default: return "" } From 223b3242d926600ec416422f6a1b8c49ab50ce76 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Wed, 10 Nov 2021 11:35:00 +0300 Subject: [PATCH 33/45] docs: add temporary workaround --- docs/TEST.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/TEST.md b/docs/TEST.md index 6e732c2412..011de05c11 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -60,15 +60,15 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t Для выполнения тестов потребуется: - Для single-az 1 нода мастер и 1 воркер -- Для multi-az 3 ноды мастеров в разных аз и 3 воркера в разных аз +- Для multi-az 3 ноды мастеров в разных аз и 1 воркер в одной аз Как запустить тесты Для запуска тестов нам понадобится. - [создать](https://docs.cloud.croc.ru/ru/services/kubernetes.html#creating) бубернетес кластер в ц2 кдауде - попасть по ссш на мастер ноду и выполнить ```sudo -i``` - проверить что в руте настроен kubectl - выполнить: ```kubectl get nodes``` - установить голанг: -- - cd /tmp && curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz -- - tar -xzf go1.16.10.linux-amd64.tar.gz +- - cd /tmp && curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz +- - tar -xzf go1.16.9.linux-amd64.tar.gz - - mv go /usr/local - - export GOROOT=/usr/local/go - - export PATH=$GOROOT/bin:$PATH @@ -80,19 +80,20 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t - задать переменные окружения для подлкючения тестов к облаку: - - export AWS_EC2_ENDPOINT="https://api.cloud.croc.ru" - - export AWS_AVAILABILITY_ZONES="ru-msk-comp1p" +- - Воркер должен быть в той же аз что и указана - - export AWS_SECRET_ACCESS_KEY="" - - export AWS_ACCESS_KEY_ID="" -- установить ginkgo: -- - go get -u github.com/onsi/ginkgo/ginkgo - задать переменные окружения для подлючения тестов к k8s: - - export KUBECONFIG=$HOME/.kube/config - запустить юнит тесты (проверить что код собирается) - - cd +- - Выполнить ```go get -u modernc.org/cc@v1.0.0``` (временный воркераунд, связанный с недоступностью go зависимостей) - - make test +- установить ginkgo: +- - go get github.com/onsi/ginkgo/ginkgo@v1.11.0 - запустить e2e тесты для single az: - - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[single-az\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config - запустить e2e тесты для multi az: -- - export AWS_AVAILABILITY_ZONES="ru-msk-comp1p,ru-msk-vol51,ru-msk-vol52" - - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[multi-az\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config Какие тесты есть: From 837adc3f040de457c97351a4c6b374395073e096 Mon Sep 17 00:00:00 2001 From: Alexandr Chernev Date: Thu, 11 Nov 2021 15:27:08 +0300 Subject: [PATCH 34/45] deploy: bump version --- deploy/kubernetes/overlays/stable/k_bundle.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index cd031d1945..17313b5d1a 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -374,7 +374,7 @@ spec: key: access_key name: aws-secret optional: true - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC2 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC3 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -527,7 +527,7 @@ spec: fieldPath: spec.nodeName - name: AWS_REGION value: croc - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC2 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC3 livenessProbe: failureThreshold: 5 httpGet: From b6a8f282d5cf94fc626c7b2e6eae57b0d5c6f7e3 Mon Sep 17 00:00:00 2001 From: Margarita Fedotova Date: Wed, 23 Mar 2022 17:15:26 +0300 Subject: [PATCH 35/45] controller, cloud: add method ResizeDiskC2, update tests AWS ResizeDisk method doesn't work with C2 cloud, because its EC2 API doesn't implement DescribeVolumesModifications method, which is used for detecting pending modifications. The resize functionality for C2 cloud was implemented in new method: ResizeDiskC2. It checks pending modifications via ModifyVolume API method. Now ResizeDiskC2 is used instead of ResizeDisk in all project (including tests). --- pkg/cloud/cloud.go | 72 +++++++++++++++++++++++++++++++++++ pkg/cloud/cloud_interface.go | 1 + pkg/cloud/mock_cloud.go | 15 ++++++++ pkg/driver/controller.go | 2 +- pkg/driver/controller_test.go | 2 +- pkg/driver/sanity_test.go | 4 ++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 894a0be606..28ef37f2dc 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -1422,6 +1422,78 @@ func (c *cloud) ResizeDisk(ctx context.Context, volumeID string, newSizeBytes in return c.checkDesiredSize(ctx, volumeID, newSizeGiB) } +// ResizeDiskC2 resizes an EBS volume in C2 cloud. +// It returns the volume size after this call or an error if the size couldn't be determined. +// +// ResizeDiskC2 is an adaptation of ResizeDisk function for C2 cloud. Differences: +// 1. Pending volume modifications are detected by ModifyVolume request which returns a lock error if another operation is in progress. +// 2. C2 implementation of ModifyVolume uses 8-GiB increments. +func (c *cloud) ResizeDiskC2(ctx context.Context, volumeID string, newSizeBytes int64) (int64, error) { + describeVolumesReq := &ec2.DescribeVolumesInput{ + VolumeIds: []*string{ + aws.String(volumeID), + }, + } + + volume, err := c.getVolume(ctx, describeVolumesReq) + if err != nil { + return 0, err + } + + newSizeGiB := util.RoundUpGiB(newSizeBytes) + oldSizeGiB := aws.Int64Value(volume.Size) + + // According to CSI spec: if a volume corresponding to the specified volume ID is already larger than + // or equal to the target capacity, the plugin should reply without errors. + if oldSizeGiB >= newSizeGiB { + klog.V(5).Infof("[Debug] Volume %q current size (%d GiB) is greater or equal to the new size (%d GiB)", volumeID, oldSizeGiB, newSizeGiB) + + // Need to check that there are no pending volume modifications (via ModifyVolume request). + newSizeGiB = oldSizeGiB + klog.V(4).Infof("Requested size value changed to current size value (%d GiB)", newSizeGiB) + } + + modifyVolumeReq := &ec2.ModifyVolumeInput{ + VolumeId: aws.String(volumeID), + Size: aws.Int64(newSizeGiB), + } + + klog.V(4).Infof("Expanding volume %q to size %d", volumeID, newSizeGiB) + _, err = c.ec2.ModifyVolumeWithContext(ctx, modifyVolumeReq) + if err != nil { + return 0, fmt.Errorf("could not modify C2 volume %q: %v", volumeID, err) + } + + backoff := wait.Backoff{ + Duration: volumeModificationDuration, + Factor: volumeModificationWaitFactor, + Steps: volumeModificationWaitSteps, + } + + var actualSizeGiB int64 + waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) { + + volume, err := c.getVolume(ctx, describeVolumesReq) + if err != nil { + return true, err + } + + oldSizeGiB := aws.Int64Value(volume.Size) + if oldSizeGiB >= newSizeGiB { + actualSizeGiB = oldSizeGiB + return true, nil + } + + return false, nil + }) + + if waitErr != nil { + return 0, waitErr + } + + return actualSizeGiB, nil +} + // Checks for desired size on volume by also verifying volume size by describing volume. // This is to get around potential eventual consistency problems with describing volume modifications // objects and ensuring that we read two different objects to verify volume state. diff --git a/pkg/cloud/cloud_interface.go b/pkg/cloud/cloud_interface.go index 72852819ac..477359bed5 100644 --- a/pkg/cloud/cloud_interface.go +++ b/pkg/cloud/cloud_interface.go @@ -13,6 +13,7 @@ type Cloud interface { AttachDisk(ctx context.Context, volumeID string, nodeID string) (devicePath string, err error) DetachDisk(ctx context.Context, volumeID string, nodeID string) (err error) ResizeDisk(ctx context.Context, volumeID string, reqSize int64) (newSize int64, err error) + ResizeDiskC2(ctx context.Context, volumeID string, reqSize int64) (newSize int64, err error) WaitForAttachmentState(ctx context.Context, volumeID, expectedState string, expectedInstance string, expectedDevice string, alreadyAssigned bool) (*ec2.VolumeAttachment, error) GetDiskByName(ctx context.Context, name string, capacityBytes int64) (disk *Disk, err error) GetDiskByID(ctx context.Context, volumeID string) (disk *Disk, err error) diff --git a/pkg/cloud/mock_cloud.go b/pkg/cloud/mock_cloud.go index 9acb9376db..848be4cc26 100644 --- a/pkg/cloud/mock_cloud.go +++ b/pkg/cloud/mock_cloud.go @@ -272,6 +272,21 @@ func (mr *MockCloudMockRecorder) ResizeDisk(ctx, volumeID, reqSize interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResizeDisk", reflect.TypeOf((*MockCloud)(nil).ResizeDisk), ctx, volumeID, reqSize) } +// ResizeDiskC2 mocks base method. +func (m *MockCloud) ResizeDiskC2(ctx context.Context, volumeID string, reqSize int64) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResizeDiskC2", ctx, volumeID, reqSize) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResizeDiskC2 indicates an expected call of ResizeDiskC2. +func (mr *MockCloudMockRecorder) ResizeDiskC2(ctx, volumeID, reqSize interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResizeDiskC2", reflect.TypeOf((*MockCloud)(nil).ResizeDiskC2), ctx, volumeID, reqSize) +} + // WaitForAttachmentState mocks base method. func (m *MockCloud) WaitForAttachmentState(ctx context.Context, volumeID, expectedState, expectedInstance, expectedDevice string, alreadyAssigned bool) (*ec2.VolumeAttachment, error) { m.ctrl.T.Helper() diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 5fa10c7b0f..ac34ddd0be 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -508,7 +508,7 @@ func (d *controllerService) ControllerExpandVolume(ctx context.Context, req *csi return nil, status.Error(codes.InvalidArgument, "After round-up, volume size exceeds the limit specified") } - actualSizeGiB, err := d.cloud.ResizeDisk(ctx, volumeID, newSize) + actualSizeGiB, err := d.cloud.ResizeDiskC2(ctx, volumeID, newSize) if err != nil { return nil, status.Errorf(codes.Internal, "Could not resize volume %q: %v", volumeID, err) } diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go index e2db0d7d28..6e714c051c 100644 --- a/pkg/driver/controller_test.go +++ b/pkg/driver/controller_test.go @@ -3418,7 +3418,7 @@ func TestControllerExpandVolume(t *testing.T) { } mockCloud := cloud.NewMockCloud(mockCtl) - mockCloud.EXPECT().ResizeDisk(gomock.Eq(ctx), gomock.Eq(tc.req.VolumeId), gomock.Any()).Return(retSizeGiB, nil).AnyTimes() + mockCloud.EXPECT().ResizeDiskC2(gomock.Eq(ctx), gomock.Eq(tc.req.VolumeId), gomock.Any()).Return(retSizeGiB, nil).AnyTimes() awsDriver := controllerService{ cloud: mockCloud, diff --git a/pkg/driver/sanity_test.go b/pkg/driver/sanity_test.go index 3bf6e6e609..593a47994d 100644 --- a/pkg/driver/sanity_test.go +++ b/pkg/driver/sanity_test.go @@ -319,6 +319,10 @@ func (c *fakeCloudProvider) ResizeDisk(ctx context.Context, volumeID string, new return 0, cloud.ErrNotFound } +func (c *fakeCloudProvider) ResizeDiskC2(ctx context.Context, volumeID string, newSize int64) (int64, error) { + return c.ResizeDisk(ctx, volumeID, newSize) +} + type fakeMounter struct { mount_utils.Interface } From fb50efb18a7d68b5df30b198c1efd84d20fbbd07 Mon Sep 17 00:00:00 2001 From: Margarita Fedotova Date: Sun, 27 Mar 2022 23:59:53 +0300 Subject: [PATCH 36/45] e2e, docs: change size increment in resize test, update test docs Original and target sizes of C2 volumes must be divisible by 8. Test documentation updates: 1. fix commands for e2e test run: escape "[]" symbols 2. move resize test from "red" to "green" 3. remove duplicates from e2e tests list --- docs/TEST.md | 14 +++++--------- ...dynamically_provisioned_resize_volume_tester.go | 4 +++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/TEST.md b/docs/TEST.md index 011de05c11..f8411d421f 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -92,16 +92,13 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t - установить ginkgo: - - go get github.com/onsi/ginkgo/ginkgo@v1.11.0 - запустить e2e тесты для single az: -- - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[single-az\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config +- - ~/go/bin/ginkgo -v -progress --focus="\\[ebs-csi-e2e\] \\[single-az\\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config - запустить e2e тесты для multi az: -- - ~/go/bin/ginkgo -v -progress --focus="\[ebs-csi-e2e\] \[multi-az\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config +- - ~/go/bin/ginkgo -v -progress --focus="\\[ebs-csi-e2e\] \\[multi-az\\]" /root/aws-ebs-csi-driver/tests/e2e -- -report-dir=./reports/ -kubeconfig=/root/.kube/config Какие тесты есть: -Красные: -- "should create a volume on demand and resize it" - не реализован метод describe-volumes-modifications - Зеленые: -- "should use a pre-defined snapshot and create pv from that" +- "[env] should use a pre-defined snapshot and create pv from that" - "should create a pod, write and read to it, take a volume snapshot, and create another pod from the snapshot" - "should create a volume on demand with volumeType "gp2" and encryption" - "should create a volume on demand with volumeType "st2" and encryption" @@ -118,12 +115,11 @@ $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" t - "should delete PV with reclaimPolicy "Delete"" - "[env] should retain PV with reclaimPolicy "Retain"" - "should create a deployment object, write and read to it, delete the pod and write and read to it again" +- "should create a volume on demand and resize it" - "should allow for topology aware volume scheduling" - "[env] should allow for topology aware volume with specified zone in allowedTopologies" - "[env] should write and read to a pre-provisioned volume" - "[env] should use a pre-provisioned volume and mount it as readOnly in a pod" - "[env] should use a pre-provisioned volume and retain PV with reclaimPolicy "Retain"" - "[env] should use a pre-provisioned volume and delete PV with reclaimPolicy "Delete"" -- "[env] should allow for topology aware volume with specified zone in allowedTopologies" -- "should allow for topology aware volume scheduling" -- "should create a volume on demand with provided mountOptions" \ No newline at end of file +- "should create a volume on demand with provided mountOptions" diff --git a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go index c0b301a681..6e1f1c838f 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go @@ -48,9 +48,11 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, pvcName := tpvc.persistentVolumeClaim.Name pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{}) By(fmt.Sprintf("Get pvc name: %v", pvc.Name)) + originalSize := pvc.Spec.Resources.Requests["storage"] + sizeIncrementGiB := int64(8) delta := resource.Quantity{} - delta.Set(util.GiBToBytes(1)) + delta.Set(util.GiBToBytes(sizeIncrementGiB)) originalSize.Add(delta) pvc.Spec.Resources.Requests["storage"] = originalSize From a800260ac756ed50d734c432fd762c3e4b9bbb57 Mon Sep 17 00:00:00 2001 From: Andrey Kulaev Date: Thu, 12 May 2022 11:55:58 +0300 Subject: [PATCH 37/45] deploy: bump version --- deploy/kubernetes/overlays/stable/k_bundle.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index 17313b5d1a..e7798a73da 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -374,7 +374,7 @@ spec: key: access_key name: aws-secret optional: true - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC3 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -527,7 +527,7 @@ spec: fieldPath: spec.nodeName - name: AWS_REGION value: croc - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC3 + image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 livenessProbe: failureThreshold: 5 httpGet: From 69a42b7306c8afcb8ebf2c05ae5f35c5aa4e377f Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Mon, 5 Jun 2023 14:10:20 +0300 Subject: [PATCH 38/45] fixes after rebase --- .../templates/controller.yaml | 4 ++++ charts/aws-ebs-csi-driver/templates/node.yaml | 10 ++++++---- .../kubernetes/overlays/stable/k_bundle.yaml | 18 +++++++++--------- docs/RELEASE.md | 6 +++--- pkg/cloud/cloud.go | 13 ++++++------- pkg/cloud/cloud_test.go | 1 + pkg/resizefs/resizefs_windows.go | 2 +- pkg/resizefs/resizefs_windows_test.go | 2 +- tests/e2e/driver/driver.go | 2 +- 9 files changed, 32 insertions(+), 26 deletions(-) diff --git a/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/aws-ebs-csi-driver/templates/controller.yaml index 56c7e8b035..d72f8177c3 100644 --- a/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -104,6 +104,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: AWS_EC2_ENDPOINT + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: croc - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/aws-ebs-csi-driver/templates/node.yaml index 8684870f16..5090b3681d 100644 --- a/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/aws-ebs-csi-driver/templates/node.yaml @@ -47,7 +47,7 @@ spec: operator: "Exists" {{- end }} {{- with .Values.node.securityContext }} - securityContext: + securityContext: {{- toYaml . | nindent 8 }} {{- end }} containers: @@ -77,6 +77,8 @@ spec: {{- with .Values.node.env }} {{- . | toYaml | nindent 12 }} {{- end }} + - name: AWS_REGION + value: croc {{- with .Values.controller.envFrom }} envFrom: {{- . | toYaml | nindent 12 }} @@ -106,7 +108,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- with .Values.node.containerSecurityContext }} - securityContext: + securityContext: {{- toYaml . | nindent 12 }} {{- end }} - name: node-driver-registrar @@ -155,7 +157,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} - securityContext: + securityContext: {{- toYaml . | nindent 12 }} {{- end }} - name: liveness-probe @@ -178,7 +180,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- with .Values.sidecars.livenessProbe.securityContext }} - securityContext: + securityContext: {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.imagePullSecrets }} diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index e7798a73da..523cd48bd4 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -374,7 +374,7 @@ spec: key: access_key name: aws-secret optional: true - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -410,7 +410,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: dhub.c2.croc.ru/kaas/csi-provisioner:v2.1.1 + image: registry.cloud.croc.ru/kaas/csi-provisioner:v2.1.1 name: csi-provisioner volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -422,7 +422,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: dhub.c2.croc.ru/kaas/csi-attacher:v3.1.0 + image: registry.cloud.croc.ru/kaas/csi-attacher:v3.1.0 name: csi-attacher volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -433,7 +433,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: dhub.c2.croc.ru/kaas/csi-snapshotter:v3.0.3 + image: registry.cloud.croc.ru/kaas/csi-snapshotter:v3.0.3 name: csi-snapshotter volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -444,7 +444,7 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: dhub.c2.croc.ru/kaas/csi-resizer:v1.0.0 + image: registry.cloud.croc.ru/kaas/csi-resizer:v1.0.0 imagePullPolicy: Always name: csi-resizer volumeMounts: @@ -452,7 +452,7 @@ spec: name: socket-dir - args: - --csi-address=/csi/csi.sock - image: dhub.c2.croc.ru/kaas/livenessprobe:v2.2.0 + image: registry.cloud.croc.ru/kaas/livenessprobe:v2.2.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -527,7 +527,7 @@ spec: fieldPath: spec.nodeName - name: AWS_REGION value: croc - image: dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 livenessProbe: failureThreshold: 5 httpGet: @@ -560,7 +560,7 @@ spec: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - image: dhub.c2.croc.ru/kaas/csi-node-driver-registrar:v2.1.0 + image: registry.cloud.croc.ru/kaas/csi-node-driver-registrar:v2.1.0 name: node-driver-registrar volumeMounts: - mountPath: /csi @@ -569,7 +569,7 @@ spec: name: registration-dir - args: - --csi-address=/csi/csi.sock - image: dhub.c2.croc.ru/kaas/livenessprobe:v2.2.0 + image: registry.cloud.croc.ru/kaas/livenessprobe:v2.2.0 name: liveness-probe volumeMounts: - mountPath: /csi diff --git a/docs/RELEASE.md b/docs/RELEASE.md index d191714d41..2e97827682 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -4,7 +4,7 @@ ```sh # uname -r 5.6.13-100.fc30.x86_64 -# cat /etc/os-release +# cat /etc/os-release NAME=Fedora VERSION="30 (Thirty)" ID=fedora @@ -48,6 +48,6 @@ kustomize build ./deployment/kubernetes/stable/ > ./deployment/kubernetes/stable - находясь в руте репы выполнить: ```docker build -t aws-ebs-csi-driver``` - после успешной сборки протегировать имадж: -```docker tag aws-ebs-csi-driver dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:``` +```docker tag aws-ebs-csi-driver registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:``` - запушить имадж в регистри (необходимы врайт права в регистри неймспейсе): -```docker push dhub.c2.croc.ru/kaas/aws-ebs-csi-driver:``` +```docker push registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:``` diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 28ef37f2dc..6379cf7d42 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -427,7 +427,6 @@ func NewCloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Cloud, return newEC2Cloud(region, awsSdkDebugLog, userAgentExtra) } - func newEC2Cloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Cloud, error) { var awsConfig *aws.Config @@ -1422,7 +1421,7 @@ func (c *cloud) ResizeDisk(ctx context.Context, volumeID string, newSizeBytes in return c.checkDesiredSize(ctx, volumeID, newSizeGiB) } -// ResizeDiskC2 resizes an EBS volume in C2 cloud. +// ResizeDiskC2 resizes an EBS volume in C2 cloud. // It returns the volume size after this call or an error if the size couldn't be determined. // // ResizeDiskC2 is an adaptation of ResizeDisk function for C2 cloud. Differences: @@ -1443,7 +1442,7 @@ func (c *cloud) ResizeDiskC2(ctx context.Context, volumeID string, newSizeBytes newSizeGiB := util.RoundUpGiB(newSizeBytes) oldSizeGiB := aws.Int64Value(volume.Size) - // According to CSI spec: if a volume corresponding to the specified volume ID is already larger than + // According to CSI spec: if a volume corresponding to the specified volume ID is already larger than // or equal to the target capacity, the plugin should reply without errors. if oldSizeGiB >= newSizeGiB { klog.V(5).Infof("[Debug] Volume %q current size (%d GiB) is greater or equal to the new size (%d GiB)", volumeID, oldSizeGiB, newSizeGiB) @@ -1452,7 +1451,7 @@ func (c *cloud) ResizeDiskC2(ctx context.Context, volumeID string, newSizeBytes newSizeGiB = oldSizeGiB klog.V(4).Infof("Requested size value changed to current size value (%d GiB)", newSizeGiB) } - + modifyVolumeReq := &ec2.ModifyVolumeInput{ VolumeId: aws.String(volumeID), Size: aws.Int64(newSizeGiB), @@ -1470,14 +1469,14 @@ func (c *cloud) ResizeDiskC2(ctx context.Context, volumeID string, newSizeBytes Steps: volumeModificationWaitSteps, } - var actualSizeGiB int64 + var actualSizeGiB int64 waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) { - + volume, err := c.getVolume(ctx, describeVolumesReq) if err != nil { return true, err } - + oldSizeGiB := aws.Int64Value(volume.Size) if oldSizeGiB >= newSizeGiB { actualSizeGiB = oldSizeGiB diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index 6aec6e442e..a08cafcbca 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -1867,6 +1867,7 @@ func TestWaitForAttachmentState(t *testing.T) { alreadyAssigned: false, expectError: true, }, + // disabled, for Croc cloud purposes // { // name: "failure: unexpected device", // volumeID: "vol-test-1234", diff --git a/pkg/resizefs/resizefs_windows.go b/pkg/resizefs/resizefs_windows.go index 4a8c0638c5..481758437e 100644 --- a/pkg/resizefs/resizefs_windows.go +++ b/pkg/resizefs/resizefs_windows.go @@ -4,7 +4,7 @@ package resizefs import ( - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" + "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" "k8s.io/klog/v2" ) diff --git a/pkg/resizefs/resizefs_windows_test.go b/pkg/resizefs/resizefs_windows_test.go index 088931b922..916a5eac55 100644 --- a/pkg/resizefs/resizefs_windows_test.go +++ b/pkg/resizefs/resizefs_windows_test.go @@ -6,7 +6,7 @@ package resizefs import ( "errors" "github.com/golang/mock/gomock" - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/mounter" + "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" "testing" ) diff --git a/tests/e2e/driver/driver.go b/tests/e2e/driver/driver.go index b4131904da..7f1cbc0760 100644 --- a/tests/e2e/driver/driver.go +++ b/tests/e2e/driver/driver.go @@ -23,7 +23,7 @@ import ( const ( VolumeSnapshotClassKind = "VolumeSnapshotClass" - SnapshotAPIVersion = "snapshot.storage.k8s.io/v1beta1" + SnapshotAPIVersion = "snapshot.storage.k8s.io/v1" ) type PVTestDriver interface { From 90596e3ee4fb0e99c078fb1e6e1df372b95fc1fd Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Mon, 5 Jun 2023 14:31:09 +0300 Subject: [PATCH 39/45] fix formatting after rebase --- pkg/cloud/cloud.go | 29 +++++++++++++-------------- pkg/driver/controller.go | 2 +- pkg/driver/driver.go | 2 +- pkg/driver/node_windows.go | 2 +- pkg/driver/sanity_test.go | 2 +- pkg/resizefs/resizefs_windows_test.go | 2 +- tests/e2e/driver/ebs_csi_driver.go | 2 +- tests/e2e/testsuites/testsuites.go | 2 +- 8 files changed, 21 insertions(+), 22 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 6379cf7d42..808eb46c65 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -19,8 +19,8 @@ package cloud import ( "context" "crypto/sha256" - "encoding/hex" "crypto/tls" + "encoding/hex" "errors" "fmt" "net/http" @@ -31,8 +31,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/awsutil" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" @@ -261,14 +261,13 @@ var _ Cloud = &cloud{} // This method is useful when you want to inject custom logic or configuration // into the SDK's request lifecycle. Such as custom headers, or retry logic. // +// // Example sending a request using the AttachVolumeRequest method. +// req, resp := client.AttachVolumeRequest(params) // -// // Example sending a request using the AttachVolumeRequest method. -// req, resp := client.AttachVolumeRequest(params) -// -// err := req.Send() -// if err == nil { // resp is now filled -// fmt.Println(resp) -// } +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } // // See also, https://docs.aws.amazon.com/goto/WebAPI/ec2-2016-11-15/AttachVolume func AttachVolumeRequest(c *ec2.EC2, input *AttachVolumeInput) (req *request.Request, output *ec2.VolumeAttachment) { @@ -301,15 +300,15 @@ func AttachVolumeRequest(c *ec2.EC2, input *AttachVolumeInput) (req *request.Req // // If a volume has an AWS Marketplace product code: // -// * The volume can be attached only to a stopped instance. +// - The volume can be attached only to a stopped instance. // -// * AWS Marketplace product codes are copied from the volume to the instance. +// - AWS Marketplace product codes are copied from the volume to the instance. // -// * You must be subscribed to the product. +// - You must be subscribed to the product. // -// * The instance type and operating system of the instance must support -// the product. For example, you can't detach a volume from a Windows instance -// and attach it to a Linux instance. +// - The instance type and operating system of the instance must support +// the product. For example, you can't detach a volume from a Windows instance +// and attach it to a Linux instance. // // For more information, see Attaching Amazon EBS Volumes (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-attaching-volume.html) // in the Amazon Elastic Compute Cloud User Guide. diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index ac34ddd0be..f1d7df8342 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -26,11 +26,11 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/awslabs/volume-modifier-for-k8s/pkg/rpc" - csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" "github.com/c2devel/aws-ebs-csi-driver/pkg/util" "github.com/c2devel/aws-ebs-csi-driver/pkg/util/template" + csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 64cfec65e0..19b803ad19 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -22,8 +22,8 @@ import ( "net" "github.com/awslabs/volume-modifier-for-k8s/pkg/rpc" - csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/c2devel/aws-ebs-csi-driver/pkg/util" + csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" "k8s.io/klog/v2" ) diff --git a/pkg/driver/node_windows.go b/pkg/driver/node_windows.go index c2353515fe..066524d875 100644 --- a/pkg/driver/node_windows.go +++ b/pkg/driver/node_windows.go @@ -25,9 +25,9 @@ import ( "strconv" "strings" + "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" diskapi "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1" diskclient "github.com/kubernetes-csi/csi-proxy/client/groups/disk/v1" - "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" "k8s.io/klog/v2" ) diff --git a/pkg/driver/sanity_test.go b/pkg/driver/sanity_test.go index 593a47994d..b04f35f60d 100644 --- a/pkg/driver/sanity_test.go +++ b/pkg/driver/sanity_test.go @@ -13,10 +13,10 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/kubernetes-csi/csi-test/pkg/sanity" "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" "github.com/c2devel/aws-ebs-csi-driver/pkg/driver/internal" "github.com/c2devel/aws-ebs-csi-driver/pkg/util" + "github.com/kubernetes-csi/csi-test/pkg/sanity" "k8s.io/mount-utils" mount_utils "k8s.io/mount-utils" ) diff --git a/pkg/resizefs/resizefs_windows_test.go b/pkg/resizefs/resizefs_windows_test.go index 916a5eac55..a82db26b59 100644 --- a/pkg/resizefs/resizefs_windows_test.go +++ b/pkg/resizefs/resizefs_windows_test.go @@ -5,8 +5,8 @@ package resizefs import ( "errors" - "github.com/golang/mock/gomock" "github.com/c2devel/aws-ebs-csi-driver/pkg/mounter" + "github.com/golang/mock/gomock" "testing" ) diff --git a/tests/e2e/driver/ebs_csi_driver.go b/tests/e2e/driver/ebs_csi_driver.go index 5fc32001d8..137b05db10 100644 --- a/tests/e2e/driver/ebs_csi_driver.go +++ b/tests/e2e/driver/ebs_csi_driver.go @@ -17,8 +17,8 @@ package driver import ( "fmt" - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" ebscsidriver "github.com/c2devel/aws-ebs-csi-driver/pkg/driver" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/tests/e2e/testsuites/testsuites.go b/tests/e2e/testsuites/testsuites.go index 71ad72478c..8ecf023064 100644 --- a/tests/e2e/testsuites/testsuites.go +++ b/tests/e2e/testsuites/testsuites.go @@ -22,9 +22,9 @@ import ( "github.com/aws/aws-sdk-go/aws" + awscloud "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" snapshotclientset "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" - awscloud "github.com/c2devel/aws-ebs-csi-driver/pkg/cloud" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apps "k8s.io/api/apps/v1" From 2edd61338e43d628b25fcaf9f5c76f23654034e6 Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Mon, 5 Jun 2023 18:16:07 +0300 Subject: [PATCH 40/45] fix tests after rebase --- pkg/cloud/cloud.go | 2 +- pkg/cloud/cloud_test.go | 6 +++--- pkg/cloud/devicemanager/manager_test.go | 16 ++++++++-------- pkg/cloud/metadata_test.go | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 808eb46c65..3cdae86348 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -75,7 +75,7 @@ const ( io1MaxIOPSPerGB = 50 io2MinTotalIOPS = 100 io2MaxTotalIOPS = 50000 - io2BlockExpressMaxTotalIOPS = 256000 + io2BlockExpressMaxTotalIOPS = 150000 io2MaxIOPSPerGB = 50 gp3MaxTotalIOPS = 16000 gp3MinTotalIOPS = 3000 diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index a08cafcbca..e15a3fb26a 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -518,7 +518,7 @@ func TestCreateDisk(t *testing.T) { AvailabilityZone: defaultZone, }, expCreateVolumeInput: &ec2.CreateVolumeInput{ - Iops: aws.Int64(256000), + Iops: aws.Int64(150000), }, expErr: nil, }, @@ -529,7 +529,7 @@ func TestCreateDisk(t *testing.T) { CapacityBytes: util.GiBToBytes(1), Tags: map[string]string{VolumeNameTagKey: "vol-test", AwsEbsDriverTagKey: "true"}, AvailabilityZone: snowZone, - VolumeType: "sbp1", + VolumeType: "gp2", }, expCreateVolumeInput: &ec2.CreateVolumeInput{}, expDisk: &Disk{ @@ -546,7 +546,7 @@ func TestCreateDisk(t *testing.T) { CapacityBytes: util.GiBToBytes(1), Tags: map[string]string{VolumeNameTagKey: "vol-test", AwsEbsDriverTagKey: "true"}, AvailabilityZone: snowZone, - VolumeType: "sbg1", + VolumeType: "st2", }, expCreateVolumeInput: &ec2.CreateVolumeInput{}, expCreateTagsErr: fmt.Errorf("CreateTags generic error"), diff --git a/pkg/cloud/devicemanager/manager_test.go b/pkg/cloud/devicemanager/manager_test.go index 9a9055297e..040c414a9a 100644 --- a/pkg/cloud/devicemanager/manager_test.go +++ b/pkg/cloud/devicemanager/manager_test.go @@ -103,30 +103,30 @@ func TestNewDeviceWithExistingDevice(t *testing.T) { { name: "success: different volumes", existingID: "vol-1", - existingPath: deviceNames[0], + existingPath: "/dev/disk/by-id/virtio-vol-1", volumeID: "vol-2", - expectedPath: deviceNames[1], + expectedPath: "/dev/disk/by-id/virtio-vol-2", }, { name: "success: same volumes", existingID: "vol-1", - existingPath: "/dev/xvdcc", + existingPath: "/dev/disk/by-id/virtio-vol-1", volumeID: "vol-1", - expectedPath: "/dev/xvdcc", + expectedPath: "/dev/disk/by-id/virtio-vol-1", }, { name: "success: same volumes with /dev/sdX path", existingID: "vol-3", - existingPath: "/dev/sdf", + existingPath: "/dev/disk/by-id/virtio-vol-3", volumeID: "vol-3", - expectedPath: "/dev/sdf", + expectedPath: "/dev/disk/by-id/virtio-vol-3", }, { name: "success: same volumes with weird path", existingID: "vol-42", - existingPath: "/weird/path", + existingPath: "/dev/disk/by-id/virtio-vol-42", volumeID: "vol-42", - expectedPath: "/weird/path", + expectedPath: "/dev/disk/by-id/virtio-vol-42", }, } // Use a shared DeviceManager to make sure that there are no race conditions diff --git a/pkg/cloud/metadata_test.go b/pkg/cloud/metadata_test.go index 3b033e1212..4c376569be 100644 --- a/pkg/cloud/metadata_test.go +++ b/pkg/cloud/metadata_test.go @@ -106,10 +106,10 @@ func TestNewMetadataService(t *testing.T) { Region: stdRegion, AvailabilityZone: stdAvailabilityZone, }, - isAwsRegionEnvSet: false, getMetadataValue: "foo", imdsENIOutput: "00:00:00:00:00:00", expectedENIs: 1, + isAwsRegionEnvSet: false, }, { name: "success: outpost-arn is not found", @@ -452,7 +452,7 @@ func TestNewMetadataService(t *testing.T) { if tc.isAwsRegionEnvSet && m.GetRegion() != envRegion { t.Fatalf("GetRegion() failed: expected %v, got %v", envRegion, m.GetRegion()) } - if !tc.isAwsRegionEnvSet && m.GetRegion() != stdRegion { + if !tc.isAwsRegionEnvSet && tc.regionFromSession == "" && m.GetRegion() != stdRegion { t.Fatalf("GetRegion() failed: expected %v, got %v", stdRegion, m.GetRegion()) } } From 25fcabcfe6c37eae086dffc0e26357df78ff0a2c Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Mon, 5 Jun 2023 18:50:52 +0300 Subject: [PATCH 41/45] fix for linter after rebase --- pkg/cloud/cloud.go | 4 ++-- pkg/cloud/devicemanager/manager.go | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 3cdae86348..3b4ef13f82 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -436,7 +436,7 @@ func newEC2Cloud(region string, awsSdkDebugLog bool, userAgentExtra string) (Clo var err error isEndpointInsecure, err = strconv.ParseBool(envEndpointInsecure) if err != nil { - return nil, fmt.Errorf("Unable to parse environment variable AWS_EC2_ENDPOINT_UNSECURE: %v", err) + return nil, fmt.Errorf("Unable to parse environment variable AWS_EC2_ENDPOINT_UNSECURE: %w", err) } } @@ -1459,7 +1459,7 @@ func (c *cloud) ResizeDiskC2(ctx context.Context, volumeID string, newSizeBytes klog.V(4).Infof("Expanding volume %q to size %d", volumeID, newSizeGiB) _, err = c.ec2.ModifyVolumeWithContext(ctx, modifyVolumeReq) if err != nil { - return 0, fmt.Errorf("could not modify C2 volume %q: %v", volumeID, err) + return 0, fmt.Errorf("could not modify C2 volume %q: %w", volumeID, err) } backoff := wait.Backoff{ diff --git a/pkg/cloud/devicemanager/manager.go b/pkg/cloud/devicemanager/manager.go index d65a80a936..2bcbfb8b73 100644 --- a/pkg/cloud/devicemanager/manager.go +++ b/pkg/cloud/devicemanager/manager.go @@ -18,7 +18,6 @@ package devicemanager import ( "fmt" - "strings" "sync" "github.com/aws/aws-sdk-go/aws" @@ -195,11 +194,6 @@ func (d *deviceManager) getVolumeIdsInUse(instance *ec2.Instance) []string { if blockDevice.Ebs == nil { continue } - - name := aws.StringValue(blockDevice.DeviceName) - // trim device prefix from name - name = strings.TrimPrefix(name, devPreffix) - inUse = append(inUse, *blockDevice.Ebs.VolumeId) } From 7cf923bca1b1a8eaa376df1d4e27ae347fa656f9 Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Wed, 28 Jun 2023 11:49:30 +0300 Subject: [PATCH 42/45] update specs and instructions --- Makefile | 2 +- README.md | 8 +- .../templates/controller.yaml | 12 +- charts/aws-ebs-csi-driver/values.yaml | 18 +- deploy/kubernetes/base/controller.yaml | 24 +- deploy/kubernetes/base/node.yaml | 6 +- .../overlays/stable/ecr/kustomization.yaml | 14 +- .../overlays/stable/gcr/kustomization.yaml | 14 +- .../kubernetes/overlays/stable/k_bundle.yaml | 229 +++++++++++-- docs/README.md | 311 ------------------ docs/RELEASE.md | 53 --- docs/release.md | 155 +++------ docs/{TEST.md => testing.md} | 0 13 files changed, 295 insertions(+), 551 deletions(-) delete mode 100644 docs/README.md delete mode 100644 docs/RELEASE.md rename docs/{TEST.md => testing.md} (100%) diff --git a/Makefile b/Makefile index e4baad7eb0..7f57a42d4a 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION?=v1.20.0 +VERSION?=v1.20.0-CROC1 PKG=github.com/c2devel/aws-ebs-csi-driver GIT_COMMIT?=$(shell git rev-parse HEAD) diff --git a/README.md b/README.md index 237fc1ed78..f644689892 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Amazon Elastic Block Store (EBS) CSI driver -[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/kubernetes-sigs/aws-ebs-csi-driver)](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases) -[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes-sigs/aws-ebs-csi-driver)](https://goreportcard.com/report/github.com/kubernetes-sigs/aws-ebs-csi-driver) +[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/c2devel/aws-ebs-csi-driver)](https://github.com/c2devel/aws-ebs-csi-driver/releases) +[![Go Report Card](https://goreportcard.com/badge/github.com/c2devel/aws-ebs-csi-driver)](https://goreportcard.com/report/github.com/c2devel/aws-ebs-csi-driver) + +## Скорректированный релиз процесс + +см. [release process](docs/release.md) ## Overview diff --git a/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/aws-ebs-csi-driver/templates/controller.yaml index d72f8177c3..8e2e9f9253 100644 --- a/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -104,10 +104,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: AWS_EC2_ENDPOINT - value: https://api.cloud.croc.ru - - name: AWS_REGION - value: croc - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -121,11 +117,9 @@ spec: key: access_key optional: true - name: AWS_EC2_ENDPOINT - valueFrom: - configMapKeyRef: - name: aws-meta - key: endpoint - optional: true + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: croc {{- with .Values.controller.region }} - name: AWS_REGION value: {{ . }} diff --git a/charts/aws-ebs-csi-driver/values.yaml b/charts/aws-ebs-csi-driver/values.yaml index 7ea9934ecf..e6622b0c41 100644 --- a/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/aws-ebs-csi-driver/values.yaml @@ -3,9 +3,9 @@ # Declare variables to be passed into your templates. image: - repository: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver + repository: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver # Overrides the image tag whose default is v{{ .Chart.AppVersion }} - tag: "" + tag: "v1.19.0-CROC1" pullPolicy: IfNotPresent # -- Custom labels to add into metadata @@ -18,7 +18,7 @@ sidecars: env: [] image: pullPolicy: IfNotPresent - repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + repository: registry.cloud.croc.ru/kaas/external-provisioner tag: "v3.5.0-eks-1-27-3" logLevel: 2 additionalArgs: [] @@ -40,7 +40,7 @@ sidecars: env: [] image: pullPolicy: IfNotPresent - repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher + repository: registry.cloud.croc.ru/kaas/external-attacher tag: "v4.3.0-eks-1-27-3" # Tune leader lease election for csi-attacher. # Leader election is on by default. @@ -64,7 +64,7 @@ sidecars: env: [] image: pullPolicy: IfNotPresent - repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter + repository: registry.cloud.croc.ru/kaas/csi-snapshotter tag: "v6.2.1-eks-1-27-3" logLevel: 2 additionalArgs: [] @@ -75,7 +75,7 @@ sidecars: livenessProbe: image: pullPolicy: IfNotPresent - repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + repository: registry.cloud.croc.ru/kaas/livenessprobe tag: "v2.10.0-eks-1-27-3" additionalArgs: [] resources: {} @@ -86,7 +86,7 @@ sidecars: env: [] image: pullPolicy: IfNotPresent - repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer + repository: registry.cloud.croc.ru/kaas/external-resizer tag: "v1.8.0-eks-1-27-3" # Tune leader lease election for csi-resizer. # Leader election is on by default. @@ -108,7 +108,7 @@ sidecars: env: [] image: pullPolicy: IfNotPresent - repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + repository: registry.cloud.croc.ru/kaas/node-driver-registrar tag: "v2.8.0-eks-1-27-3" logLevel: 2 additionalArgs: [] @@ -120,7 +120,7 @@ sidecars: env: [] image: pullPolicy: IfNotPresent - repository: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s + repository: registry.cloud.croc.ru/kaas/volume-modifier-for-k8s tag: "v0.1.0" leaderElection: enabled: true diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index dc1da4087d..10c1c01253 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -61,7 +61,7 @@ spec: runAsUser: 1000 containers: - name: ebs-plugin - image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.19.0 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 imagePullPolicy: IfNotPresent args: # - {all,controller,node} # specify the driver mode @@ -76,10 +76,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: AWS_EC2_ENDPOINT - value: https://api.cloud.croc.ru - - name: AWS_REGION - value: croc - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -93,11 +89,9 @@ spec: key: access_key optional: true - name: AWS_EC2_ENDPOINT - valueFrom: - configMapKeyRef: - name: aws-meta - key: endpoint - optional: true + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: croc volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -131,7 +125,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-provisioner - image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v3.5.0-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/external-provisioner:v3.5.0-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=$(ADDRESS) @@ -156,7 +150,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-attacher - image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.3.0-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/external-attacher:v4.3.0-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=$(ADDRESS) @@ -178,7 +172,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-snapshotter - image: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter:v6.2.1-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/csi-snapshotter:v6.2.1-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=$(ADDRESS) @@ -200,7 +194,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-resizer - image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.8.0-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/external-resizer:v1.8.0-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=$(ADDRESS) @@ -223,7 +217,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: liveness-probe - image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.10.0-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/livenessprobe:v2.10.0-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=/csi/csi.sock diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index 95cc15e8b4..324332417f 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -44,7 +44,7 @@ spec: runAsUser: 0 containers: - name: ebs-plugin - image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.19.0 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 imagePullPolicy: IfNotPresent args: - node @@ -91,7 +91,7 @@ spec: privileged: true readOnlyRootFilesystem: true - name: node-driver-registrar - image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.8.0-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/node-driver-registrar:v2.8.0-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=$(ADDRESS) @@ -128,7 +128,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: liveness-probe - image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.10.0-eks-1-27-3 + image: registry.cloud.croc.ru/kaas/livenessprobe:v2.10.0-eks-1-27-3 imagePullPolicy: IfNotPresent args: - --csi-address=/csi/csi.sock diff --git a/deploy/kubernetes/overlays/stable/ecr/kustomization.yaml b/deploy/kubernetes/overlays/stable/ecr/kustomization.yaml index c020fd6a1c..724a0c94c1 100644 --- a/deploy/kubernetes/overlays/stable/ecr/kustomization.yaml +++ b/deploy/kubernetes/overlays/stable/ecr/kustomization.yaml @@ -3,17 +3,17 @@ kind: Kustomization resources: - ../ecr-public images: - - name: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver + - name: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-ebs-csi-driver - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + - name: registry.cloud.croc.ru/kaas/external-provisioner newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/csi-provisioner - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher + - name: registry.cloud.croc.ru/kaas/external-attacher newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/csi-attacher - - name: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + - name: registry.cloud.croc.ru/kaas/livenessprobe newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/livenessprobe - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter + - name: registry.cloud.croc.ru/kaas/csi-snapshotter newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/csi-snapshotter - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer + - name: registry.cloud.croc.ru/kaas/external-resizer newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/csi-resizer - - name: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + - name: registry.cloud.croc.ru/kaas/node-driver-registrar newName: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/csi-node-driver-registrar diff --git a/deploy/kubernetes/overlays/stable/gcr/kustomization.yaml b/deploy/kubernetes/overlays/stable/gcr/kustomization.yaml index 5e1b376872..fff6415684 100644 --- a/deploy/kubernetes/overlays/stable/gcr/kustomization.yaml +++ b/deploy/kubernetes/overlays/stable/gcr/kustomization.yaml @@ -3,23 +3,23 @@ kind: Kustomization resources: - ../ecr-public images: - - name: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver + - name: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver newName: registry.k8s.io/provider-aws/aws-ebs-csi-driver - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + - name: registry.cloud.croc.ru/kaas/external-provisioner newName: registry.k8s.io/sig-storage/csi-provisioner newTag: v3.5.0 - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher + - name: registry.cloud.croc.ru/kaas/external-attacher newName: registry.k8s.io/sig-storage/csi-attacher newTag: v4.3.0 - - name: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + - name: registry.cloud.croc.ru/kaas/livenessprobe newName: registry.k8s.io/sig-storage/livenessprobe newTag: v2.10.0 - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter + - name: registry.cloud.croc.ru/kaas/csi-snapshotter newName: registry.k8s.io/sig-storage/csi-snapshotter newTag: v6.2.1 - - name: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer + - name: registry.cloud.croc.ru/kaas/external-resizer newName: registry.k8s.io/sig-storage/csi-resizer newTag: v1.8.0 - - name: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + - name: registry.cloud.croc.ru/kaas/node-driver-registrar newName: registry.k8s.io/sig-storage/csi-node-driver-registrar newTag: v2.8.0 diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index 523cd48bd4..7f76539c32 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -16,6 +16,21 @@ metadata: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver @@ -234,13 +249,6 @@ rules: - create - update - patch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - apiGroups: - snapshot.storage.k8s.io resources: @@ -260,6 +268,7 @@ rules: - watch - update - delete + - patch - apiGroups: - snapshot.storage.k8s.io resources: @@ -284,6 +293,21 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-getter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role +subjects: +- kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver @@ -340,16 +364,42 @@ spec: matchLabels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: app: ebs-csi-controller app.kubernetes.io/name: aws-ebs-csi-driver spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - ebs-csi-controller + topologyKey: kubernetes.io/hostname + weight: 100 containers: - args: - --endpoint=$(CSI_ENDPOINT) - - --logtostderr + - --logging-format=text + - --user-agent-extra=kustomize - --v=2 env: - name: CSI_ENDPOINT @@ -358,10 +408,6 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - - name: AWS_EC2_ENDPOINT - value: https://api.cloud.croc.ru - - name: AWS_REGION - value: croc - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -374,7 +420,12 @@ spec: key: access_key name: aws-secret optional: true - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 + - name: AWS_EC2_ENDPOINT + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: croc + envFrom: null + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -397,6 +448,15 @@ spec: initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 3 + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir @@ -410,8 +470,19 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.cloud.croc.ru/kaas/csi-provisioner:v2.1.1 + envFrom: null + image: registry.cloud.croc.ru/kaas/external-provisioner:v3.5.0-eks-1-27-3 + imagePullPolicy: IfNotPresent name: csi-provisioner + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir @@ -422,44 +493,94 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.cloud.croc.ru/kaas/csi-attacher:v3.1.0 + envFrom: null + image: registry.cloud.croc.ru/kaas/external-attacher:v4.3.0-eks-1-27-3 + imagePullPolicy: IfNotPresent name: csi-attacher + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - --leader-election=true + - --extra-create-metadata env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.cloud.croc.ru/kaas/csi-snapshotter:v3.0.3 + envFrom: null + image: registry.cloud.croc.ru/kaas/csi-snapshotter:v6.2.1-eks-1-27-3 + imagePullPolicy: IfNotPresent name: csi-snapshotter + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=$(ADDRESS) - --v=2 + - --handle-volume-inuse-error=false env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.cloud.croc.ru/kaas/csi-resizer:v1.0.0 - imagePullPolicy: Always + envFrom: null + image: registry.cloud.croc.ru/kaas/external-resizer:v1.8.0-eks-1-27-3 + imagePullPolicy: IfNotPresent name: csi-resizer + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - args: - --csi-address=/csi/csi.sock - image: registry.cloud.croc.ru/kaas/livenessprobe:v2.2.0 + envFrom: null + image: registry.cloud.croc.ru/kaas/livenessprobe:v2.10.0-eks-1-27-3 + imagePullPolicy: IfNotPresent name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /csi name: socket-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 serviceAccountName: ebs-csi-controller-sa tolerations: - key: CriticalAddonsOnly @@ -471,7 +592,7 @@ spec: - emptyDir: {} name: socket-dir --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -516,7 +637,7 @@ spec: - args: - node - --endpoint=$(CSI_ENDPOINT) - - --logtostderr + - --logging-format=text - --v=2 env: - name: CSI_ENDPOINT @@ -527,7 +648,9 @@ spec: fieldPath: spec.nodeName - name: AWS_REGION value: croc - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.1.1-CROC4 + envFrom: null + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 httpGet: @@ -541,8 +664,15 @@ spec: - containerPort: 9808 name: healthz protocol: TCP + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi securityContext: privileged: true + readOnlyRootFilesystem: true volumeMounts: - mountPath: /var/lib/kubelet mountPropagation: Bidirectional @@ -560,30 +690,64 @@ spec: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - image: registry.cloud.croc.ru/kaas/csi-node-driver-registrar:v2.1.0 + envFrom: null + image: registry.cloud.croc.ru/kaas/node-driver-registrar:v2.8.0-eks-1-27-3 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 name: node-driver-registrar + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /csi name: plugin-dir - mountPath: /registration name: registration-dir + - mountPath: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + name: probe-dir - args: - --csi-address=/csi/csi.sock - image: registry.cloud.croc.ru/kaas/livenessprobe:v2.2.0 + envFrom: null + image: registry.cloud.croc.ru/kaas/livenessprobe:v2.10.0-eks-1-27-3 + imagePullPolicy: IfNotPresent name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true volumeMounts: - mountPath: /csi name: plugin-dir nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 serviceAccountName: ebs-csi-node-sa tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - tolerationSeconds: 300 + - operator: Exists volumes: - hostPath: path: /var/lib/kubelet @@ -601,6 +765,12 @@ spec: path: /dev type: Directory name: device-dir + - emptyDir: {} + name: probe-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate --- apiVersion: storage.k8s.io/v1 kind: CSIDriver @@ -610,4 +780,5 @@ metadata: name: ebs.csi.aws.com spec: attachRequired: true + fsGroupPolicy: File podInfoOnMount: false diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 111e1a888d..0000000000 --- a/docs/README.md +++ /dev/null @@ -1,311 +0,0 @@ -[![Build Status](https://travis-ci.org/c2devel/aws-ebs-csi-driver.svg?branch=master)](https://travis-ci.org/c2devel/aws-ebs-csi-driver) -[![Coverage Status](https://coveralls.io/repos/github/c2devel/aws-ebs-csi-driver/badge.svg?branch=master)](https://coveralls.io/github/c2devel/aws-ebs-csi-driver?branch=master) -[![Go Report Card](https://goreportcard.com/badge/github.com/c2devel/aws-ebs-csi-driver)](https://goreportcard.com/report/github.com/c2devel/aws-ebs-csi-driver) - -# Amazon Elastic Block Store (EBS) CSI driver - -## Скорректированный релиз процесс - -см. [release process](RELEASE.md) - -## Overview - -The [Amazon Elastic Block Store](https://aws.amazon.com/ebs/) Container Storage Interface (CSI) Driver provides a [CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md) interface used by Container Orchestrators to manage the lifecycle of Amazon EBS volumes. - -## CSI Specification Compatibility Matrix -| AWS EBS CSI Driver \ CSI Version | v0.3.0| v1.0.0 | v1.1.0 | -|----------------------------------------|-------|--------|--------| -| master branch | no | no | yes | -| v1.1.x | no | no | yes | -| v1.0.0 | no | no | yes | -| v0.10.x | no | no | yes | -| v0.9.x | no | no | yes | -| v0.8.x | no | no | yes | -| v0.7.1 | no | no | yes | -| v0.6.0 | no | no | yes | -| v0.5.0 | no | no | yes | -| v0.4.0 | no | no | yes | -| v0.3.0 | no | yes | no | -| v0.2.0 | no | yes | no | -| v0.1.0 | yes | no | no | - -## Features -The following CSI gRPC calls are implemented: -* **Controller Service**: CreateVolume, DeleteVolume, ControllerPublishVolume, ControllerUnpublishVolume, ControllerGetCapabilities, ValidateVolumeCapabilities, CreateSnapshot, DeleteSnapshot, ListSnapshots -* **Node Service**: NodeStageVolume, NodeUnstageVolume, NodePublishVolume, NodeUnpublishVolume, NodeGetCapabilities, NodeGetInfo -* **Identity Service**: GetPluginInfo, GetPluginCapabilities, Probe - -## CreateVolume Parameters -There are several optional parameters that could be passed into `CreateVolumeRequest.parameters` map, these parameters can be configured in StorageClass, see [example](../examples/kubernetes/storageclass): - -| Parameters | Values | Default | Description | -|-----------------------------|----------------------------------------|----------|---------------------| -| "csi.storage.k8s.io/fstype" | xfs, ext2, ext3, ext4 | ext4 | File system type that will be formatted during volume creation. This parameter is case sensitive! | -| "type" | io1, io2, gp2, gp3, sc1, st1,standard | gp3* | EBS volume type | -| "iopsPerGB" | | | I/O operations per second per GiB. Required when io1 or io2 volume type is specified. If this value multiplied by the size of a requested volume produces a value above the maximum IOPs allowed for the volume type, as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), AWS will cap the IOPS to maximum supported value. If the value is lower than minimal supported IOPS value per volume, either error is returned (the default behavior) or the value is increased to fit into the supported range when `allowautoiopspergbincrease` is `"true"`.| -| "allowAutoIOPSPerGBIncrease"| true, false | false | When `"true"`, the CSI driver increases IOPS for a volume when `iopsPerGB * ` is too low to fit into IOPS range supported by AWS. This allows dynamic provisioning to always succeed, even when user specifies too small PVC capacity or `iopsPerGB` value. On the other hand, it may introduce additional costs, as such volumes have higher IOPS than requested in `iopsPerGB`.| -| "iops" | | 3000 | I/O operations per second. Only effetive when gp3 volume type is specified. If empty, it will set to 3000 as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). | -| "throughput" | | 125 | Throughput in MiB/s. Only effective when gp3 volume type is specified. If empty, it will set to 125MiB/s as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). | -| "encrypted" | | | Whether the volume should be encrypted or not. Valid values are "true" or "false" | -| "kmsKeyId" | | | The full ARN of the key to use when encrypting the volume. When not specified, the default KMS key is used | - -**Notes**: -* `gp3` is currently not supported on outposts. Outpost customers need to use a different type for their volumes. -* Unless explicitly noted, all parameters are case insensitive (e.g. "kmsKeyId", "kmskeyid" and any other combination of upper/lowercase characters can be used). - -## Tagging -To help manage volumes in the aws account, CSI driver will automatically add tags to the volumes it manages. - -| TagKey | TagValue | sample | Description | -|------------------------|---------------------------|---------------------------------------------------------------------|---------------------| -| CSIVolumeName | pvcName | CSIVolumeName = pvc-a3ab0567-3a48-4608-8cb6-4e3b1485c808 | add to all volumes, for recording associated pvc id and checking if a given volume was already created so that ControllerPublish/CreateVolume is idempotent. | -| CSISnapshotName | volumeSnapshotContentName | CSISnapshotName = snapcontent-69477690-803b-4d3e-a61a-03c7b2592a76 | add to all snapshots, for recording associated VolumeSnapshot id and checking if a given snapshot was already created | -| ebs.csi.aws.com/cluster| true | ebs.csi.aws.com/cluster = true | add to all volumes and snapshots, for allowing users to use a policy to limit csi driver's permission to just the resources it manages. | -| kubernetes.io/cluster/X| owned | kubernetes.io/cluster/aws-cluster-id-1 = owned | add to all volumes and snapshots if k8s-tag-cluster-id argument is set to X.| -| extra-key | extra-value | extra-key = extra-value | add to all volumes and snapshots if extraTags argument is set| - -## Driver Options -There are couple driver options that can be passed as arguments when starting driver container. - -| Option argument | value sample | default | Description | -|-----------------------------|---------------------------------------------------|-----------------------------------------------------|---------------------| -| endpoint | tcp://127.0.0.1:10000/ | unix:///var/lib/csi/sockets/pluginproxy/csi.sock | added to all volumes, for checking if a given volume was already created so that ControllerPublish/CreateVolume is idempotent. | -| volume-attach-limit | 1,2,3 ... | -1 | Value for the maximum number of volumes attachable per node. If specified, the limit applies to all nodes. If not specified, the value is approximated from the instance type. | -| extra-tags | key1=value1,key2=value2 | | Extra tags to attach to each dynamically provisioned resource.| -| k8s-tag-cluster-id | aws-cluster-id-1 | | ID of the Kubernetes cluster used for tagging provisioned EBS volumes.| -| aws-sdk-debug-log | true | false | if true, driver will enable the aws sdk debug log level| - -# EBS CSI Driver on Kubernetes -Following sections are Kubernetes specific. If you are Kubernetes user, use followings for driver features, installation steps and examples. - -## Kubernetes Version Compatibility Matrix -| AWS EBS CSI Driver \ Kubernetes Version| v1.12 | v1.13 | v1.14 | v1.15 | v1.16 | v1.17 | v1.18+ | -|----------------------------------------|-------|-------|-------|-------|-------|-------|-------| -| master branch | no | no+ | no | no | no | yes | yes | -| v1.1.0 | no | no+ | no | no | no | yes | yes | -| v1.0.0 | no | no+ | no | no | no | yes | yes | -| v0.10.x | no | no+ | no | no | no | yes | yes | -| v0.9.x | no | no+ | no | no | no | yes | yes | -| v0.8.x | no | no+ | yes | yes | yes | yes | yes | -| v0.7.1 | no | no+ | yes | yes | yes | yes | yes | -| v0.6.0 | no | no+ | yes | yes | yes | yes | yes | -| v0.5.0 | no | no+ | yes | yes | yes | yes | yes | -| v0.4.0 | no | no+ | yes | yes | no | no | no | -| v0.3.0 | no | no+ | yes | no | no | no | no | -| v0.2.0 | no | yes | yes | no | no | no | no | -| v0.1.0 | yes | yes | yes | no | no | no | no | - -**Note**: for the entry with `+` sign, it means the driver's default released manifest doesn't work with corresponding Kubernetes version, but the driver container image is compatiable with the Kubernetes version if an older version's manifest is used. - -## Container Images: -|AWS EBS CSI Driver Version | Image | -|---------------------------|--------------------------------------------------| -|v1.1.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.1.1 | -|v1.1.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.1.0 | -|v1.0.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.0.0 | -|v0.10.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.10.1| -|v0.10.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.10.0| -|v0.9.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.9.1 | -|v0.9.0 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.9.0 | -|v0.8.1 |k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v0.8.1 | -|v0.7.1 |amazon/aws-ebs-csi-driver:v0.7.1 | -|v0.6.0 |amazon/aws-ebs-csi-driver:v0.6.0 | -|v0.5.0 |amazon/aws-ebs-csi-driver:v0.5.0 | -|v0.4.0 |amazon/aws-ebs-csi-driver:v0.4.0 | -|v0.3.0 |amazon/aws-ebs-csi-driver:v0.3.0 | -|v0.2.0 |amazon/aws-ebs-csi-driver:0.2.0 | -|v0.1.0 |amazon/aws-ebs-csi-driver:0.1.0-alpha | - -## Features -* **Static Provisioning** - create a new or migrating existing EBS volumes, then create persistence volume (PV) from the EBS volume and consume the PV from container using persistence volume claim (PVC). -* **Dynamic Provisioning** - uses persistence volume claim (PVC) to request the Kuberenetes to create the EBS volume on behalf of user and consumes the volume from inside container. Storage class's **allowedTopologies** could be used to restrict which AZ the volume should be provisioned in. The topology key should be **topology.ebs.csi.aws.com/zone**. -* **Mount Option** - mount options could be specified in persistence volume (PV) to define how the volume should be mounted. -* **NVMe** - consume NVMe EBS volume from EC2 [Nitro instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). -* **[Block Volume](https://kubernetes-csi.github.io/docs/raw-block.html)** - consumes the EBS volume as a raw block device for latency sensitive application eg. MySql. The corresponding CSI feature (`CSIBlockVolume`) is GA since Kubernetes 1.18. -* **[Volume Snapshot](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html)** - creating volume snapshots and restore volume from snapshot. The corresponding CSI feature (`VolumeSnapshotDataSource`) is beta since Kubernetes 1.17. -* **[Volume Resizing](https://kubernetes-csi.github.io/docs/volume-expansion.html)** - expand the volume size. The corresponding CSI feature (`ExpandCSIVolumes`) is beta since Kubernetes 1.16. - -## Prerequisites -* If you are managing EBS volumes using static provisioning, get yourself familiar with [EBS volume](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AmazonEBS.html). -* Get yourself familiar with how to setup Kubernetes on AWS and have a working Kubernetes cluster: - * Enable flag `--allow-privileged=true` for `kubelet` and `kube-apiserver` - * Enable `kube-apiserver` feature gates `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true` - * Enable `kubelet` feature gates `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true,CSIBlockVolume=true` -* If you intend to use the csi-snapshotter functionality you will need to first install the [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) - -## Installation -#### Set up driver permission -The driver requires IAM permission to talk to Amazon EBS to manage the volume on user's behalf. [The example policy here](./example-iam-policy.json) defines these permissions. There are several methods to grant the driver IAM permission: -* Using IAM [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) - attach the policy to the instance profile IAM role and turn on access to [instance metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) for the instance(s) on which the driver Deployment will run -* EKS only: Using [IAM roles for ServiceAccounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) - create an IAM role, attach the policy to it, then follow the IRSA documentation to associate the IAM role with the driver Deployment service account, which if you are installing via helm is determined by value `controller.serviceAccount.name`, `ebs-csi-controller-sa` by default -* Using secret object - create an IAM user, attach the policy to it, put that user's credentials in [secret manifest](../deploy/kubernetes/secret.yaml), then deploy the secret -```sh -curl https://raw.githubusercontent.com/c2devel/aws-ebs-csi-driver/master/deploy/kubernetes/secret.yaml > secret.yaml -# Edit the secret with user credentials -kubectl apply -f secret.yaml -``` - -#### Config node toleration settings -By default, driver tolerates taint `CriticalAddonsOnly` and has `tolerationSeconds` configured as `300`, to deploy the driver on all nodes, please set helm `Value.node.tolerateAllTaints` to true before deployment - -#### Deploy driver -Please see the compatibility matrix above before you deploy the driver - -To deploy the CSI driver: -```sh -kubectl apply -k "github.com/c2devel/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=master" -``` - -Verify driver is running: -```sh -kubectl get pods -n kube-system -``` - -Alternatively, you could also install the driver using helm: - -Add the aws-ebs-csi-driver Helm repository: -```sh -helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver -helm repo update -``` - -Then install a release of the driver using the chart -```sh -helm upgrade --install aws-ebs-csi-driver \ - --namespace kube-system \ - aws-ebs-csi-driver/aws-ebs-csi-driver -``` - -##### Upgrading from version 1.X to 2.X of the helm chart -Version 2.0.0 remove support for helm v2 and now requires helm v3 or above - -The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. - -The following deprecated values have been removed, and you should now use their counterparts under the `controller` and `node` maps which have been available since chart version 1.1.0 -* affinity -* extraCreateMetadata -* extraVolumeTags -* k8sTagClusterId -* nodeSelector -* podAnnotations -* priorityClassName -* region -* replicaCount -* resources -* tolerations -* topologySpreadConstraints -* volumeAttachLimit - -The values under `serviceAccount.controller` have been relocated to `controller.serviceAccount` -The values under `serviceAccount.node` have been relocated to `node.serviceAccount` - -The following `sidecars` values have been reorganized from -```yaml -sidecars: - provisionerImage: - attacherImage: - snapshotterImage: - livenessProbeImage: - resizerImage: - nodeDriverRegistrarImage: -``` -to -```yaml -sidecars: - provisioner: - image: - attacher: - image: - snapshotter: - image: - livenessProbe: - image: - resizer: - image: - nodeDriverRegistrar: - image: -``` - -With the above reorganization `controller.containerResources`, `controller.env`, `node.containerResources`, and `node.env` were also moved into the sidecars structure as follows -```yaml -sidecars: - provisioner: - env: [] - resources: {} - attacher: - env: [] - resources: {} - snapshotter: - env: [] - resources: {} - livenessProbe: - resources: {} - resizer: - env: [] - resources: {} - nodeDriverRegistrar: - env: [] - resources: {} -``` - -#### Deploy driver with debug mode -To view driver debug logs, run the CSI driver with `-v=5` command line option - -To enable aws sdk debug logs, run the CSI driver with `--aws-sdk-debug-log=true` command line option. - -## Examples -Make sure you follow the [Prerequisites](README.md#Prerequisites) before the examples: -* [Dynamic Provisioning](../examples/kubernetes/dynamic-provisioning) -* [Block Volume](../examples/kubernetes/block-volume) -* [Volume Snapshot](../examples/kubernetes/snapshot) -* [Configure StorageClass](../examples/kubernetes/storageclass) -* [Volume Resizing](../examples/kubernetes/resizing) - - -## Migrating from in-tree EBS plugin -Starting from Kubernetes 1.17, CSI migration is supported as beta feature (alpha since 1.14). If you have persistent volumes that are created with in-tree `kubernetes.io/aws-ebs` plugin, you can migrate to use EBS CSI driver. To turn on the migration, set `CSIMigration` and `CSIMigrationAWS` feature gates to `true` for `kube-controller-manager`. Then drain Nodes and set the same feature gates to `true` for `kubelet`. - -To make sure dynamically provisioned EBS volumes have all tags that the in-tree volume plugin used: -* Run the external-provisioner sidecar with `--extra-create-metadata=true` cmdline option. The helm chart sets this option true by default. -* Run the CSI driver with `--k8s-tag-cluster-id=` command line option. - -To make sure that the CSI driver has permission to Attach, Detach, and Delete volumes that were dynamically provisioned and tagged by the in-tree plugin prior to migration being turned on, the IAM policy has to grant permission to operate on volumes with tag `kubernetes.io/cluster/": "owned"` like in [the example policy](./example-iam-policy.json#L85). - -**Warning**: -* kubelet *must* be drained of all pods with mounted EBS volumes ***before*** changing its CSI migration feature flags. Failure to do this will cause deleted pods to get stuck in `Terminating`, requiring a forced delete which can cause filesystem corruption. See [#679](../../../issues/679) for more details. - -## Development -Please go through [CSI Spec](https://github.com/container-storage-interface/spec/blob/master/spec.md) and [General CSI driver development guideline](https://kubernetes-csi.github.io/docs/developing.html) to get some basic understanding of CSI driver before you start. - -### Requirements -* Golang 1.15.+ -* [Ginkgo](https://github.com/onsi/ginkgo) in your PATH for integration testing and end-to-end testing -* Docker 17.05+ for releasing - -### Dependency -Dependencies are managed through go module. To build the project, first turn on go mod using `export GO111MODULE=on`, then build the project using: `make` - -### Testing -* To execute all unit tests, run: `make test` -* To execute sanity test run: `make test-sanity` -* To execute integration tests, run: `make test-integration` -* To execute e2e tests, run: `make test-e2e-single-az` and `make test-e2e-multi-az` - -### Release Process -Please see [Release Process](./RELEASE.md). - -**Notes**: -* Sanity tests make sure the driver complies with the CSI specification -* EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk. See [Integration Testing](../tests/integration/README.md) for more details. -* E2E tests exercises various driver functionalities in Kubernetes cluster. See [E2E Testing](../tests/e2e/README.md) for more details. - -### Helm and manifests -The helm chart for this project is in the `charts/aws-ebs-csi-driver` directory. The manifests for this project are in the `deploy/kubernetes` directory. All of the manifests except kustomize patches are generated by running `helm template`. This keeps the helm chart and the manifests in sync. - -When updating the helm chart: -* Generate manifests: `make generate-kustomize` -* There are values files in `deploy/kubernetes/values` used for generating some of the manifests -* When adding a new resource template to the helm chart please update the `generate-kustomize` make target, the `deploy/kubernetes/values` files, and the appropriate kustomization.yaml file(s). - -## Milestone -[Milestones page](https://github.com/c2devel/aws-ebs-csi-driver/milestones) diff --git a/docs/RELEASE.md b/docs/RELEASE.md deleted file mode 100644 index 2e97827682..0000000000 --- a/docs/RELEASE.md +++ /dev/null @@ -1,53 +0,0 @@ -# Инструкция по релизу новой версии - -Инструкция протестирована на: -```sh -# uname -r -5.6.13-100.fc30.x86_64 -# cat /etc/os-release -NAME=Fedora -VERSION="30 (Thirty)" -ID=fedora -VERSION_ID=30 -VERSION_CODENAME="" -PLATFORM_ID="platform:f30" -PRETTY_NAME="Fedora 30 (Thirty)" -ANSI_COLOR="0;34" -LOGO=fedora-logo-icon -CPE_NAME="cpe:/o:fedoraproject:fedora:30" -HOME_URL="https://fedoraproject.org/" -DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/" -SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help" -BUG_REPORT_URL="https://bugzilla.redhat.com/" -REDHAT_BUGZILLA_PRODUCT="Fedora" -REDHAT_BUGZILLA_PRODUCT_VERSION=30 -REDHAT_SUPPORT_PRODUCT="Fedora" -REDHAT_SUPPORT_PRODUCT_VERSION=30 -PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy" -# docker --version -Docker version 19.03.12, build 48a66213fe -# ./kustomize version -{Version:kustomize/v3.6.1 GitCommit:c97fa946d576eb6ed559f17f2ac43b3b5a8d5dbd BuildDate:2020-05-27T20:47:35Z GoOs:linux GoArch:amd64} -``` -## Версионирование - -Используется следующая схема версионирования - -CROC. Где X - инкрементируется с каждым новым релизом. Например при текущей версии апстрима v0.5.0 и текущей версии этой репы v0.5.0-CROC1 следующая версия будет v0.5.0-CROC2. При обновлении версии апстрима, например до v0.6.0, успешный ребейз на новый апстрим будет результирован в версию v0.6.0-CROC2. Предполагается суппорт только актуальных версий. - -Версии обозначаются гит тегами. Тегируется мастер ветка используя механизм релизов гитхаба. При создании нового релиза, описание релиза заполняется краткой сводкой изменений в новом релизе. После создания нового релиза (и тега), тег забирается на локалку (git pull upstream master --tags) и выполняется ручная сборка и публикация артефактов. - -## Артефакты - -Релизными артефактами этой репы является докер имадж и deployment конфиги для бубернетеса. При любом новом релизе необходимо обновлять kustomization.yaml и генерить бандл (например при релизе v0.5.0-CROC1): -- в файле deployment/kubernetes/stable/kustomization.yaml изменить ```newTag``` на новый актуальный (v0.5.0-CROC1) -- используя утилиту [kustomize](https://github.com/kubernetes-sigs/kustomize) собрать сингл-yaml-файл бандл для деплоймента: -``` -kustomize build ./deployment/kubernetes/stable/ > ./deployment/kubernetes/stable/k_bundle.yaml -``` - -Для создания докер имаджа необходимы установленный и настроенный докер демон - https://docs.docker.com/get-docker/ . Для сборки имаджа необходимо: -- находясь в руте репы выполнить: -```docker build -t aws-ebs-csi-driver``` -- после успешной сборки протегировать имадж: -```docker tag aws-ebs-csi-driver registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:``` -- запушить имадж в регистри (необходимы врайт права в регистри неймспейсе): -```docker push registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:``` diff --git a/docs/release.md b/docs/release.md index cd63620c9f..3efc81e0de 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,112 +1,57 @@ -# Amazon Elastic Block Store (EBS) CSI driver Release Process - -## Choose the release version and release branch - -1. Find the latest release: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases. For example, - `v1.3.1`. (Ignore helm releases prefixed by `helm-chart` like - `helm-chart-aws-ebs-csi-driver-2.5.0`). -2. Increment the version according to semantic versioning https://semver.org/. - For example, for a release that only contains bug fixes or an updated Amazon - Linux 2 base image, `v1.3.2`. -3. Find or create the corresponding release branch. Release branches correspond - to minor version. For example, for `v1.3.2` the release branch would be - `release-1.3` and it would already exist. For `v1.4.0` it would be - `release-1.4` and the branch would need to be created. If you do not have - permission to create the branch, ask an OWNER to do it - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/OWNERS. - -## Create the release commit in the release branch - -Checkout the release branch you chose above, for example `git checkout release-1.3`. - -### Update `CHANGELOG-0.x.md` - -1. Generate a Personal Access Token with `repos` permissions. -2. Run hack/release with arguments according to the version and branch you chose above: - - `--since`: the release version immediately preceding your chosen release version and the chosen release branch to generate the changelog. For example, for v1.3.2 pass `--since v1.3.1`. - - `--branch`: the release branch you chose. For example, for v1.3.2 pass `--branch release-1.3`. -``` -python3 hack/release --github-user=$GITHUB_USER --github-token=$GITHUB_TOKEN note --since $PREVIOUS_VERSION --branch $BRANCH +# Инструкция по релизу новой версии + +Инструкция протестирована на: +```sh +# uname -r +5.6.13-100.fc30.x86_64 +# cat /etc/os-release +NAME=Fedora +VERSION="30 (Thirty)" +ID=fedora +VERSION_ID=30 +VERSION_CODENAME="" +PLATFORM_ID="platform:f30" +PRETTY_NAME="Fedora 30 (Thirty)" +ANSI_COLOR="0;34" +LOGO=fedora-logo-icon +CPE_NAME="cpe:/o:fedoraproject:fedora:30" +HOME_URL="https://fedoraproject.org/" +DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/" +SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help" +BUG_REPORT_URL="https://bugzilla.redhat.com/" +REDHAT_BUGZILLA_PRODUCT="Fedora" +REDHAT_BUGZILLA_PRODUCT_VERSION=30 +REDHAT_SUPPORT_PRODUCT="Fedora" +REDHAT_SUPPORT_PRODUCT_VERSION=30 +PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy" +# docker --version +Docker version 19.03.12, build 48a66213fe +# ./kustomize version +{Version:kustomize/v3.6.1 GitCommit:c97fa946d576eb6ed559f17f2ac43b3b5a8d5dbd BuildDate:2020-05-27T20:47:35Z GoOs:linux GoArch:amd64} ``` -This will print the CHANGELOG to stdout. -3. Create a new section for the new version and copy the output there. Organize and prune the CHANGELOG at your own discretion. For example, release commits like "Release v1.3.3" are not useful and should be removed or put in a "Misc." section. - -### Update `docs/README.md` - -Search for any references to the previous version on the README, and update them if necessary. +## Версионирование -### Update `Makefile` +Используется следующая схема версионирования - -CROC. Где X - инкрементируется с каждым новым релизом. Например при текущей версии апстрима v0.5.0 и текущей версии этой репы v0.5.0-CROC1 следующая версия будет v0.5.0-CROC2. При обновлении версии апстрима, например до v0.6.0, успешный ребейз на новый апстрим будет результирован в версию v0.6.0-CROC2. Предполагается суппорт только актуальных версий. -Update the VERSION variable in the Makefile +Версии обозначаются гит тегами. Тегируется мастер ветка используя механизм релизов гитхаба. При создании нового релиза, описание релиза заполняется краткой сводкой изменений в новом релизе. После создания нового релиза (и тега), тег забирается на локалку (git pull upstream master --tags) и выполняется ручная сборка и публикация артефактов. -### Send a release PR to the release branch - -At this point you should have all changes required for the release commit. Verify the changes via `git diff` and send a new PR with the release commit against the release branch. Note that if it doesn't exist, you'll need someone with write privileges to create it for you. - -## Tag the release - -Once the PR is merged, pull the release branch locally and tag the release commit with the relase tag. You'll need push privileges for this step. +## Артефакты +Релизными артефактами этой репы является докер имадж и deployment конфиги для кубернетеса. +При любом новом релизе необходимо обновлять номер релиза в файлах Makefile и charts/aws-ebs-csi-driver/values.yaml и генерить бандл (например при релизе v1.19.0-CROC1): +- в файле Makefile в строке 15 в значении ```VERSION``` указать версию добавив актуальный суффикс CROC, например v1.19.0-CROC1 +- в файле charts/aws-ebs-csi-driver/values.yaml в строке 8 в значении ```tag``` указать версию добавив актуальный суффикс CROC, например v1.19.0-CROC1 +- запустить ```make generate-kustomize``` +- используя утилиту [kustomize](https://github.com/kubernetes-sigs/kustomize) собрать сингл-yaml-файл бандл для деплоймента: ``` -git checkout release-0.7 -git pull upstream release-0.7 -git tag v0.7.0 -git push upstream v0.7.0 +kubectl kustomize ./deploy/kubernetes/overlays/stable/ > ./deploy/kubernetes/overlays/stable/k_bundle.yaml ``` - -## Verify the release on GitHub - -The new tag should trigger a new Github release. It should be a pre-release true because images are not available yet and documentation, like README and CHANGELOG in master branch, does not yet reflect the new release. Verify that it has run by going to [Releases](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases). Then, click on the new version and verify all assets have been created: - -- Source code (zip) -- Source code (tar.gz) - -## Promote the new image on ECR - -Follow the AWS-internal process. - -## Verify the images are available - -In ECR Public: - - `docker pull public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:{release version}` - -In ECR: - - `aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 602401143452.dkr.ecr.us-west-2.amazonaws.com` - - `docker pull 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-ebs-csi-driver:{release version}` - -## Create the post-release commit in the release branch - -### Update `charts/aws-ebs-csi-driver` - -1. Update Helm `appVersion`, `version`, `tag`, and CHANGELOG - - `charts/aws-ebs-csi-driver/Chart.yaml` - - `charts/aws-ebs-csi-driver/values.yaml` - - `charts/aws-ebs-csi-driver/CHANGELOG.md` - -### Update `deploy/kubernetes` - -1. Update the kustomize overlays - - `deploy/kubernetes/overlays/stable/kustomization.yaml` - - `deploy/kubernetes/overlays/stable/ecr/kustomization.yaml` -2. Run make generate-kustomize - -### Send a post-release PR to the release branch - -The helm and kustomize deployment files must not be updated to refer to the new images until after the images have been verified available, therefore it's necessary to make these changes in a post-release PR rather than the original release PR. - -## Merge the release and post-release commits to the main branch - -Send a PR to merge both the release and post-release commits to the main branch. - -## Verify the helm chart release - -Visit the [Releases](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases) pages to verify we have a new helm chart release. - -## Update the GitHub release to be pre-release false - -Now that images are available and documentation is updated, uncheck "This is a pre-release". - -## Update AWS EKS documentation - -Update the AWS EKS documentation https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html by submitting a PR https://github.com/awsdocs/amazon-eks-user-guide/blob/master/doc_source/ebs-csi.md. For example, if the release raises the Kubernetes version requirement then the doc must reflect that. +- собранный бандл файл k_bundle.yaml надо скопировать в репозиторий kaas-resource-initializer в файлы ebs/ebs.yaml каждой версии kubenetes + +Для создания докер имаджа необходимы установленный и настроенный докер демон - https://docs.docker.com/get-docker/ . Для сборки имаджа необходимо: +- находясь в руте репы выполнить: +```docker buildx build -t aws-ebs-csi-driver .``` +- после успешной сборки протегировать имадж: +```docker tag aws-ebs-csi-driver registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:``` +- запушить имадж в регистри (необходимы врайт права в регистри неймспейсе): +```docker push registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:``` diff --git a/docs/TEST.md b/docs/testing.md similarity index 100% rename from docs/TEST.md rename to docs/testing.md From d0db321f7090b54246dc27899d17eb70a74c82ae Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Wed, 28 Jun 2023 13:38:29 +0300 Subject: [PATCH 43/45] add tolerations and labels for ebs --- charts/aws-ebs-csi-driver/values.yaml | 4 +++- deploy/kubernetes/base/controller.yaml | 2 ++ deploy/kubernetes/overlays/stable/k_bundle.yaml | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/aws-ebs-csi-driver/values.yaml b/charts/aws-ebs-csi-driver/values.yaml index e6622b0c41..66b4c91a33 100644 --- a/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/aws-ebs-csi-driver/values.yaml @@ -205,7 +205,8 @@ controller: k8sTagClusterId: logLevel: 2 userAgentExtra: "helm" - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/master: "" podAnnotations: {} podLabels: {} priorityClassName: system-cluster-critical @@ -241,6 +242,7 @@ controller: - effect: NoExecute operator: Exists tolerationSeconds: 300 + - operator: Exists # TSCs without the label selector stanza # # Example: diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index 10c1c01253..a8e4588235 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -25,6 +25,7 @@ spec: spec: nodeSelector: kubernetes.io/os: linux + node-role.kubernetes.io/master: "" serviceAccountName: ebs-csi-controller-sa priorityClassName: system-cluster-critical affinity: @@ -54,6 +55,7 @@ spec: - effect: NoExecute operator: Exists tolerationSeconds: 300 + - operator: Exists securityContext: fsGroup: 1000 runAsGroup: 1000 diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index 7f76539c32..ac04346398 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -575,6 +575,7 @@ spec: name: socket-dir nodeSelector: kubernetes.io/os: linux + node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical securityContext: fsGroup: 1000 @@ -588,6 +589,7 @@ spec: - effect: NoExecute operator: Exists tolerationSeconds: 300 + - operator: Exists volumes: - emptyDir: {} name: socket-dir From 207063940e0316c9cb4d6f2a01d387b4b8f61884 Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Wed, 5 Jul 2023 13:01:04 +0300 Subject: [PATCH 44/45] add instruction how to debug --- README.md | 4 + docs/debuging.md | 462 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 466 insertions(+) create mode 100644 docs/debuging.md diff --git a/README.md b/README.md index f644689892..6251cc3e5a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,10 @@ см. [release process](docs/release.md) +## Инструкция по отладке + +см. [debuging](docs/debuging.md) + ## Overview The [Amazon Elastic Block Store](https://aws.amazon.com/ebs/) Container Storage Interface (CSI) Driver provides a [CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md) interface used by Container Orchestrators to manage the lifecycle of Amazon EBS volumes. diff --git a/docs/debuging.md b/docs/debuging.md new file mode 100644 index 0000000000..f733bf4049 --- /dev/null +++ b/docs/debuging.md @@ -0,0 +1,462 @@ +# Вывод логов + +```sh +tail -f -n 20 $(find /var/log/pods/kube-system_ebs-csi-controller-* -type l | xargs) +``` + +# Тестирование кастомных сборок + +Для тестирования своей сборки нужно настроить стенд на свой реестр образов и S3 бакет. +Ниже - перечень шагов для подготовки стенда для тестирования. + +1. В реестре образов https://registry.hosting.croc.ru должен быть свой неймспейс, например awesomedev, сделать его публичным. + +2. Скопировать из неймспейса kaas в свой неймспейс все образы. + +3. Из файла charts/aws-ebs-csi-driver/values.yaml выписать все образы с версиями и скопировать их в свой неймспейс используя утилиту scopeo. +Пример для версии v1.19: +```sh +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.8.0-eks-1-27-3 docker://registry.cloud.croc.ru/awesomedev/node-driver-registrar:v2.8.0-eks-1-27-3 +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.10.0-eks-1-27-3 docker://registry.cloud.croc.ru/awesomedev/livenessprobe:v2.10.0-eks-1-27-3 +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.3.0-eks-1-27-3 docker://registry.cloud.croc.ru/awesomedev/external-attacher:v4.3.0-eks-1-27-3 +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v3.5.0-eks-1-27-3 docker://registry.cloud.croc.ru/awesomedev/external-provisioner:v3.5.0-eks-1-27-3 +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.8.0-eks-1-27-3 docker://registry.cloud.croc.ru/awesomedev/external-resizer:v1.8.0-eks-1-27-3 +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter:v6.2.1-eks-1-27-3 docker://registry.cloud.croc.ru/awesomedev/csi-snapshotter:v6.2.1-eks-1-27-3 +skopeo copy --dcreds 'awesomedev:secretpassword' docker://public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s:v0.1.0 docker://registry.cloud.croc.ru/awesomedev/volume-modifier-for-k8s:v0.1.0 +``` + +4. В файлах Makefile, charts/aws-ebs-csi-driver/values.yaml, deploy/kubernetes/overlays/stable/ecr/kustomization.yaml и deploy/kubernetes/overlays/stable/gcr/kustomization.yaml везде заменить неймспейс kaas на свой (в примере на awesomedev). + +5. Сгенерировать деплоймент файлы по шаблонам командой: +```sh +make generate-kustomize +``` + +6. Сгенерировать файл k_bundle.yaml командой: +```sh +kubectl kustomize ./deploy/kubernetes/overlays/stable/ > ./deploy/kubernetes/overlays/stable/k_bundle.yaml +``` + +7. Скопировать сгенерированный k_bundle.yaml в склонированный репозиторий [kaas-resource-initializer](https://ghe.cloud.croc.ru/c2/kaas-resource-initializer) в каждую версию кубернетиса: deployment/<версия кубера>/ebs/ebs.yaml + +8. Находясь в каталоге со склонированным репозиторием [kaas-resource-initializer](https://ghe.cloud.croc.ru/c2/kaas-resource-initializer), сгенерировать новые конфиги и rpm пакеты командами: +```sh +external_repo=http://172.25.6.71/pub/repos/slices/23.7/7.8/addons/ make +external_repo=http://172.25.6.71/pub/repos/slices/23.7/7.8/addons/ make create +``` + +9. В своём **продовском** облаке создать новый S3 бакет, например с именем awesomekaas + +10. Залить сгенерированные конфиги и rpm пакеты в бакет из kaas-resource-initializer с помощью утилиты aws (реквизиты профиля default к облаку прописаны в ~/.aws/credentials): +```sh +aws --no-verify-ssl --profile default --endpoint-url https://storage.cloud.croc.ru s3 cp --acl public-read --recursive ./output/ s3://awesomekaas/ +``` + +11. Находясь в каталоге с проектом aws-ebs-csi-driver собрать тестируемый образ, протегировать и залить в свой неймспейс реестра образов. +Пример командя для версии v1.19.0-CROC1: +```sh +docker login registry.cloud.croc.ru/awesomedev +docker buildx build -t aws-ebs-csi-driver . +docker tag aws-ebs-csi-driver registry.cloud.croc.ru/awesomedev/aws-ebs-csi-driver:v1.19.0-CROC1 +docker push registry.cloud.croc.ru/awesomedev/aws-ebs-csi-driver:v1.19.0-CROC1 +``` + +12. В конфиге тестового стенда облака /etc/c2.deployment.conf, прописать свои регистри и s3 бакет. +- в секции KUBERNETES поменять docker_registry_namespace с kaas на awesomedev +- в секции KUBERNETES поменять bucket_address с kaas на awesomekaas + +13. В MongoDB в коллекции kubernetes.versions поменять настройки на свои: +```js +db["kubernetes.versions"].updateOne({"_id": "base"}, {"$set": {"bucket_address": "https://storage.cloud.croc.ru/awesomekaas", "docker_registry_namespace": "awesomedev"}}) +``` + +14. Перезапустить сервисы стенда c2-deploy и все c2-ks-* + +15. На стенде в консоли облака создать пользователя для EBS провайдера. + +16. На стенде в консоли облака создать новый кубернетес кластер, присвоив SSH ключ и Elastic IP, и активирвоать EBS провайдер с указанием созданного пользователя для EBS провайдера. + +17. Если стенд не железный, а dev, то на нём не будет резолвится хост AWS_EC2_ENDPOINT, и его надо прописать вручную. +Для этого надо: +- зайти по ssh на мастер ноду +- изменить файл /tmp/ebs/api_enpoint.yaml добавив секцию с hostAliases и внешним IP адресов dev стенда, чтобы получилось так (этот файл генерируется в коде из репозитория [cloud-init-configs](https://ghe.cloud.croc.ru/c2/cloud-init-configs)): +```yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + template: + spec: + hostAliases: + - ip: "here must be external IP address of dev stand" + hostnames: + - "api.dev.c2.croc.ru" + containers: + - name: ebs-plugin + env: + - name: AWS_EC2_ENDPOINT + value: https://api.dev.c2.croc.ru:8443 + - name: AWS_EC2_ENDPOINT_UNSECURE + value: "true" +``` +- передеплоить ebs провайдер командой: +```sh +kubectl apply -k /tmp/ebs +``` + +18. На этом всё, кубернетес должен будет подниматься с кастомной сборкой aws-ebs-csi-driver. + + +# Активация отладчика + +Для использования go отладчика delve нужно внести ряд изменений в сборочные и конфигурационные файлы. +Ниже - перечень шагов по изменению и сборки образа с отладчиком. Эти шаги подразумевают, что были выполнены предыдущие необходимые шаги для кастомной сборки. + +1. В Dockerfile на этап сборки добавить шаги по сборке и установки отладчика delve и запуска драйвера через отладчик. +На этап сборки добавить строки для скачивания отладчика delve: +```Dockerfile +ENV CGO_ENABLED 0 +RUN go install github.com/go-delve/delve/cmd/dlv@latest +``` +Где переменная окружение CGO_ENABLED в значении 0 указывает скачать статически собранный delve, что требуется т.к. в docker образе отсутствуют нужные динамические библиотеки. + +Изменить строку ENTRYPOINT указав запуск aws-ebs-csi-driver через отладчик: +```Dockerfile +ENTRYPOINT ["/dlv", "--listen=:40000", "--headless", "--accept-multiclient", "--continue", "--api-version=2", "--log", "exec", "/bin/aws-ebs-csi-driver", "--"] +``` +Где: +- `--listen=:40000` - указывает запустить отладчик на порту 40000. +- `--headless` - указывает запустить только сервеную часть отладчика. +- `--accept-multiclient` - разрешает принимать множество клиентских соединений. +- `--continue` - разрешает запускаемому процессу выполняться без подключенных клиентов отладчика. +- `--log` - логгирует события отладчика. +- `--api-version=2` - указывает использовать новую версию JSON-RPC API. +- `exec /bin/aws-ebs-csi-driver` - указывает исполняемый файл EBS драйвера для исполнения под отладчиком. +- `--` - разделитель используемый для аргументов исполняемого файла EBS драйвера, которые передаются в docker контейнер. +Пример итогового Dockerfile: +```Dockerfile +FROM --platform=$BUILDPLATFORM golang:1.20 AS builder + +ENV CGO_ENABLED 0 +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +WORKDIR /go/src/github.com/c2devel/aws-ebs-csi-driver +COPY go.* . +ARG GOPROXY +RUN go mod download +COPY . . +ARG TARGETOS +ARG TARGETARCH +ARG VERSION +RUN OS=$TARGETOS ARCH=$TARGETARCH make $TARGETOS/$TARGETARCH + +FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest.2 AS linux-amazon +COPY --from=builder /go/src/github.com/c2devel/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver + +COPY --from=builder /go/bin/dlv / + +ENTRYPOINT ["/dlv", "--listen=:40000", "--headless", "--continue", "--accept-multiclient", "--api-version=2", "--log", "exec", "/bin/aws-ebs-csi-driver", "--"] +``` + +2. В Makefile убрать флаги сборки `-s -w`, которые убирают отладочные символы из собранного исполняемого файла. +Пример строки сборочных флагов с этими флагами: +```sh +LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/cloud.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE} -s -w" +``` +Пример без них (целевой пример): +```sh +LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/cloud.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE}" +``` + +3. В сгенерированном файле k_bundle.yaml поправить Deployment приложения ebs-csi-controller: +- Указать количество реплик `replicas` равное 1, чтобы отладчик всегда был подключен к той реплике, на которой выполняются действия. +- В аргументах запуска контейнера `ebs-plugin` изменить значение аргумента `--v` с 2 на 10, чтобы повысить детальность вывода в лог. +- В секцию `ports` контейнера `ebs-plugin` добавить порт отладчика 40000: `- containerPort: 40000` +- Увеличить количество оперативной памяти в запрашиваемых ресурсах до 256Mi и в лимитах до 512Mi, т.к. приложение под отладчиком потрбляет больше ресурсов, а с ресурсами по умолчанию выходит за пределы лимитов и убивается out of memory киллером. + +Пример секции Deployment целиком: +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - ebs-csi-controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logging-format=text + - --user-agent-extra=kustomize + - --v=10 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + value: https://api.cloud.croc.ru + - name: AWS_REGION + value: croc + envFrom: null + image: registry.cloud.croc.ru/vladkuznetsov/aws-ebs-csi-driver:v1.19.0-CROC1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 40000 + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + memory: 512Mi + requests: + cpu: 10m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: registry.cloud.croc.ru/vladkuznetsov/external-provisioner:v3.5.0-eks-1-27-3 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: registry.cloud.croc.ru/vladkuznetsov/external-attacher:v4.3.0-eks-1-27-3 + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --leader-election=true + - --extra-create-metadata + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: registry.cloud.croc.ru/vladkuznetsov/csi-snapshotter:v6.2.1-eks-1-27-3 + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --handle-volume-inuse-error=false + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: null + image: registry.cloud.croc.ru/vladkuznetsov/external-resizer:v1.8.0-eks-1-27-3 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + envFrom: null + image: registry.cloud.croc.ru/vladkuznetsov/livenessprobe:v2.10.0-eks-1-27-3 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + node-role.kubernetes.io/master: "" + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - operator: Exists + volumes: + - emptyDir: {} + name: socket-dir +``` +Скопировать k_bundle.yaml в склонированный репозиторий [kaas-resource-initializer](https://ghe.cloud.croc.ru/c2/kaas-resource-initializer) в каждую версию кубернетиса: deployment/<версия кубера>/ebs/ebs.yaml, сгенерировать конфигурационные файлы и выполнить заливку в S3 бакет в соответствии с шагами по тестированию кастомной сборки. + +4. На стенде в консоли облака в Security Groups открыть порт отладчика 40000 + +5. В соответствии с шагами по тестированию кастомной сборки выполнить сборку и заливку образа. + +6. В соответствии с шагами по тестированию кастомной сборки создать новый kubernetes калстер. + +7. Зайти по SSH на мастер ноду кубернетеса и выполнять коману для пробасывания порта отладчика на хост машину: +```sh +kubectl port-forward --address 0.0.0.0 deployment/ebs-csi-controller 40000:40000 --namespace=kube-system +``` + +8. В IDE vscode в проекте и исходным кодом aws-ebs-csi-driver настроить `launch.json` указав Elastiс IP мастер ноды kubernetes: +``` +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Remote debug", + "type": "go", + "debugAdapter": "dlv-dap", + "request": "attach", + "mode": "remote", + "port": 40000, + "host": "Elastiс IP мастер ноды kubernetes", + "substitutePath": [ + { "from": "${workspaceFolder}", "to": "/go/src/github.com/c2devel/aws-ebs-csi-driver" }, + ] + } + ], +} +``` + +9. Запустить отладку в vscode. + +10. Предполагаемая входная точка аттачмента диска будет в файле pkg/driver/controller.go в функции ControllerPublishVolume. +В её первой строке можно установить breakpoint для начала отладки аттачмента диска. From 2e0ed34500a5d6efdff7b6c4a1f23309b2050ac6 Mon Sep 17 00:00:00 2001 From: Vladimir Kuznetsov Date: Tue, 11 Jul 2023 14:32:08 +0300 Subject: [PATCH 45/45] fix kustomize after rebase --- charts/aws-ebs-csi-driver/values.yaml | 2 +- deploy/kubernetes/base/controller.yaml | 2 +- deploy/kubernetes/base/node.yaml | 3 +- .../kubernetes/overlays/stable/k_bundle.yaml | 61 ++++++++++++------- pkg/cloud/devicemanager/device_names.go | 2 +- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/charts/aws-ebs-csi-driver/values.yaml b/charts/aws-ebs-csi-driver/values.yaml index 66b4c91a33..7ec10c06bc 100644 --- a/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/aws-ebs-csi-driver/values.yaml @@ -5,7 +5,7 @@ image: repository: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver # Overrides the image tag whose default is v{{ .Chart.AppVersion }} - tag: "v1.19.0-CROC1" + tag: "v1.20.0-CROC1" pullPolicy: IfNotPresent # -- Custom labels to add into metadata diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index a8e4588235..426981ba2f 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -63,7 +63,7 @@ spec: runAsUser: 1000 containers: - name: ebs-plugin - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.20.0-CROC1 imagePullPolicy: IfNotPresent args: # - {all,controller,node} # specify the driver mode diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node.yaml index 324332417f..d1704a9510 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node.yaml @@ -44,7 +44,7 @@ spec: runAsUser: 0 containers: - name: ebs-plugin - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.20.0-CROC1 imagePullPolicy: IfNotPresent args: - node @@ -60,7 +60,6 @@ spec: fieldPath: spec.nodeName - name: AWS_REGION value: croc - envFrom: volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet diff --git a/deploy/kubernetes/overlays/stable/k_bundle.yaml b/deploy/kubernetes/overlays/stable/k_bundle.yaml index ac04346398..52c38de395 100644 --- a/deploy/kubernetes/overlays/stable/k_bundle.yaml +++ b/deploy/kubernetes/overlays/stable/k_bundle.yaml @@ -1,4 +1,5 @@ apiVersion: v1 +automountServiceAccountToken: true kind: ServiceAccount metadata: labels: @@ -7,6 +8,7 @@ metadata: namespace: kube-system --- apiVersion: v1 +automountServiceAccountToken: true kind: ServiceAccount metadata: labels: @@ -15,6 +17,24 @@ metadata: namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ebs-csi-leases-role + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: @@ -153,17 +173,6 @@ rules: - get - list - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - delete - - update - - create - apiGroups: - storage.k8s.io resources: @@ -277,6 +286,22 @@ rules: - update --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-leases-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ebs-csi-leases-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -424,8 +449,7 @@ spec: value: https://api.cloud.croc.ru - name: AWS_REGION value: croc - envFrom: null - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.20.0-CROC1 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -470,7 +494,6 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/external-provisioner:v3.5.0-eks-1-27-3 imagePullPolicy: IfNotPresent name: csi-provisioner @@ -493,7 +516,6 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/external-attacher:v4.3.0-eks-1-27-3 imagePullPolicy: IfNotPresent name: csi-attacher @@ -516,7 +538,6 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/csi-snapshotter:v6.2.1-eks-1-27-3 imagePullPolicy: IfNotPresent name: csi-snapshotter @@ -536,10 +557,10 @@ spec: - --csi-address=$(ADDRESS) - --v=2 - --handle-volume-inuse-error=false + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/external-resizer:v1.8.0-eks-1-27-3 imagePullPolicy: IfNotPresent name: csi-resizer @@ -557,7 +578,6 @@ spec: name: socket-dir - args: - --csi-address=/csi/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/livenessprobe:v2.10.0-eks-1-27-3 imagePullPolicy: IfNotPresent name: liveness-probe @@ -650,8 +670,7 @@ spec: fieldPath: spec.nodeName - name: AWS_REGION value: croc - envFrom: null - image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.19.0-CROC1 + image: registry.cloud.croc.ru/kaas/aws-ebs-csi-driver:v1.20.0-CROC1 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 @@ -692,7 +711,6 @@ spec: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/node-driver-registrar:v2.8.0-eks-1-27-3 imagePullPolicy: IfNotPresent livenessProbe: @@ -723,7 +741,6 @@ spec: name: probe-dir - args: - --csi-address=/csi/csi.sock - envFrom: null image: registry.cloud.croc.ru/kaas/livenessprobe:v2.10.0-eks-1-27-3 imagePullPolicy: IfNotPresent name: liveness-probe diff --git a/pkg/cloud/devicemanager/device_names.go b/pkg/cloud/devicemanager/device_names.go index 5928f79c02..aa441aeec5 100644 --- a/pkg/cloud/devicemanager/device_names.go +++ b/pkg/cloud/devicemanager/device_names.go @@ -21,7 +21,7 @@ package devicemanager // /dev/xvda is broken on Windows (despite the API allowing it) // /dev/xvddx is the last allowed device name in the /dev/xvd{a-z}{a-z} series // /dev/xvdc{a-z} don't work on some Windows instance types -var deviceNames = []string{ +var deviceNames = []string{ //nolint:unused "/dev/xvdb", "/dev/xvdc", "/dev/xvdd",