diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 6fcc6014f..622e30745 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -11,7 +11,7 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: "1.21.4"
+  GO_VERSION: "1.21.6"
   K8S_VERSION: "v1.27.3"
   KIND_CLUSTER_NAME: "kind"
 
@@ -33,9 +33,10 @@ jobs:
       run: |
         mkdir _output
 
-        docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master -f artifacts/images/agent-build.Dockerfile .
+        # TODO: match BASEIMAGE with Makefile default (nonroot variant)
+        docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master -f artifacts/images/agent-build.Dockerfile --build-arg BUILDERIMAGE=golang:${{ env.GO_VERSION }} --build-arg BASEIMAGE=gcr.io/distroless/static-debian11 .
         docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
-        docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-server:master -f artifacts/images/server-build.Dockerfile .
+        docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-server:master -f artifacts/images/server-build.Dockerfile --build-arg BUILDERIMAGE=golang:${{ env.GO_VERSION }} --build-arg BASEIMAGE=gcr.io/distroless/static-debian11 .
         docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar
 
     - uses: actions/upload-artifact@v2
diff --git a/Makefile b/Makefile
index f79b18e98..e3aac724e 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,8 @@ ARCH_LIST ?= amd64 arm arm64 ppc64le s390x
 RELEASE_ARCH_LIST = amd64 arm64
 # The output type could either be docker (local), or registry.
 OUTPUT_TYPE ?= docker
+BUILDERIMAGE ?= golang:1.21.6
+BASEIMAGE ?= gcr.io/distroless/static-debian11:nonroot
 
 ifeq ($(GOPATH),)
 export GOPATH := $(shell go env GOPATH)
@@ -199,7 +201,7 @@ docker-push: docker-push/proxy-agent docker-push/proxy-server
 docker-build/proxy-agent: cmd/agent/main.go proto/agent/agent.pb.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
 	echo "Building proxy-agent for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg ARCH=$(ARCH) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg BUILDERIMAGE=$(BUILDERIMAGE) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(ARCH):${TAG}
 
 .PHONY: docker-push/proxy-agent
 docker-push/proxy-agent: docker-build/proxy-agent
@@ -210,7 +212,7 @@ docker-push/proxy-agent: docker-build/proxy-agent
 docker-build/proxy-server: cmd/server/main.go proto/agent/agent.pb.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
 	echo "Building proxy-server for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg ARCH=$(ARCH) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg BUILDERIMAGE=$(BUILDERIMAGE) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
 
 .PHONY: docker-push/proxy-server
 docker-push/proxy-server: docker-build/proxy-server
@@ -221,7 +223,7 @@ docker-push/proxy-server: docker-build/proxy-server
 docker-build/proxy-test-client: cmd/test-client/main.go proto/agent/agent.pb.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
 	echo "Building proxy-test-client for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg ARCH=$(ARCH) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg BUILDERIMAGE=$(BUILDERIMAGE) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
 
 .PHONY: docker-push/proxy-test-client
 docker-push/proxy-test-client: docker-build/proxy-test-client
@@ -232,7 +234,7 @@ docker-push/proxy-test-client: docker-build/proxy-test-client
 docker-build/http-test-server: cmd/test-server/main.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
 	echo "Building http-test-server for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg ARCH=$(ARCH) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg BUILDERIMAGE=$(BUILDERIMAGE) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG}
 
 .PHONY: docker-push/http-test-server
 docker-push/http-test-server: docker-build/http-test-server
diff --git a/artifacts/images/agent-build.Dockerfile b/artifacts/images/agent-build.Dockerfile
index f7e91ae3c..27f12398f 100644
--- a/artifacts/images/agent-build.Dockerfile
+++ b/artifacts/images/agent-build.Dockerfile
@@ -1,5 +1,9 @@
 # Build the proxy-agent binary
-FROM golang:1.21.6 as builder
+
+ARG BUILDERIMAGE
+ARG BASEIMAGE
+
+FROM ${BUILDERIMAGE} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -24,8 +28,8 @@ COPY proto/  proto/
 ARG ARCH
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent
 
-# Copy the loader into a thin image
-FROM gcr.io/distroless/static-debian11:nonroot
+FROM ${BASEIMAGE}
+
 WORKDIR /
 COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-agent .
 ENTRYPOINT ["/proxy-agent"]
diff --git a/artifacts/images/server-build.Dockerfile b/artifacts/images/server-build.Dockerfile
index 130b7112b..97579b97e 100644
--- a/artifacts/images/server-build.Dockerfile
+++ b/artifacts/images/server-build.Dockerfile
@@ -1,5 +1,9 @@
 # Build the proxy-server binary
-FROM golang:1.21.6 as builder
+
+ARG BUILDERIMAGE
+ARG BASEIMAGE
+
+FROM ${BUILDERIMAGE} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -23,8 +27,8 @@ COPY proto/  proto/
 ARG ARCH
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server
 
-# Copy the loader into a thin image
-FROM gcr.io/distroless/static-debian11:nonroot
+FROM ${BASEIMAGE}
+
 WORKDIR /
 COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-server .
 ENTRYPOINT ["/proxy-server"]
diff --git a/artifacts/images/test-client-build.Dockerfile b/artifacts/images/test-client-build.Dockerfile
index d68facaa1..4b1deba14 100644
--- a/artifacts/images/test-client-build.Dockerfile
+++ b/artifacts/images/test-client-build.Dockerfile
@@ -1,5 +1,9 @@
 # Build the client binary
-FROM golang:1.21.6 as builder
+
+ARG BUILDERIMAGE
+ARG BASEIMAGE
+
+FROM ${BUILDERIMAGE} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -23,8 +27,8 @@ COPY proto/  proto/
 ARG ARCH
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client
 
-# Copy the loader into a thin image
-FROM gcr.io/distroless/static-debian11
+FROM ${BASEIMAGE}
+
 WORKDIR /
 COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-test-client .
 ENTRYPOINT ["/proxy-test-client"]
diff --git a/artifacts/images/test-server-build.Dockerfile b/artifacts/images/test-server-build.Dockerfile
index f7723eb98..8f09f8d6b 100644
--- a/artifacts/images/test-server-build.Dockerfile
+++ b/artifacts/images/test-server-build.Dockerfile
@@ -1,5 +1,9 @@
 # Build the http test server binary
-FROM golang:1.21.6 as builder
+
+ARG BUILDERIMAGE
+ARG BASEIMAGE
+
+FROM ${BUILDERIMAGE} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -22,8 +26,8 @@ COPY cmd/    cmd/
 ARG ARCH
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server
 
-# Copy the loader into a thin image
-FROM gcr.io/distroless/static-debian11
+FROM ${BASEIMAGE}
+
 WORKDIR /
 COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/http-test-server .
 ENTRYPOINT ["/http-test-server"]