Add support for local registry use to the local-dev tooling

.gitignore

@@ -36,3 +36,6 @@ policy-tester
 *.swp
 
 gha-creds-*.json
+
+# Kind cluster configuration produced by the local-dev tool
+kind.yaml

README.md

@@ -47,7 +47,15 @@ Then run it pointing to a YAML file containing a ClusterImagePolicy, and an imag
 ## Local Development
 
 You can spin up a local [Kind](https://kind.sigs.k8s.io/) K8s cluster to test local changes to the policy controller using the ``
-CLI tool. Build the tool with `make local-dev` and then run it with `./bin/local-dev setup --cluster-name=<my cluster name> --ko-docker-repo=<some-docker-repo>`.
+CLI tool. Build the tool with `make local-dev` and then run it with `./bin/local-dev setup`.
+
+It optionally accepts the following:
+
+```
+--cluster-name
+--k8s-version
+--registry-url
+```
 
 You can clean up the cluster with `./bin/local-dev clean --cluster-name=<my cluster name>`.
 
@@ -57,6 +65,12 @@ You will need to have the following tools installed to use this:
 - [ko](https://ko.build/install/)
 - [kubectl](https://kubernetes.io/docs/tasks/tools/)
 
+### Use local registry
+
+If you would like to use the local Kind registry instead of a live one,
+do not include the `registry-url` flag when calling the CLI. It will default to using the local registry. But before running the CLI, you must add the following line to your `/etc/hosts` file first:
+`127.0.0.1 registry.local`
+
 ## Support Policy
 
 This policy-controller's versions are able to run in the following versions of Kubernetes:

cmd/local-dev/clean.go

@@ -17,10 +17,15 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"log"
 	"os/exec"
 
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -54,11 +59,43 @@ func clean() {
 
 	// clean up the local cluster
 	clusterName := viper.GetString("cluster-name")
-	fmt.Printf("Cleaning up the kind cluster %s...", clusterName)
+	fmt.Printf("Cleaning up the kind cluster %s...\n", clusterName)
 
 	removeCluster := exec.Command("kind", "delete", "cluster", "--name", clusterName)
 	removeCluster.Stderr = &stderr
 	if err := removeCluster.Run(); err != nil {
 		log.Fatal(buildFatalMessage(err, stderr))
 	}
+
+	if err := cleanUpRegistry(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func cleanUpRegistry() error {
+	ctx := context.Background()
+	dockerCLI, err := client.NewClientWithOpts(
+		client.FromEnv,
+		client.WithAPIVersionNegotiation(),
+	)
+	if err != nil {
+		return err
+	}
+	defer dockerCLI.Close()
+
+	containers, err := dockerCLI.ContainerList(ctx, types.ContainerListOptions{Filters: filters.NewArgs(filters.KeyValuePair{Key: "name", Value: "registry.local"})})
+	if err != nil {
+		return err
+	}
+
+	if len(containers) > 0 {
+		fmt.Println("Cleaning up registry.local...")
+		if err := dockerCLI.ContainerStop(ctx, containers[0].ID, container.StopOptions{}); err != nil {
+			return err
+		}
+		if err := dockerCLI.ContainerRemove(ctx, containers[0].ID, types.ContainerRemoveOptions{}); err != nil {
+			return err
+		}
+	}
+	return nil
 }

cmd/local-dev/setup.go

@@ -17,21 +17,54 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io/fs"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
+	"strconv"
 
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
+const (
+	localRegistryName            = "registry.local"
+	localRegistryPort            = 5001
+	defaultKindestNodeVersionTag = "v1.27.3"
+)
+
+var kindClusterConfig = `
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+name: "%s"
+nodes:
+- role: control-plane
+  image: "%s"
+# Configure registry for KinD.
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s:%d"]
+    endpoint = ["http://%s:%d"]
+`
+
+// check that a supplied image version is in the expected semver format: v<major>.<minor>.<patch>
+var semverRegexp = regexp.MustCompile("^v[0-9]+.[0-9]+.[0-9]+$")
+
+// check that registry URLs are in the expected format <url>:<port>
+var registryURLRegexp = regexp.MustCompile("^[a-zA-Z0-9]+.[a-z]+:[0-9]+$")
+
 func addSetupFlags(cmd *cobra.Command) {
 	cmd.Flags().String("cluster-name", "policy-controller-demo", "name of the dev policy controller cluster")
-	cmd.Flags().String("ko-docker-repo", "", "name of the Ko Docker repository to use")
-	cmd.MarkFlagRequired("ko-docker-repo") //nolint:errcheck
+	cmd.Flags().String("k8s-version", defaultKindestNodeVersionTag, "name of the Ko Docker repository to use")
+	cmd.Flags().String("registry-url", "registry.local", "URL and port of the Ko Docker registry to use. Expected format: <url>:<port>. If no registry is provided, the local Kind registry will be used")
 }
 
 var setupCmd = &cobra.Command{
@@ -56,21 +89,50 @@ func buildFatalMessage(err error, stderr bytes.Buffer) string {
 func setup() {
 	var stderr bytes.Buffer
 
-	koDockerRepo := viper.GetString("ko-docker-repo")
-	err := os.Setenv("KO_DOCKER_REPO", koDockerRepo)
-	if err != nil {
-		log.Fatal(buildFatalMessage(err, stderr))
+	registryURL := viper.GetString("registry-url")
+	if registryURL == localRegistryName {
+		fullLocalRegistryURL := fmt.Sprintf("%s:%d/sigstore", localRegistryName, localRegistryPort)
+		err := os.Setenv("KO_DOCKER_REPO", fullLocalRegistryURL)
+		if err != nil {
+			log.Fatal(buildFatalMessage(err, stderr))
+		}
+	} else {
+		if !registryURLRegexp.Match([]byte(registryURL)) {
+			log.Fatal(fmt.Errorf("provided registry URL is not in the expected format: <url>:<port>"))
+		}
+		err := os.Setenv("KO_DOCKER_REPO", registryURL)
+		if err != nil {
+			log.Fatal(buildFatalMessage(err, stderr))
+		}
 	}
 
 	// Create the new Kind cluster
 	clusterName := viper.GetString("cluster-name")
-	fmt.Println("Creating Kind cluster " + clusterName)
-	startKindCluster := exec.Command("kind", "create", "cluster", "--name", clusterName)
+	fmt.Printf("Creating Kind cluster %s...\n", clusterName)
+
+	clusterConfig, err := createKindConfig(clusterName, viper.GetString("k8s-version"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	configBytes := []byte(clusterConfig)
+	err = os.WriteFile("kind.yaml", configBytes, 0600)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	startKindCluster := exec.Command("kind", "create", "cluster", "--config", "kind.yaml")
 	startKindCluster.Stderr = &stderr
-	if err = startKindCluster.Run(); err != nil {
+	if err := startKindCluster.Run(); err != nil {
 		log.Fatal(buildFatalMessage(err, stderr))
 	}
 
+	if registryURL == localRegistryName {
+		if err = setupLocalRegistry(); err != nil {
+			log.Fatal(err)
+		}
+	}
+
 	setGitHash := exec.Command("git", "rev-parse", "HEAD")
 	setGitHash.Stderr = &stderr
 	outBytes, err := setGitHash.Output()
@@ -121,6 +183,54 @@ func setup() {
 	}
 }
 
+func createKindConfig(clusterName, k8sVersion string) (string, error) {
+	// check that the provided version is in the expected format and use it
+	if !semverRegexp.Match([]byte(k8sVersion)) {
+		return "", fmt.Errorf("provided k8s version %s is not in the expected semver format v<major>.<minor>.<patch>", k8sVersion)
+	}
+
+	kindImage := fmt.Sprintf("kindest/node:%s", k8sVersion)
+	return fmt.Sprintf(kindClusterConfig, clusterName, kindImage, localRegistryName, localRegistryPort, localRegistryName, localRegistryPort), nil
+}
+
+func setupLocalRegistry() error {
+	dockerCLI, err := client.NewClientWithOpts(
+		client.FromEnv,
+		client.WithAPIVersionNegotiation(),
+	)
+	if err != nil {
+		return nil
+	}
+	defer dockerCLI.Close()
+
+	fmt.Printf("\nStarting local registry %s...\n", localRegistryName)
+
+	ctx := context.Background()
+	resp, err := dockerCLI.ContainerCreate(ctx, &container.Config{
+		Image:        "registry:2",
+		Env:          []string{fmt.Sprintf("REGISTRY_HTTP_ADDR=0.0.0.0:%d", localRegistryPort)},
+		ExposedPorts: nat.PortSet{"5001/tcp": struct{}{}},
+	}, &container.HostConfig{
+		RestartPolicy: container.RestartPolicy{Name: "always"},
+		PortBindings: nat.PortMap{
+			"5001/tcp": []nat.PortBinding{
+				{HostIP: "127.0.0.1", HostPort: strconv.Itoa(localRegistryPort)},
+			},
+		},
+	}, nil, nil, localRegistryName)
+	if err != nil {
+		return err
+	}
+
+	if err := dockerCLI.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
+		return err
+	}
+
+	fmt.Println("Connecting network between kind with local registry ...")
+
+	return dockerCLI.NetworkConnect(ctx, "kind", localRegistryName, nil)
+}
+
 func init() {
 	addSetupFlags(setupCmd)
 	rootCmd.AddCommand(setupCmd)

go.mod

@@ -59,6 +59,8 @@ require (
 require github.com/spf13/cobra v1.7.0
 
 require (
+	github.com/docker/docker v24.0.0+incompatible
+	github.com/docker/go-connections v0.4.0
 	github.com/go-jose/go-jose/v3 v3.0.0
 	github.com/sigstore/sigstore/pkg/signature/kms/aws v1.7.1
 	github.com/sigstore/sigstore/pkg/signature/kms/azure v1.7.1
@@ -91,6 +93,7 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
 	github.com/ThalesIgnite/crypto11 v1.2.5 // indirect
@@ -142,8 +145,8 @@ require (
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/cli v24.0.0+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/docker v24.0.0+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect

go.sum

@@ -67,6 +67,7 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0 h1:4Kynh6
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v0.12.0/go.mod h1:Q28U+75mpCaSCDowNEmhIo/rmgdkqmkmzI7N6TGR4UY=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h1:cw4zVQgBby0Z5f2v0itn6se2dDP17nTjbZFXW5uPyHA=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc=
@@ -94,6 +95,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDm
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
@@ -287,6 +290,10 @@ github.com/docker/docker v24.0.0+incompatible h1:z4bf8HvONXX9Tde5lGBMQ7yCJgNahmJ
 github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
@@ -672,6 +679,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -680,6 +688,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/mozillazg/docker-credential-acr-helper v0.3.0 h1:DVWFZ3/O8BP6Ue3iS/Olw+G07u1hCq1EOVCDZZjCIBI=
 github.com/mozillazg/docker-credential-acr-helper v0.3.0/go.mod h1:cZlu3tof523ujmLuiNUb6JsjtHcNA70u1jitrrdnuyA=
 github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto=

third_party/VENDOR-LICENSE/github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys/LICENSE.txt

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

third_party/VENDOR-LICENSE/github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal/LICENSE.txt

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE