ci: add kind testing

.github/workflows/kind.yaml

@@ -0,0 +1,110 @@
+name: Create Cluster with Registry
+
+on:
+  push: {}
+
+jobs:
+  setup-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker registry
+        run: |
+          docker network create kind || true
+          docker run -d --network kind --name registry -p 5000:5000 registry:2
+
+      - name: Write config file
+        run: |
+          cat <<EOF > kind-config.yaml
+          apiVersion: kind.x-k8s.io/v1alpha4
+          kind: Cluster
+          containerdConfigPatches:
+          - |
+            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
+              endpoint = ["http://registry:5000"]
+          
+          kubeadmConfigPatches:
+          - |
+            kind: ClusterConfiguration
+            apiServer:
+              extraArgs:
+                "service-node-port-range": "30000-30005"
+
+          nodes:
+          - role: control-plane
+            extraPortMappings:
+            - containerPort: 80
+              hostPort: 80
+            - containerPort: 30000
+              hostPort: 30000
+            - containerPort: 30001
+              hostPort: 30001
+            - containerPort: 30002
+              hostPort: 30002
+            - containerPort: 30003
+              hostPort: 30003
+            - containerPort: 30004
+              hostPort: 30004
+            - containerPort: 30005
+              hostPort: 30005
+          EOF
+
+      - name: Set up Kind cluster
+        uses: helm/kind-action@v1.7.0
+        with:
+          version: v0.20.0
+          config: kind-config.yaml
+          cluster_name: kind
+        env:
+          KIND_EXPERIMENTAL_DOCKER_NETWORK: kind
+
+      - name: Build and push CM
+        run: |
+          docker build \
+              -t localhost:5000/ctferio/chall-manager:${{ github.sha }} \
+              -f Dockerfile.chall-manager \
+              .
+          docker push localhost:5000/ctferio/chall-manager:${{ github.sha }}
+
+      - name: Build and push CMJ
+        run: |
+          docker build \
+              -t localhost:5000/ctferio/chall-manager-janitor:${{ github.sha }} \
+              -f Dockerfile.chall-manager-janitor \
+              .
+          docker push localhost:5000/ctferio/chall-manager-janitor:${{ github.sha }}
+
+      - name: Install Pulumi
+        uses: pulumi/actions@v4
+      - name: Prepare environment
+        run: |
+          pulumi login --local
+          kubectl create ns cm-in-ci
+
+      - name: Romeo environment
+        id: env
+        uses: ctfer-io/romeo/environment@1c42bc54350537e321f25b78acd8b2525a57f654
+        with:
+          kubeconfig: ~/.kube/config
+          namespace: cm-in-ci
+          storage-class-name: standard
+
+      - name: Configure stack
+        run: |
+          export PULUMI_CONFIG_PASSPHRASE=""
+
+          cd deploy
+          pulumi stack init dev
+          pulumi config set private-registry "localhost:5000/"
+          pulumi config set tag ${{ github.sha }}
+          pulumi config set --path romeo.claim-name
+          pulumi config set namespace ${{ steps.env.outputs.namespace }}
+          pulumi config set pvc-access-mode ReadWriteOnce
+          pulumi config set expose true
+          pulumi up -y
+
+          URL="http://$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane):$(pulumi stack output exposed_port)"
+          echo $URL
+          curl "${URL}/healthcheck"

deploy/Pulumi.yaml

@@ -30,6 +30,14 @@ config:
     type: boolean
     description: If set to true, turns on the REST API Swagger UI. Do not activate in production. (Optional)
     default: false
+  pvc-access-mode:
+    type: string
+    description: The access mode to use for the PVC. (Optional)
+    default: ReadWriteMany
+  expose:
+    type: boolean
+    description: Whether to expose to external networking the Chall-Manager service. DO NOT TURN ON IF YOU DON'T UNDERSTAND THE IMPACT.
+    default: false
   otel.endpoint:
     type: string
     description: The OpenTelemetry Collector endpoint to set signals to. (Optional)

deploy/main.go

@@ -30,9 +30,13 @@ func main() {
 			PrivateRegistry: pulumi.String(cfg.PrivateRegistry),
 			Replicas:        pulumi.Int(cfg.Replicas),
 			Swagger:         cfg.Swagger,
-			EtcdReplicas:    nil,
-			JanitorCron:     nil,
-			Otel:            nil,
+			PVCAccessModes: pulumi.ToStringArray([]string{
+				cfg.PVCAccessMode,
+			}),
+			Expose:       cfg.Expose,
+			EtcdReplicas: nil,
+			JanitorCron:  nil,
+			Otel:         nil,
 		}
 		if cfg.Etcd != nil {
 			args.EtcdReplicas = pulumi.IntPtr(cfg.Etcd.Replicas)
@@ -53,6 +57,7 @@ func main() {
 		}
 
 		ctx.Export("endpoint", cm.Endpoint)
+		ctx.Export("exposed_port", cm.ExposedPort)
 
 		return nil
 	})
@@ -67,6 +72,8 @@ type (
 		Replicas        int            `json:"replicas"`
 		Janitor         *JanitorConfig `json:"janitor"`
 		Swagger         bool           `json:"swagger"`
+		PVCAccessMode   string         `json:"pvc-access-mode"`
+		Expose          bool           `json:"expose"`
 		Otel            *OtelConfig    `json:"otel"`
 	}
 
@@ -92,6 +99,8 @@ func loadConfig(ctx *pulumi.Context) *Config {
 		PrivateRegistry: cfg.Get("private-registry"),
 		Replicas:        cfg.GetInt("replicas"),
 		Swagger:         cfg.GetBool("swagger"),
+		PVCAccessMode:   cfg.Get("pvc-access-mode"),
+		Expose:          cfg.GetBool("expose"),
 	}
 
 	var etcdC EtcdConfig

deploy/services/chall-manager.go

@@ -1,10 +1,14 @@
 package services
 
 import (
+	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/ctfer-io/chall-manager/deploy/common"
 	"github.com/ctfer-io/chall-manager/deploy/services/parts"
+	corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
+	v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
 	netwv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
@@ -19,13 +23,17 @@ type (
 		cm   *parts.ChallManager
 		cmj  *parts.ChallManagerJanitor
 
+		// Exposure
+		svc *corev1.Service
+
 		// Interface & ports network policies
 		cmToEtcd *netwv1.NetworkPolicy
 		cmjToCm  *netwv1.NetworkPolicy
 
 		// Outputs
 
-		Endpoint pulumi.StringOutput
+		Endpoint    pulumi.StringOutput
+		ExposedPort pulumi.IntPtrOutput
 	}
 
 	// ChallManagerArgs contains all the parametrization of a Chall-Manager
@@ -47,7 +55,11 @@ type (
 		JanitorCron  pulumi.StringPtrInput
 		janitorCron  pulumi.StringOutput
 
-		Swagger bool
+		// PVCAccessModes defines the access modes supported by the PVC.
+		PVCAccessModes pulumi.StringArrayInput
+		pvcAccessModes pulumi.StringArrayOutput
+
+		Swagger, Expose bool
 
 		Otel *common.OtelArgs
 	}
@@ -62,19 +74,41 @@ const (
 //
 // It is not made to be exposed to outer world (outside of the cluster).
 func NewChallManager(ctx *pulumi.Context, name string, args *ChallManagerArgs, opts ...pulumi.ResourceOption) (*ChallManager, error) {
+	cm := &ChallManager{}
+	args, err := cm.validate(args)
+	if err != nil {
+		return nil, err
+	}
+	if err := ctx.RegisterComponentResource("ctfer-io:chall-manager", name, cm, opts...); err != nil {
+		return nil, err
+	}
+	opts = append(opts, pulumi.Parent(cm))
+	if err := cm.provision(ctx, args, opts...); err != nil {
+		return nil, err
+	}
+	if err := cm.outputs(ctx); err != nil {
+		return nil, err
+	}
+	return cm, nil
+}
+
+func (cm *ChallManager) validate(args *ChallManagerArgs) (*ChallManagerArgs, error) {
 	if args == nil {
 		args = &ChallManagerArgs{}
 	}
+
 	if args.Tag == nil || args.Tag == pulumi.String("") {
 		args.tag = pulumi.String("dev").ToStringOutput()
 	} else {
 		args.tag = args.Tag.ToStringPtrOutput().Elem()
 	}
+
 	if args.JanitorCron == nil || args.JanitorCron == pulumi.String("") {
 		args.janitorCron = pulumi.String(defaultCron).ToStringOutput()
 	} else {
 		args.janitorCron = args.JanitorCron.ToStringPtrOutput().Elem()
 	}
+
 	if args.PrivateRegistry == nil {
 		args.privateRegistry = pulumi.String("").ToStringOutput()
 	} else {
@@ -92,23 +126,22 @@ func NewChallManager(ctx *pulumi.Context, name string, args *ChallManagerArgs, o
 			return str
 		}).(pulumi.StringOutput)
 	}
+
 	if args.Replicas == nil {
 		args.replicas = pulumi.Int(1).ToIntOutput()
 	} else {
 		args.replicas = args.Replicas.ToIntPtrOutput().Elem()
 	}
 
-	cm := &ChallManager{}
-	if err := ctx.RegisterComponentResource("ctfer-io:chall-manager", name, cm, opts...); err != nil {
-		return nil, err
-	}
-	opts = append(opts, pulumi.Parent(cm))
-	if err := cm.provision(ctx, args, opts...); err != nil {
-		return nil, err
+	if args.PVCAccessModes == nil {
+		args.pvcAccessModes = pulumi.ToStringArray([]string{
+			"ReadWriteMany",
+		}).ToStringArrayOutput()
+	} else {
+		args.pvcAccessModes = args.PVCAccessModes.ToStringArrayOutput()
 	}
-	cm.outputs()
 
-	return cm, nil
+	return args, nil
 }
 
 func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, opts ...pulumi.ResourceOption) (err error) {
@@ -149,9 +182,10 @@ func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, o
 			}
 			return 1 // default replicas to 1
 		}).(pulumi.IntOutput),
-		Etcd:    nil,
-		Swagger: args.Swagger,
-		Otel:    nil,
+		Etcd:           nil,
+		Swagger:        args.Swagger,
+		PVCAccessModes: args.pvcAccessModes,
+		Otel:           nil,
 	}
 	if args.EtcdReplicas != nil {
 		cmArgs.Etcd = &parts.ChallManagerEtcdArgs{
@@ -172,6 +206,34 @@ func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, o
 		return
 	}
 
+	if args.Expose {
+		cm.svc, err = corev1.NewService(ctx, "cm-exposed", &corev1.ServiceArgs{
+			Metadata: v1.ObjectMetaArgs{
+				Labels:    cm.cm.PodLabels,
+				Namespace: args.Namespace,
+			},
+			Spec: corev1.ServiceSpecArgs{
+				Type:     pulumi.String("NodePort"),
+				Selector: cm.cm.PodLabels,
+				Ports: corev1.ServicePortArray{
+					corev1.ServicePortArgs{
+						Port: cm.cm.Endpoint.ApplyT(func(edp string) int {
+							// On bootstrap there is no valid URL, but port is assigned
+							pts := strings.Split(edp, ":")
+							p := pts[len(pts)-1]
+							port, _ := strconv.Atoi(p)
+							fmt.Printf("port: %v\n", port)
+							return port
+						}).(pulumi.IntOutput),
+					},
+				},
+			},
+		}, opts...)
+		if err != nil {
+			return
+		}
+	}
+
 	// Deploy janitor
 	var cmjOtel *common.OtelArgs
 	if args.Otel != nil {
@@ -294,6 +356,16 @@ func (cm *ChallManager) provision(ctx *pulumi.Context, args *ChallManagerArgs, o
 	return
 }
 
-func (cm *ChallManager) outputs() {
+func (cm *ChallManager) outputs(ctx *pulumi.Context) error {
 	cm.Endpoint = cm.cm.Endpoint
+	if cm.svc != nil {
+		cm.ExposedPort = cm.svc.Spec.ApplyT(func(spec corev1.ServiceSpec) *int {
+			return spec.Ports[0].NodePort
+		}).(pulumi.IntPtrOutput)
+	}
+
+	return ctx.RegisterResourceOutputs(cm, pulumi.Map{
+		"endpoint":     cm.Endpoint,
+		"exposed_port": cm.ExposedPort,
+	})
 }