Merge pull request #837 from traPtitech/buildpack-helper

Introduce buildpack-helper component

.github/workflows/ci-protoc.yaml

@@ -21,7 +21,7 @@ jobs:
           go-version-file: ./go.mod
       - uses: arduino/setup-protoc@v2
         with:
-          version: "24.4"
+          version: "25.1"
       - name: Install proto tools
         run: make init-protoc-tools
       - name: Generate code

.local-dev/config/ns.yaml

@@ -27,13 +27,11 @@ components:
     buildkit:
       address: tcp://buildkitd:1234
     buildpack:
-      backend: docker
-      docker:
-        containerName: neoshowcase-buildpack-1
-        remoteDir: /workspace
-        user: cnb
-        group: cnb
-        platformAPI: "0.11"
+      helper:
+        address: http://buildpack:1235
+        listenPort: 1235
+      remoteDir: /workspace
+      platformAPI: "0.11"
     controller:
       url: http://ns-controller:10000
     priority: 0

.local-dev/ext-builder/Makefile

@@ -0,0 +1,48 @@
+.DEFAULT_GOAL := help
+CLUSTER_NAME = ext-builder
+
+.PHONY: help
+help: ## Display this help screen
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: k3d-up
+k3d-up: ## Spin up k3d cluster
+	k3d cluster create $(CLUSTER_NAME) --image rancher/k3s:latest --no-lb --k3s-arg "--disable=traefik,servicelb,metrics-server"
+
+.PHONY: ensure-context
+ensure-context:
+	kubectl config use-context k3d-$(CLUSTER_NAME)
+
+.PHONY: import
+import: ensure-context ## Import ns-builder image into the cluster
+	sh -c 'cd ../../ && make build'
+	k3d image import --cluster $(CLUSTER_NAME) ghcr.io/traptitech/ns-builder:main
+
+.PHONY: apply
+apply: ensure-context ## Apply manifests to k3d cluster
+	kustomize build . | kubectl apply -f -
+
+REPLICAS?=1
+.PHONY: scale
+scale: ensure-context ## Scale the number of builder instances
+	kubectl scale deployment/ns-builder --replicas=$(REPLICAS)
+
+.PHONY: events
+events: ensure-context ## Tail cluster events
+	kubectl get events --watch
+
+.PHONY: logs
+logs: ensure-context ## Tail builder logs
+	kubectl logs --prefix --timestamps --all-containers --max-log-requests=100 -l app=ns-builder -f
+
+.PHONY: k3d-down
+k3d-down: ## Spin down k3d cluster
+	k3d cluster delete $(CLUSTER_NAME)
+
+# --- all in one commands ---
+
+.PHONY: up
+up: k3d-up import apply events
+
+.PHONY: down
+down: k3d-down

.local-dev/ext-builder/README.md

@@ -4,10 +4,43 @@ Join external builder instance from outside the production (cluster) deployment
 
 ## Usage
 
-1. Set config `./config.yaml` accordingly
-   - Fetch configuration from production if necessary
-2. Set token (`NS_COMPONENTS_CONTROLLER_TOKEN`) in `.env` file
-3. Prepare local forward if controller port is unreachable from the internet
-   - `ssh -L 0.0.0.0:10000:10.43.193.98:10000 c1-203`
-4. `docker compose pull`
-5. `docker compose up -d`
+tl;dr:
+Do "First time setup", and run `make up` / `make down`
+
+### First time setup
+
+1. Set up config files
+   - `./config/controller-url.txt` for controller URL
+   - `./config/controller-token.txt` for controller token
+   - `./config/known_hosts` for known hosts configuration
+     - `cp ~/.ssh/known_hosts ./config` should be enough
+2. Prepare local forward if controller port is unreachable from the internet
+   - e.g. `ssh -L 0.0.0.0:10000:10.43.193.98:10000 c1-203`
+
+### Importing images
+
+To import ns images to the k3d cluster, `make import`
+
+### Spin up
+
+1. `make k3d-up`
+2. `make apply`
+
+### Workaround for local registry
+
+ref: https://zenn.dev/toshikish/articles/7f555dbf1b4b7d
+
+Edit and add `rewrite name registry.local host.k3d.internal` inside the `.:53 {}` section:
+`kubectl edit cm -n kube-system coredns`
+
+Restart CoreDNS afterwards:
+`kubectl restart deployment/coredns -n kube-system`
+
+### Managing
+
+- To scale the number of builder instances, `make scale REPLICAS=3`
+- To tail builder logs, `make logs`
+
+### Spin down
+
+1. `make k3d-down`

.local-dev/ext-builder/builder-deployment.yaml

@@ -0,0 +1,108 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ns-builder
+
+spec:
+  replicas: 1
+  revisionHistoryLimit: 0
+  selector:
+    matchLabels:
+      app: ns-builder
+  template:
+    metadata:
+      labels:
+        app: ns-builder
+    spec:
+      volumes:
+        - name: ns-bin
+          emptyDir: {}
+        - name: socket
+          emptyDir: {}
+        - name: config
+          configMap:
+            name: ns-config
+        - name: known-hosts
+          configMap:
+            name: known-hosts
+
+      initContainers:
+        - name: buildpack-install
+          image: ghcr.io/traptitech/ns-builder:main
+          imagePullPolicy: Never
+          command:
+            - cp
+            - /app/ns
+            - /ns-bin
+          volumeMounts:
+            - mountPath: /ns-bin
+              name: ns-bin
+
+      containers:
+        - name: buildpack
+          image: paketobuildpacks/builder-jammy-full:latest
+          imagePullPolicy: Always
+          command:
+            - /ns-bin/ns
+            - buildpack-helper
+          args:
+            - --loglevel=debug
+            - --config=/opt/config/ns.yaml
+          volumeMounts:
+            - mountPath: /ns-bin
+              name: ns-bin
+            - mountPath: /opt/config/ns.yaml
+              name: config
+              subPath: ns.yaml
+
+        - name: buildkitd
+          image: moby/buildkit:latest
+          imagePullPolicy: Always
+          args:
+            - --addr
+            - unix:///run/buildkit/buildkitd.sock
+          readinessProbe:
+            exec:
+              command:
+                - buildctl
+                - debug
+                - workers
+            initialDelaySeconds: 5
+            periodSeconds: 30
+          livenessProbe:
+            exec:
+              command:
+                - buildctl
+                - debug
+                - workers
+            initialDelaySeconds: 5
+            periodSeconds: 30
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /run/buildkit
+              name: socket
+            - mountPath: /etc/buildkit/buildkitd.toml
+              name: config
+              subPath: buildkitd.toml
+
+        - name: builder
+          image: ghcr.io/traptitech/ns-builder:main
+          imagePullPolicy: Never
+          args:
+            - --loglevel=debug
+            - --config=/opt/config/ns.yaml
+          envFrom:
+            - configMapRef:
+                name: custom-config
+            - secretRef:
+                name: token
+          volumeMounts:
+            - mountPath: /run/buildkit
+              name: socket
+            - mountPath: /opt/config/ns.yaml
+              name: config
+              subPath: ns.yaml
+            - mountPath: /root/.ssh/known_hosts
+              name: known-hosts
+              subPath: known_hosts

.local-dev/ext-builder/config/.gitignore

@@ -0,0 +1,3 @@
+controller-token.txt
+controller-url.txt
+known_hosts

.local-dev/ext-builder/config/buildkitd.toml

@@ -0,0 +1,11 @@
+[registry."registry.local"]
+http = true
+
+[worker.oci]
+enabled = true
+gc = true
+gckeepstorage = 9000
+
+[[worker.oci.gcpolicy]]
+all = true
+keepBytes = 1024000000

.local-dev/ext-builder/config/ns.yaml

@@ -0,0 +1,16 @@
+components:
+  builder:
+    buildkit:
+      address: unix:///run/buildkit/buildkitd.sock
+    buildpack:
+      helper:
+        address: http://localhost:1235
+        listenPort: 1235
+      remoteDir: /workspace
+      platformAPI: "0.11"
+    # controller:
+      # url: <from env> # e.g. http://host.k3d.internal:10000
+    priority: 10
+
+  # controller:
+    # token: <from env>

.local-dev/ext-builder/kustomization.yaml

@@ -0,0 +1,19 @@
+resources:
+  - builder-deployment.yaml
+
+configMapGenerator:
+  - name: known-hosts
+    files:
+      - ./config/known_hosts
+  - name: ns-config
+    files:
+      - ./config/ns.yaml
+      - ./config/buildkitd.toml
+  - name: custom-config
+    files:
+      - NS_COMPONENTS_BUILDER_CONTROLLER_URL=./config/controller-url.txt
+
+secretGenerator:
+  - name: token
+    files:
+      - NS_COMPONENTS_CONTROLLER_TOKEN=./config/controller-token.txt