Ref #4513: Allow to remote debug the Operator

.gitignore

@@ -176,3 +176,7 @@ config/**/*.gen.json
 # Fabric8 CRDs
 java/target
 pkg/resources/resources.go
+
+# MAC OS files
+.DS_Store
+

build/Dockerfile

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM eclipse-temurin:17-jdk
+FROM eclipse-temurin:17-jdk as base
 
 ARG MAVEN_DEFAULT_VERSION="3.8.6"
 ARG MAVEN_HOME="/usr/share/maven"
@@ -56,3 +56,11 @@ RUN mkdir -p /etc/maven/m2 \
 USER 1000
 
 ADD build/_output/bin/kamel /usr/local/bin/kamel
+
+FROM golang:1.18 as go
+
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+FROM base as debug
+
+COPY --from=go /go/bin/dlv /usr/local/bin/dlv

docs/modules/ROOT/pages/contributing/remote-debugging.adoc

@@ -0,0 +1,90 @@
+[[remote-debugging]]
+= Remote Debugging Camel-K
+
+In this article, we describe the steps needed to be able to remotely debug the Camel-K operator directly from the K8s cluster.
+By doing so, you are sure that the operator is executed in the same context as your target environment, which is not the case
+if the operator is launched on the local machine.
+
+[[publish-image]]
+== Publish the image
+
+The first thing to do is to build a specific docker image of the Camel-K operator for the debug mode, indeed the `kamel` program
+will then be built without compiler optimizations, and inlining but also the docker image will launch the operator through
+https://github.com/go-delve/delve[`delve`] to be able to remote debug it.
+
+[source,shell]
+----
+DEBUG_MODE=true make images
+----
+
+Once done, a docker image of type `docker.io/apache/camel-k-debug:2.0.0-SNAPSHOT` has been pushed into your local docker image registry.
+
+If you are using Minikube, before executing the previous command make sure to set up properly the environment
+variables in your terminal by executing the command `eval $(minikube -p minikube docker-env)`, in that case the image is
+directly pushed into the registry of Minikube, so you can skip the end of the section.
+
+For other clusters like for example `kind` where the registry is accessible locally from `localhost:5001`, simply tag the
+image to match with the new host and port with the next command:
+
+[source,shell]
+----
+docker tag docker.io/apache/camel-k-debug:2.0.0-SNAPSHOT localhost:5001/apache/camel-k-debug:2.0.0-SNAPSHOT
+----
+
+Then push the image to the target registry with the next command:
+[source,shell]
+----
+docker push localhost:5001/apache/camel-k-debug:2.0.0-SNAPSHOT
+----
+
+To ensure that the image has been pushed with success, let's query the registry https://docs.docker.com/registry/spec/api/#listing-repositories[using the API]
+[source,shell]
+----
+curl http://localhost:5001/v2/_catalog
+{"repositories":["apache/camel-k-debug"]}
+----
+
+[[install-operator]]
+== Install the operator
+
+Since the docker image is ready to be used, we can now install the operator with the debugging flags to make sure that
+the operator will be launched properly with the debug port open on its pod.
+
+First, let's create a namespace in which the operator will be installed, here the namespace is `test`.
+[source,shell]
+----
+kubectl create ns test
+namespace/test created
+----
+
+Then, install the operator with the image that we built before
+[source,shell]
+----
+./kamel install --olm=false --operator-image apache/camel-k-debug:2.0.0-SNAPSHOT --debugging -n test
+----
+It will install the operator using `apache/camel-k-debug:2.0.0-SNAPSHOT` as docker image and launch it in debug mode.
+
+[[port-forward]]
+== Open the port on the pod
+
+The operator is now waiting for a remote connection, but to make it possible, we need to make the debugging port
+accessible from outside the cluster thanks to the following `port-forward` command:
+
+[source,shell]
+----
+kubectl port-forward -n test $(kubectl get po -l app=camel-k -oname -n test) 4040:4040
+Forwarding from 127.0.0.1:4040 -> 4040
+Forwarding from [::1]:4040 -> 4040
+----
+This command port forwards the port `4040` of the pod to the local port `4040` which makes it accessible from `localhost`.
+Where `4040` is the default port of delve set on the pod, but it can be changed when installing the operator with the flag
+`--debugging-port=4040`
+
+[[configure-ide]]
+== Configure your IDE
+
+At this stage, you simply need to configure your favorite IDE to remote debug the operator using `localhost` as host and
+`4040` as port:
+
+* https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html#step-3-create-the-remote-run-debug-configuration-on-the-client-computer[Configure IDEA]
+* https://go.googlesource.com/vscode-go/+/HEAD/docs/debugging.md#remote-debugging[Configure VSCode]

pkg/cmd/install.go

@@ -140,6 +140,11 @@ func newCmdInstall(rootCmdOptions *RootCmdOptions) (*cobra.Command, *installCmdO
 	cmd.Flags().Bool("monitoring", false, "To enable or disable the operator monitoring")
 	cmd.Flags().Int("monitoring-port", 8080, "The port of the metrics endpoint")
 
+	// debugging
+	cmd.Flags().Bool("debugging", false, "To enable or disable the operator debugging")
+	cmd.Flags().Int("debugging-port", 4040, "The port of the debugger")
+	cmd.Flags().String("debugging-path", "/usr/local/bin/kamel", "The path to the kamel executable file")
+
 	// Operator settings
 	cmd.Flags().StringArray("toleration", nil, "Add a Toleration to the operator Pod")
 	cmd.Flags().StringArray("node-selector", nil, "Add a NodeSelector to the operator Pod")
@@ -196,6 +201,9 @@ type installCmdOptions struct {
 	MaxRunningBuilds            int32    `mapstructure:"max-running-pipelines"`
 	Monitoring                  bool     `mapstructure:"monitoring"`
 	MonitoringPort              int32    `mapstructure:"monitoring-port"`
+	Debugging                   bool     `mapstructure:"debugging"`
+	DebuggingPort               int32    `mapstructure:"debugging-port"`
+	DebuggingPath               string   `mapstructure:"debugging-path"`
 	TraitProfile                string   `mapstructure:"trait-profile"`
 	Tolerations                 []string `mapstructure:"tolerations"`
 	NodeSelectors               []string `mapstructure:"node-selectors"`
@@ -427,6 +435,11 @@ func (o *installCmdOptions) setupOperator(
 			Enabled: o.Monitoring,
 			Port:    o.MonitoringPort,
 		},
+		Debugging: install.OperatorDebuggingConfiguration{
+			Enabled: o.Debugging,
+			Port:    o.DebuggingPort,
+			Path:    o.DebuggingPath,
+		},
 		Tolerations:           o.Tolerations,
 		NodeSelectors:         o.NodeSelectors,
 		ResourcesRequirements: o.ResourcesRequirements,

pkg/install/operator.go

@@ -57,6 +57,7 @@ type OperatorConfiguration struct {
 	ClusterType           string
 	Health                OperatorHealthConfiguration
 	Monitoring            OperatorMonitoringConfiguration
+	Debugging             OperatorDebuggingConfiguration
 	Tolerations           []string
 	NodeSelectors         []string
 	ResourcesRequirements []string
@@ -68,6 +69,12 @@ type OperatorHealthConfiguration struct {
 	Port int32
 }
 
+type OperatorDebuggingConfiguration struct {
+	Enabled bool
+	Port    int32
+	Path    string
+}
+
 type OperatorMonitoringConfiguration struct {
 	Enabled bool
 	Port    int32
@@ -210,6 +217,22 @@ func OperatorOrCollect(ctx context.Context, cmd *cobra.Command, c client.Client,
 				FSGroup: &ugfid,
 			}
 		}
+		if cfg.Debugging.Enabled {
+			if d, ok := o.(*appsv1.Deployment); ok {
+				if d.Labels["camel.apache.org/component"] == "operator" {
+					d.Spec.Template.Spec.Containers[0].Command = []string{"dlv",
+						fmt.Sprintf("--listen=:%d", cfg.Debugging.Port), "--headless=true", "--api-version=2",
+						"exec", cfg.Debugging.Path, "--", "operator", "--leader-election=false"}
+					d.Spec.Template.Spec.Containers[0].Ports = append(d.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{
+						Name:          "delve",
+						ContainerPort: cfg.Debugging.Port,
+					})
+					// In debug mode, the Liveness probe must be removed otherwise K8s will consider the pod as dead
+					// while debugging
+					d.Spec.Template.Spec.Containers[0].LivenessProbe = nil
+				}
+			}
+		}
 
 		if cfg.Global {
 			if d, ok := o.(*appsv1.Deployment); ok {

script/Makefile

@@ -54,7 +54,7 @@ RELEASE_GIT_REMOTE := origin
 GIT_COMMIT := $(shell if [ -d .git ]; then git rev-list -1 HEAD; else echo "$(CUSTOM_VERSION)"; fi)
 LINT_GOGC := 10
 LINT_DEADLINE := 10m
-
+DEBUG_MODE ?= false
 
 # olm bundle vars
 MANAGER := config/manager
@@ -121,6 +121,10 @@ endif
 
 GOFLAGS = -ldflags "$(GOLDFLAGS)" -trimpath
 
+ifeq ($(DEBUG_MODE),true)
+GOFLAGS += -gcflags="all=-N -l"
+endif
+
 define LICENSE_HEADER
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
@@ -432,18 +436,24 @@ else
 endif
 	cp build/_output/bin/kamel-$(IMAGE_ARCH) build/_output/bin/kamel
 
+TARGET_STAGE := base
+ifeq ($(DEBUG_MODE),true)
+	TARGET_STAGE := debug
+	CUSTOM_IMAGE := $(CUSTOM_IMAGE)-debug
+endif
+
+DOCKER_TAG := $(CUSTOM_IMAGE):$(CUSTOM_VERSION)
+ifneq ($(IMAGE_ARCH), amd64)
+	DOCKER_TAG := $(DOCKER_TAG)-$(IMAGE_ARCH)
+endif
 
 images: build kamel-overlay maven-overlay bundle-kamelets
 ifneq (,$(findstring SNAPSHOT,$(RUNTIME_VERSION)))
 	./script/package_maven_artifacts.sh -s "$(STAGING_RUNTIME_REPO)" -d "$(CAMEL_K_RUNTIME_DIR)" $(RUNTIME_VERSION)
 endif
 	@echo "####### Building Camel K operator arch $(IMAGE_ARCH) container image..."
 	mkdir -p build/_maven_output
-ifeq ($(IMAGE_ARCH), amd64)
-	docker build --platform=linux/$(IMAGE_ARCH) -t $(CUSTOM_IMAGE):$(CUSTOM_VERSION) -f build/Dockerfile .
-else
-	docker build --platform=linux/$(IMAGE_ARCH) -t $(CUSTOM_IMAGE):$(CUSTOM_VERSION)-$(IMAGE_ARCH) -f build/Dockerfile .
-endif
+	docker build --target $(TARGET_STAGE) --platform=linux/$(IMAGE_ARCH) -t $(DOCKER_TAG) -f build/Dockerfile .
 
 # Mainly used for internal CI purposes
 image-push: