diff --git a/pkg/debug/debug_component.go b/pkg/debug/debug_component.go new file mode 100644 index 0000000..4a2ca26 --- /dev/null +++ b/pkg/debug/debug_component.go @@ -0,0 +1,286 @@ +package debug + +import ( + "fmt" + "time" + "strings" + + "bunnyshell.com/dev/pkg/k8s" + "bunnyshell.com/dev/pkg/remote/container" + "bunnyshell.com/dev/pkg/util" + + "github.com/briandowns/spinner" + appsV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" +) + +// +enum +type ResourceType string + +const ( + Deployment ResourceType = "deployment" + StatefulSet ResourceType = "statefulset" + DaemonSet ResourceType = "daemonset" +) + +type DebugComponent struct { + ContainerName string + ContainerConfig container.Config + + AutoSelectSingleResource bool + + spinner *spinner.Spinner + + kubernetesClient *k8s.KubernetesClient + + namespace *coreV1.Namespace + resourceType ResourceType + deployment *appsV1.Deployment + statefulSet *appsV1.StatefulSet + daemonSet *appsV1.DaemonSet + container *coreV1.Container + + isInitContainer bool + + shouldPrepareResource bool + + stopChannel chan bool + + startedAt int64 + waitTimeout int64 +} + +func NewDebugComponent() *DebugComponent { + return &DebugComponent{ + ContainerConfig: *container.NewConfig(), + + AutoSelectSingleResource: true, + + shouldPrepareResource: true, + + stopChannel: make(chan bool), + spinner: util.MakeSpinner(" Debug"), + startedAt: time.Now().Unix(), + waitTimeout: 120, + } +} + +func (d *DebugComponent) WithKubernetesClient(kubeConfigPath string) *DebugComponent { + kubernetesClient, err := k8s.NewKubernetesClient(kubeConfigPath) + if err != nil { + panic(err) + } + + d.kubernetesClient = kubernetesClient + + return d +} + +func (d *DebugComponent) WithNamespace(namespace *coreV1.Namespace) *DebugComponent { + d.namespace = namespace + return d +} + +func (d *DebugComponent) WithNamespaceName(namespaceName string) *DebugComponent { + namespace, err := d.kubernetesClient.GetNamespace(namespaceName) + if err != nil { + panic(err) + } + + return d.WithNamespace(namespace) +} + +func (d *DebugComponent) WithNamespaceFromKubeConfig() *DebugComponent { + namespace, err := d.kubernetesClient.GetKubeConfigNamespace() + if err != nil { + panic(err) + } + + return d.WithNamespaceName(namespace) +} + +func (d *DebugComponent) WithResourceType(resourceType ResourceType) *DebugComponent { + d.resourceType = resourceType + return d +} + +func (d *DebugComponent) WithDeployment(deployment *appsV1.Deployment) *DebugComponent { + if d.namespace == nil { + panic(ErrNoNamespaceSelected) + } + + if d.namespace.GetName() != deployment.GetNamespace() { + panic(fmt.Errorf( + "the deployment's namespace(\"%s\") doesn't match the selected namespace \"%s\"", + deployment.GetNamespace(), + d.namespace.GetName(), + )) + } + + d.WithResourceType(Deployment) + d.deployment = deployment + return d +} + +func (d *DebugComponent) WithDeploymentName(deploymentName string) *DebugComponent { + deployment, err := d.kubernetesClient.GetDeployment(d.namespace.GetName(), deploymentName) + if err != nil { + panic(err) + } + + return d.WithDeployment(deployment) +} + +func (d *DebugComponent) WithStatefulSet(statefulSet *appsV1.StatefulSet) *DebugComponent { + if d.namespace == nil { + panic(ErrNoNamespaceSelected) + } + + if d.namespace.GetName() != statefulSet.GetNamespace() { + panic(fmt.Errorf( + "the statefulset's namespace(\"%s\") doesn't match the selected namespace \"%s\"", + statefulSet.GetNamespace(), + d.namespace.GetName(), + )) + } + + d.WithResourceType(StatefulSet) + d.statefulSet = statefulSet + return d +} + +func (d *DebugComponent) WithStatefulSetName(name string) *DebugComponent { + statefulSet, err := d.kubernetesClient.GetStatefulSet(d.namespace.GetName(), name) + if err != nil { + panic(err) + } + + return d.WithStatefulSet(statefulSet) +} + +func (d *DebugComponent) WithDaemonSet(daemonSet *appsV1.DaemonSet) *DebugComponent { + if d.namespace == nil { + panic(ErrNoNamespaceSelected) + } + + if d.namespace.GetName() != daemonSet.GetNamespace() { + panic(fmt.Errorf( + "the daemonset's namespace(\"%s\") doesn't match the selected namespace \"%s\"", + daemonSet.GetNamespace(), + d.namespace.GetName(), + )) + } + + d.WithResourceType(DaemonSet) + d.daemonSet = daemonSet + return d +} + +func (d *DebugComponent) WithDaemonSetName(name string) *DebugComponent { + daemonSet, err := d.kubernetesClient.GetDaemonSet(d.namespace.GetName(), name) + if err != nil { + panic(err) + } + + return d.WithDaemonSet(daemonSet) +} + +func (d *DebugComponent) WithContainer(container *coreV1.Container) *DebugComponent { + if d.resourceType == "" { + panic(fmt.Errorf("please select a resource first")) + } + + d.container = container + d.isInitContainer = false + return d +} + +func (d *DebugComponent) WithInitContainer(container *coreV1.Container) *DebugComponent { + if d.resourceType == "" { + panic(fmt.Errorf("please select a resource first")) + } + + d.container = container + d.isInitContainer = true + return d +} + +func (d *DebugComponent) WithContainerName(containerName string) *DebugComponent { + container, err := d.getResourceContainer(containerName) + if err != nil { + if !strings.HasSuffix(err.Error(), " not found") { + panic(err) + } + + initContainer, err := d.getResourceInitContainer(containerName) + if err != nil { + panic(err) + } + + return d.WithInitContainer(initContainer) + } + + return d.WithContainer(container) +} + +func (d *DebugComponent) getResource() (Resource, error) { + switch d.resourceType { + case Deployment: + return d.deployment, nil + case StatefulSet: + return d.statefulSet, nil + case DaemonSet: + return d.daemonSet, nil + default: + return nil, d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) getResourceType(resource Resource) (ResourceType, error) { + switch resource.(type) { + case *appsV1.Deployment: + return Deployment, nil + case *appsV1.StatefulSet: + return StatefulSet, nil + case *appsV1.DaemonSet: + return DaemonSet, nil + default: + return "", ErrInvalidResourceType + } +} + +func (d *DebugComponent) WithResource(resource Resource) *DebugComponent { + resourceType, err := d.getResourceType(resource) + if err != nil { + panic(err) + } + + switch resourceType { + case Deployment: + d.WithDeployment(resource.(*appsV1.Deployment)) + case StatefulSet: + d.WithStatefulSet(resource.(*appsV1.StatefulSet)) + case DaemonSet: + d.WithDaemonSet(resource.(*appsV1.DaemonSet)) + default: + panic(fmt.Errorf( + "could not determine the resource Kind for resource type \"%s\"", + resourceType, + )) + } + + return d +} + +func (d *DebugComponent) WithWaitTimeout(waitTimeout int64) *DebugComponent { + d.waitTimeout = waitTimeout + return d +} + +func (d *DebugComponent) GetSelectedContainerName() (string, error) { + if d.container == nil { + panic(fmt.Errorf("please select a container first")) + } + + return d.container.Name, nil +} diff --git a/pkg/debug/interactive.go b/pkg/debug/interactive.go new file mode 100644 index 0000000..9e3a292 --- /dev/null +++ b/pkg/debug/interactive.go @@ -0,0 +1,369 @@ +package debug + + +import ( + "fmt" + "strings" + + coreV1 "k8s.io/api/core/v1" + + "bunnyshell.com/dev/pkg/util" +) + +var ( + ErrNoNamespaces = fmt.Errorf("no namespaces available") + + ErrNoResources = fmt.Errorf("no resources available") + + ErrNoDeployments = fmt.Errorf("no deployments available") + ErrNoStatefulSets = fmt.Errorf("no statefulsets available") + ErrNoDaemonSets = fmt.Errorf("no daemonset available") + + ErrNoNamespaceSelected = fmt.Errorf("no namespace selected") + ErrNoResourceSelected = fmt.Errorf("no resource selected") + + ErrContainerNotFound = fmt.Errorf("container not found") +) + +func (d *DebugComponent) SelectNamespace() error { + namespaces, err := d.kubernetesClient.ListNamespaces() + if err != nil { + return err + } + + if len(namespaces.Items) == 0 { + return ErrNoNamespaces + } + + if len(namespaces.Items) == 1 && d.AutoSelectSingleResource { + d.namespace = namespaces.Items[0].DeepCopy() + return nil + } + + items := []string{} + for _, item := range namespaces.Items { + items = append(items, item.GetName()) + } + + namespace, err := util.Select("Select namespace", items) + if err != nil { + return err + } + + for _, item := range namespaces.Items { + if item.GetName() != namespace { + continue + } + + d.WithNamespace(item.DeepCopy()) + return nil + } + + return nil +} + +func (d *DebugComponent) SelectResource() error { + availableResources, err := d.getAvailableResourceFromNamespace(d.namespace.GetName()) + if err != nil { + return err + } + + if len(availableResources) == 0 { + return ErrNoResources + } + + if len(availableResources) == 1 && d.AutoSelectSingleResource { + d.WithResource(availableResources[0]) + + return nil + } + + selectItems := []string{} + resourcesItemsMap := map[string]Resource{} + for _, resourceItem := range availableResources { + resourceType, err := d.getResourceType(resourceItem) + if err != nil { + return err + } + + resourceItemLabel := fmt.Sprintf("%s / %s", resourceType, resourceItem.GetName()) + selectItems = append(selectItems, resourceItemLabel) + resourcesItemsMap[resourceItemLabel] = resourceItem + } + + selectedResourceItemLabel, err := util.Select("Select resource", selectItems) + if err != nil { + return err + } + + d.WithResource(resourcesItemsMap[selectedResourceItemLabel]) + return nil +} + +func (d *DebugComponent) getAvailableResourceFromNamespace(namespace string) ([]Resource, error) { + availableResources := []Resource{} + + deployments, err := d.kubernetesClient.ListDeployments(namespace) + if err != nil { + return nil, err + } + for _, deploymentItem := range deployments.Items { + item := deploymentItem + availableResources = append(availableResources, &item) + } + + statefulsets, err := d.kubernetesClient.ListStatefulSets(namespace) + if err != nil { + return nil, err + } + for _, statefulsetItem := range statefulsets.Items { + item := statefulsetItem + availableResources = append(availableResources, &item) + } + + daemonsets, err := d.kubernetesClient.ListDaemonSets(namespace) + if err != nil { + return nil, err + } + for _, daemonsetItem := range daemonsets.Items { + item := daemonsetItem + availableResources = append(availableResources, &item) + } + + return availableResources, nil +} + +func (d *DebugComponent) SelectDeployment() error { + if d.namespace == nil { + return ErrNoNamespaceSelected + } + + deployments, err := d.kubernetesClient.ListDeployments(d.namespace.GetName()) + if err != nil { + return err + } + + if len(deployments.Items) == 0 { + return ErrNoDeployments + } + + if len(deployments.Items) == 1 && d.AutoSelectSingleResource { + d.WithDeployment(deployments.Items[0].DeepCopy()) + + return nil + } + + items := []string{} + for _, item := range deployments.Items { + items = append(items, item.GetName()) + } + + deployment, err := util.Select("Select deployment", items) + if err != nil { + return err + } + + for _, item := range deployments.Items { + if item.GetName() != deployment { + continue + } + + d.WithDeployment(item.DeepCopy()) + return nil + } + + return nil +} + +func (d *DebugComponent) SelectStatefulSet() error { + if d.namespace == nil { + return ErrNoNamespaceSelected + } + + statefulSets, err := d.kubernetesClient.ListStatefulSets(d.namespace.GetName()) + if err != nil { + return err + } + + if len(statefulSets.Items) == 0 { + return ErrNoStatefulSets + } + + if len(statefulSets.Items) == 1 && d.AutoSelectSingleResource { + d.WithStatefulSet(statefulSets.Items[0].DeepCopy()) + + return nil + } + + items := []string{} + for _, item := range statefulSets.Items { + items = append(items, item.GetName()) + } + + statefulSet, err := util.Select("Select statefulset", items) + if err != nil { + return err + } + + for _, item := range statefulSets.Items { + if item.GetName() != statefulSet { + continue + } + + d.WithStatefulSet(item.DeepCopy()) + return nil + } + + return nil +} + +func (d *DebugComponent) SelectDaemonSet() error { + if d.namespace == nil { + return ErrNoNamespaceSelected + } + + daemonSets, err := d.kubernetesClient.ListDaemonSets(d.namespace.GetName()) + if err != nil { + return err + } + + if len(daemonSets.Items) == 0 { + return ErrNoDaemonSets + } + + if len(daemonSets.Items) == 1 && d.AutoSelectSingleResource { + d.WithDaemonSet(daemonSets.Items[0].DeepCopy()) + + return nil + } + + items := []string{} + for _, item := range daemonSets.Items { + items = append(items, item.GetName()) + } + + daemonSet, err := util.Select("Select daemonset", items) + if err != nil { + return err + } + + for _, item := range daemonSets.Items { + if item.GetName() != daemonSet { + continue + } + + d.WithDaemonSet(item.DeepCopy()) + return nil + } + + return nil +} + +func (d *DebugComponent) SelectContainer() error { + containers, err := d.getResourceContainers() + if err != nil { + return err + } + + + initContainers, err := d.getResourceInitContainers() + if err != nil { + return err + } + + initContainers = d.excludeRestrictedInitContainers(initContainers) + + // Pods created from Bunnyshell have unique names in the unified initContainers and containers collection + if d.ContainerName != "" { + for _, container := range containers { + if container.Name == d.ContainerName { + d.WithContainer(container.DeepCopy()) + return nil + } + } + + for _, initContainer := range initContainers { + if initContainer.Name == d.ContainerName { + d.WithInitContainer(initContainer.DeepCopy()) + return nil + } + } + + return ErrContainerNotFound + } + + container, isInit, err := d.selectContainer(containers, initContainers) + if err != nil { + return err + } + + if isInit { + d.WithInitContainer(container.DeepCopy()) + } else { + d.WithContainer(container.DeepCopy()) + } + + return nil +} + +func (d *DebugComponent) selectContainer(containers []coreV1.Container, initContainers []coreV1.Container) (*coreV1.Container, bool, error) { + if (len(containers) + len(initContainers)) == 1 && d.AutoSelectSingleResource { + if len(containers) == 1 { + return &containers[0], false, nil + } + + return &initContainers[0], true, nil + } + + initPrefix := "init - " + items := []string{} + for _, item := range initContainers { + items = append(items, initPrefix + item.Name) + } + for _, item := range containers { + items = append(items, item.Name) + } + + container, err := util.Select("Select container", items) + if err != nil { + return nil, false, err + } + + for _, item := range initContainers { + if initPrefix + item.Name == container { + return &item, true, nil + } + } + for _, item := range containers { + if item.Name == container { + return &item, false, nil + } + } + + return nil, false, ErrContainerNotFound +} + +func (d *DebugComponent) excludeRestrictedInitContainers(containers []coreV1.Container) []coreV1.Container { + restrictedNames := []string{"bns-volume-permissions"} + + // Convert restricted names to a map for O(1) lookups + restrictedMap := make(map[string]struct{}) + for _, name := range restrictedNames { + restrictedMap[name] = struct{}{} + } + + var result []coreV1.Container + for _, container := range containers { + if _, isRestricted := restrictedMap[container.Name]; isRestricted { + continue + } + + if strings.HasPrefix(container.Name, "init-shared-path-") { + continue + } + + result = append(result, container) + } + + return result +} diff --git a/pkg/debug/k8s.go b/pkg/debug/k8s.go new file mode 100644 index 0000000..5721196 --- /dev/null +++ b/pkg/debug/k8s.go @@ -0,0 +1,562 @@ +package debug + +import ( + "encoding/json" + "fmt" + "strconv" + "time" + + "bunnyshell.com/dev/pkg/k8s/patch" + + k8sTools "bunnyshell.com/dev/pkg/k8s/tools" + appsV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" + apiMetaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + appsCoreV1 "k8s.io/client-go/applyconfigurations/apps/v1" + applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1" + applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +const ( + MetadataPrefix = "debug.bunnyshell.com/" + MetadataActive = MetadataPrefix + "active" + MetadataStartedAt = MetadataPrefix + "started-at" + MetadataService = MetadataPrefix + "service" + MetadataContainer = MetadataPrefix + "container" + MetadataRollback = MetadataPrefix + "rollback-manifest" + + MetadataKubeCTLLastAppliedConf = "kubectl.kubernetes.io/last-applied-configuration" + MetadataK8SRevision = "deployment.kubernetes.io/revision" + + RemoteDevMetadataActive = "remote-dev.bunnyshell.com/active" +) + +var ( + ErrInvalidResourceType = fmt.Errorf("invalid resource type") +) + +type Resource interface { + GetName() string + GetNamespace() string + GetAnnotations() map[string]string + GetLabels() map[string]string +} + +func (d *DebugComponent) resourceTypeNotSupportedError() error { + return fmt.Errorf("resource type \"%s\" not supported", d.resourceType) +} + +func (d *DebugComponent) getResourcePatch() (patch.Resource, error) { + switch d.resourceType { + case Deployment: + var replicas int32 = 1 + strategy := appsV1.RecreateDeploymentStrategyType + return &patch.DeploymentPatchConfiguration{ + ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{}, + Spec: &patch.DeploymentSpecPatchConfiguration{ + Strategy: &patch.DeploymentStrategyPatchConfiguration{ + Type: &strategy, + RollingUpdate: nil, + }, + Replicas: &replicas, + }, + }, nil + case StatefulSet: + var replicas int32 = 1 + strategy := appsV1.OnDeleteStatefulSetStrategyType + return &patch.StatefulSetPatchConfiguration{ + ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{}, + Spec: &patch.StatefulSetSpecPatchConfiguration{ + UpdateStrategy: &patch.StatefulSetStrategyPatchConfiguration{ + Type: &strategy, + RollingUpdate: nil, + }, + Replicas: &replicas, + }, + }, nil + case DaemonSet: + strategy := appsV1.OnDeleteDaemonSetStrategyType + return &patch.DaemonSetPatchConfiguration{ + ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{}, + Spec: &patch.DaemonSetSpecPatchConfiguration{ + UpdateStrategy: &patch.DaemonSetStrategyPatchConfiguration{ + Type: &strategy, + RollingUpdate: nil, + }, + }, + }, nil + default: + return nil, d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) prepareResource() error { + d.StartSpinner(" Setup k8s pod for debugging") + defer d.StopSpinner() + + currentManifestSnapshot, err := d.getCurrentManifestSnapshot() + if err != nil { + return err + } + + resource, err := d.getResource() + if err != nil { + return err + } + + resourcePatch, err := d.getResourcePatch() + if err != nil { + return err + } + + annotations := make(map[string]string) + annotations[MetadataStartedAt] = strconv.FormatInt(d.startedAt, 10) + annotations[MetadataContainer] = d.container.Name + _, ok := resource.GetAnnotations()[MetadataRollback] + if !ok { + annotations[MetadataRollback] = string(currentManifestSnapshot) + } + labels := make(map[string]string) + labels[MetadataActive] = "true" + + resourcePatch.WithAnnotations(annotations).WithLabels(labels) + + podTemplateSpec := applyCoreV1.PodTemplateSpec() + if err := d.preparePodTemplateSpec(podTemplateSpec); err != nil { + return err + } + resourcePatch.WithSpecTemplate(podTemplateSpec) + + data, err := json.Marshal(resourcePatch) + if err != nil { + return err + } + + if err := d.resetResourceContainer(resource); err != nil { + return fmt.Errorf("cannot reset container: %w", err) + } + + switch d.resourceType { + case Deployment: + return d.kubernetesClient.PatchDeployment(resource.GetNamespace(), resource.GetName(), data) + case StatefulSet: + return d.kubernetesClient.PatchStatefulSet(resource.GetNamespace(), resource.GetName(), data) + case DaemonSet: + return d.kubernetesClient.PatchDaemonSet(resource.GetNamespace(), resource.GetName(), data) + default: + return d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) resetResourceContainer(resource Resource) error { + containerIndex, isInit, err := d.getContainerIndex() + if err != nil { + return err + } + + specPath := "containers" + if isInit { + specPath = "initContainers" + } + + // we need to use replace because remove fails if the path is missing + resetJSON, err := json.Marshal([]map[string]any{ + { + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/%s/%d/args", specPath, containerIndex), + "value": []string{}, + }, + { + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/%s/%d/readinessProbe", specPath, containerIndex), + "value": nil, + }, + { + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/%s/%d/livenessProbe", specPath, containerIndex), + "value": nil, + }, + { + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/%s/%d/startupProbe", specPath, containerIndex), + "value": nil, + }, + }) + + if err != nil { + return err + } + + switch d.resourceType { + case Deployment: + return d.kubernetesClient.BatchPatchDeployment(resource.GetNamespace(), resource.GetName(), resetJSON) + case StatefulSet: + return d.kubernetesClient.BatchPatchStatefulSet(resource.GetNamespace(), resource.GetName(), resetJSON) + case DaemonSet: + return d.kubernetesClient.BatchPatchDaemonSet(resource.GetNamespace(), resource.GetName(), resetJSON) + default: + return d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) restoreDeployment() error { + resource, err := d.getResource() + if err != nil { + return err + } + + snapshot, ok := resource.GetAnnotations()[MetadataRollback] + if !ok { + return fmt.Errorf("no rollback manifest available") + } + + switch d.resourceType { + case Deployment: + deployment := &appsV1.Deployment{} + if err := json.Unmarshal([]byte(snapshot), deployment); err != nil { + return err + } + + _, err = d.kubernetesClient.UpdateDeployment(deployment.GetNamespace(), deployment) + return err + case StatefulSet: + statefulSet := &appsV1.StatefulSet{} + if err := json.Unmarshal([]byte(snapshot), statefulSet); err != nil { + return err + } + + _, err = d.kubernetesClient.UpdateStatefulSet(statefulSet.GetNamespace(), statefulSet) + return err + case DaemonSet: + daemonSet := &appsV1.DaemonSet{} + if err := json.Unmarshal([]byte(snapshot), daemonSet); err != nil { + return err + } + + _, err = d.kubernetesClient.UpdateDaemonSet(daemonSet.GetNamespace(), daemonSet) + return err + default: + return d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) getResourceManifest() ([]byte, error) { + resource, err := d.getResource() + if err != nil { + return nil, err + } + + return json.Marshal(resource) +} + +func (d *DebugComponent) getCurrentManifestSnapshot() (string, error) { + fullSnapshot, err := d.getResourceManifest() + if err != nil { + return "", err + } + + var snapshot []byte + switch d.resourceType { + case Deployment: + applyResource := &appsCoreV1.DeploymentApplyConfiguration{} + if err := json.Unmarshal(fullSnapshot, applyResource); err != nil { + return "", err + } + + // strip unnecessary data + applyResource.WithStatus(nil) + applyResource.Generation = nil + applyResource.UID = nil + applyResource.ResourceVersion = nil + annotations := make(map[string]string) + for key, value := range applyResource.Annotations { + if key == MetadataK8SRevision || key == MetadataKubeCTLLastAppliedConf { + continue + } + + annotations[key] = value + } + applyResource.Annotations = annotations + + snapshot, err = json.Marshal(applyResource) + if err != nil { + return "", err + } + case StatefulSet: + applyResource := &appsCoreV1.StatefulSetApplyConfiguration{} + if err := json.Unmarshal(fullSnapshot, applyResource); err != nil { + return "", err + } + + // strip unnecessary data + applyResource.WithStatus(nil) + applyResource.Generation = nil + applyResource.UID = nil + applyResource.ResourceVersion = nil + annotations := make(map[string]string) + for key, value := range applyResource.Annotations { + if key == MetadataK8SRevision || key == MetadataKubeCTLLastAppliedConf { + continue + } + + annotations[key] = value + } + applyResource.Annotations = annotations + + snapshot, err = json.Marshal(applyResource) + if err != nil { + return "", err + } + case DaemonSet: + applyResource := &appsCoreV1.DaemonSetApplyConfiguration{} + if err := json.Unmarshal(fullSnapshot, applyResource); err != nil { + return "", err + } + + // strip unnecessary data + applyResource.WithStatus(nil) + applyResource.Generation = nil + applyResource.UID = nil + applyResource.ResourceVersion = nil + annotations := make(map[string]string) + for key, value := range applyResource.Annotations { + if key == MetadataK8SRevision || key == MetadataKubeCTLLastAppliedConf { + continue + } + + annotations[key] = value + } + applyResource.Annotations = annotations + + snapshot, err = json.Marshal(applyResource) + if err != nil { + return "", err + } + default: + return "", d.resourceTypeNotSupportedError() + } + + return string(snapshot), nil +} + +func (d *DebugComponent) preparePodTemplateSpec(podTemplateSpec *applyCoreV1.PodTemplateSpecApplyConfiguration) error { + resource, err := d.getResource() + if err != nil { + return err + } + + podAnnotations := make(map[string]string) + podAnnotations[MetadataStartedAt] = strconv.FormatInt(d.startedAt, 10) + podAnnotations[MetadataContainer] = d.container.Name + podLabels := make(map[string]string) + podLabels[MetadataActive] = "true" + podLabels[MetadataService] = resource.GetName() + + podTemplateSpec. + WithAnnotations(podAnnotations). + WithLabels(podLabels) + + return d.preparePodSpec(podTemplateSpec) +} + +func (d *DebugComponent) preparePodSpec(podTemplateSpec *applyCoreV1.PodTemplateSpecApplyConfiguration) error { + podSpec := applyCoreV1.PodSpec() + + + if err := d.prepareContainer(podSpec); err != nil { + return err + } + + podTemplateSpec.WithSpec(podSpec) + + return nil +} + +func (d *DebugComponent) prepareContainer(podSpec *applyCoreV1.PodSpecApplyConfiguration) error { + container := applyCoreV1.Container(). + WithName(d.container.Name). + WithCommand("sh", "-c", "tail -f /dev/null") + + if !d.isInitContainer { + nullProbe := d.getNullProbeApplyConfiguration() + + container. + WithLivenessProbe(nullProbe). + WithReadinessProbe(nullProbe). + WithStartupProbe(nullProbe) + } + + d.ContainerConfig.ApplyTo(container) + + if d.isInitContainer { + podSpec.WithInitContainers(container) + } else { + podSpec.WithContainers(container) + } + + return nil +} + +func (d *DebugComponent) getNullProbeApplyConfiguration() *applyCoreV1.ProbeApplyConfiguration { + return applyCoreV1.Probe(). + WithExec(applyCoreV1.ExecAction().WithCommand("true")). + WithPeriodSeconds(5) +} + +func (d *DebugComponent) getResourceSelector() (*apiMetaV1.LabelSelector, error) { + switch d.resourceType { + case Deployment: + return d.deployment.Spec.Selector, nil + case StatefulSet: + return d.statefulSet.Spec.Selector, nil + case DaemonSet: + return d.daemonSet.Spec.Selector, nil + default: + return nil, d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) waitPodReady() error { + d.StartSpinner(" Waiting for pod to be ready") + defer d.StopSpinner() + + resource, err := d.getResource() + if err != nil { + return err + } + + resourceSelector, err := d.getResourceSelector() + if err != nil { + return err + } + + namespace := resource.GetNamespace() + labelSelector := apiMetaV1.LabelSelector{MatchLabels: resourceSelector.MatchLabels} + listOptions := apiMetaV1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + } + + startTimestamp := time.Now().Unix() + for { + time.Sleep(1 * time.Second) + podList, err := d.kubernetesClient.ListPods(namespace, listOptions) + if err != nil { + return err + } + + for _, pod := range podList.Items { + if d.isInitContainer { + if pod.DeletionTimestamp != nil || pod.Status.Phase != coreV1.PodPending { + continue + } + + for _, containerStatus := range pod.Status.InitContainerStatuses { + if containerStatus.Name == d.container.Name && containerStatus.Started != nil && *containerStatus.Started { + return nil + } + } + } else { + if pod.DeletionTimestamp != nil || pod.Status.Phase != coreV1.PodRunning { + continue + } + + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.Name == d.container.Name && containerStatus.Ready { + return nil + } + } + } + } + + nowTimestamp := time.Now().Unix() + if nowTimestamp-startTimestamp >= d.waitTimeout { + break + } + } + + // timeout reached + return fmt.Errorf("pod not ready for debugging") +} + +func (d *DebugComponent) getResourceContainers() ([]coreV1.Container, error) { + switch d.resourceType { + case Deployment: + return k8sTools.GetDeploymentContainers(d.deployment), nil + case StatefulSet: + return k8sTools.GetStatefulSetContainers(d.statefulSet), nil + case DaemonSet: + return k8sTools.GetDaemonSetContainers(d.daemonSet), nil + default: + return []coreV1.Container{}, d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) getResourceContainer(containerName string) (*coreV1.Container, error) { + switch d.resourceType { + case Deployment: + return k8sTools.GetDeploymentContainerByName(d.deployment, containerName) + case StatefulSet: + return k8sTools.GetStatefulSetContainerByName(d.statefulSet, containerName) + case DaemonSet: + return k8sTools.GetDaemonSetContainerByName(d.daemonSet, containerName) + default: + return nil, ErrNoResourceSelected + } +} + +func (d *DebugComponent) getResourceInitContainers() ([]coreV1.Container, error) { + switch d.resourceType { + case Deployment: + return k8sTools.GetDeploymentInitContainers(d.deployment), nil + case StatefulSet: + return k8sTools.GetStatefulSetInitContainers(d.statefulSet), nil + case DaemonSet: + return k8sTools.GetDaemonSetInitContainers(d.daemonSet), nil + default: + return []coreV1.Container{}, d.resourceTypeNotSupportedError() + } +} + +func (d *DebugComponent) getResourceInitContainer(containerName string) (*coreV1.Container, error) { + switch d.resourceType { + case Deployment: + return k8sTools.GetDeploymentInitContainerByName(d.deployment, containerName) + case StatefulSet: + return k8sTools.GetStatefulSetInitContainerByName(d.statefulSet, containerName) + case DaemonSet: + return k8sTools.GetDaemonSetInitContainerByName(d.daemonSet, containerName) + default: + return nil, ErrNoResourceSelected + } +} + +func (d *DebugComponent) getContainerIndex() (int, bool, error) { + if d.container == nil { + return -1, false, fmt.Errorf("%w: %s", ErrContainerNotFound, "") + } + + containers, err := d.getResourceContainers() + if err != nil { + return -1, false, err + } + + for i, container := range containers { + if container.Name == d.container.Name { + return i, false, nil + } + } + + initContainers, err := d.getResourceInitContainers() + if err != nil { + return -1, true, err + } + + for i, initContainer := range initContainers { + if initContainer.Name == d.container.Name { + return i, true, nil + } + } + + return -1, false, fmt.Errorf("%w: %s", ErrContainerNotFound, d.container.Name) +} diff --git a/pkg/debug/main.go b/pkg/debug/main.go new file mode 100644 index 0000000..312c2a9 --- /dev/null +++ b/pkg/debug/main.go @@ -0,0 +1,94 @@ +package debug + +import ( + "fmt" + "os" + "os/signal" + + "bunnyshell.com/dev/pkg/util" +) + +func (d *DebugComponent) CanUp(forceRecreateResource bool) error { + resource, err := d.getResource() + if err != nil { + return err + } + + labels := resource.GetLabels() + if active, found := labels[RemoteDevMetadataActive]; found { + if active == "true" { + return fmt.Errorf("cannot start debug session, Pod already in a remote-development session") + } + } + + if active, found := labels[MetadataActive]; found { + if active == "true" { + annotations := resource.GetAnnotations() + if containerName, found := annotations[MetadataContainer]; found { + if (containerName == d.container.Name) { + d.shouldPrepareResource = forceRecreateResource + + return nil; + } + + if forceRecreateResource { + d.shouldPrepareResource = true + + return nil; + } else { + return fmt.Errorf("cannot start debug session, Pod already in another debug session on container %s.\nRun \"bns debug stop\" command then try again", containerName) + } + } + } + } + + d.shouldPrepareResource = true + + return nil +} + +func (d *DebugComponent) Up() error { + if (d.shouldPrepareResource) { + if err := d.prepareResource(); err != nil { + return err + } + } else { + fmt.Print("Skip recreating Pod\n") + } + + if err := d.waitPodReady(); err != nil { + return err + } + + return nil +} + +func (d *DebugComponent) Down() error { + if err := d.restoreDeployment(); err != nil { + return err + } + + return nil +} + +func (d *DebugComponent) Wait() error { + // close channels on cli signal interrupt + signalTermination := make(chan os.Signal, 1) + signal.Notify(signalTermination, util.TerminationSignals...) + defer signal.Stop(signalTermination) + + select { + case sig := <-signalTermination: + d.Close() + return fmt.Errorf("terminated by signal: %s", sig) + case <-d.stopChannel: + return nil + } +} + +func (d *DebugComponent) Close() { + // close cli command + if d.stopChannel != nil { + close(d.stopChannel) + } +} diff --git a/pkg/debug/spinner.go b/pkg/debug/spinner.go new file mode 100644 index 0000000..a0a5f15 --- /dev/null +++ b/pkg/debug/spinner.go @@ -0,0 +1,13 @@ +package debug + +func (d *DebugComponent) StartSpinner(suffix string) { + if suffix != "" { + d.spinner.Suffix = suffix + } + + d.spinner.Start() +} + +func (d *DebugComponent) StopSpinner() { + d.spinner.Stop() +} diff --git a/pkg/k8s/tools/util.go b/pkg/k8s/tools/util.go index d2a3913..55a3147 100644 --- a/pkg/k8s/tools/util.go +++ b/pkg/k8s/tools/util.go @@ -43,3 +43,30 @@ func GetDaemonSetContainerByName(daemonSet *appsV1.DaemonSet, containerName stri containers := GetDaemonSetContainers(daemonSet) return FilterContainerByName(containers, containerName) } + +func GetDeploymentInitContainers(deployment *appsV1.Deployment) []coreV1.Container { + return deployment.Spec.Template.Spec.InitContainers +} + +func GetDeploymentInitContainerByName(deployment *appsV1.Deployment, initContainerName string) (*coreV1.Container, error) { + containers := GetDeploymentInitContainers(deployment) + return FilterContainerByName(containers, initContainerName) +} + +func GetStatefulSetInitContainers(statefulSet *appsV1.StatefulSet) []coreV1.Container { + return statefulSet.Spec.Template.Spec.InitContainers +} + +func GetStatefulSetInitContainerByName(statefulSet *appsV1.StatefulSet, initContainerName string) (*coreV1.Container, error) { + containers := GetStatefulSetInitContainers(statefulSet) + return FilterContainerByName(containers, initContainerName) +} + +func GetDaemonSetInitContainers(daemonSet *appsV1.DaemonSet) []coreV1.Container { + return daemonSet.Spec.Template.Spec.InitContainers +} + +func GetDaemonSetInitContainerByName(daemonSet *appsV1.DaemonSet, initContainerName string) (*coreV1.Container, error) { + containers := GetDaemonSetInitContainers(daemonSet) + return FilterContainerByName(containers, initContainerName) +} \ No newline at end of file diff --git a/pkg/remote/k8s.go b/pkg/remote/k8s.go index 2f4c50b..1d77958 100644 --- a/pkg/remote/k8s.go +++ b/pkg/remote/k8s.go @@ -35,6 +35,8 @@ const ( MetadataKubeCTLLastAppliedConf = "kubectl.kubernetes.io/last-applied-configuration" MetadataK8SRevision = "deployment.kubernetes.io/revision" + DebugMetadataActive = "debug.bunnyshell.com/active" + VolumeNameBinaries = "remote-dev-bin" VolumeNameConfig = "remote-dev-config" VolumeNameWork = "remote-dev-work" diff --git a/pkg/remote/main.go b/pkg/remote/main.go index bd36263..2962ed7 100644 --- a/pkg/remote/main.go +++ b/pkg/remote/main.go @@ -8,6 +8,22 @@ import ( "bunnyshell.com/dev/pkg/util" ) +func (r *RemoteDevelopment) CanUp() error { + resource, err := r.getResource() + if err != nil { + return err + } + + labels := resource.GetLabels() + if active, found := labels[DebugMetadataActive]; found { + if active == "true" { + return fmt.Errorf("cannot start remote-development session, Pod already in a debug session") + } + } + + return nil +} + func (r *RemoteDevelopment) Up() error { if err := r.ensureSSHKeys(); err != nil { return err