Merge pull request #1 from simoncaron/feature/restart-linked-containers

Feature/restart linked containers

README.md

@@ -52,6 +52,7 @@ Restart your unhealthy containers safely
 
 1. Set labels on containers:
     - To restart containers if they go unhealthy, use the label `deunhealth.restart.on.unhealthy=true`
+    - To restart another container, when an unhealthy one is restarted by deunhealth, use the label `deunhealth.restart.with.unhealthy.container=<deunhealth-monitored container name>`
 
 1. You can update the image with `docker pull qmcgaw/deunhealth:latest` or use one of the [tags available](https://hub.docker.com/r/qmcgaw/deunhealth/tags). ⚠️ You might want to use tagged images since `latest` will likely break compatibility until we reach a `v1.0.0` release.
 

integration_test.sh

@@ -10,6 +10,7 @@ finish() {
   [ "$nohealthMarkedID" != "" ] && docker rm -f "$nohealthMarkedID"
   [ "$healthyMarkedID" != "" ] && docker rm -f "$healthyMarkedID"
   [ "$unhealthyMarkedID" != "" ] && docker rm -f "$unhealthyMarkedID"
+  [ "$nohealthIDLinkedID" != "" ] && docker rm -f "$nohealthIDLinkedID"
   echo "done"
 }
 trap finish EXIT
@@ -23,6 +24,7 @@ docker --version
 
 healthFlags="--health-start-period=0s --health-interval=40ms --health-retries=1"
 restartOnUnhealthyLabel="--label deunhealth.restart.on.unhealthy=true"
+linkedOnUnhealthyLabel="--label deunhealth.restart.with.unhealthy.container="
 
 echo "launching test containers"
 
@@ -40,6 +42,9 @@ nohealthMarkedName="$(docker inspect -f '{{ .Name }}' $nohealthMarkedID | sed -r
 healthyMarkedName="$(docker inspect -f '{{ .Name }}' $healthyMarkedID | sed -r 's/^\///')"
 unhealthyMarkedName="$(docker inspect -f '{{ .Name }}' $unhealthyMarkedID | sed -r 's/^\///')"
 
+nohealthLinkedID="$(docker run -d --init $linkedOnUnhealthyLabel$unhealthyMarkedName alpine:3.15 sleep 30)"
+nohealthLinkedName="$(docker inspect -f '{{ .Name }}' $nohealthLinkedID | sed -r 's/^\///')"
+
 echo "launching deunhealth"
 
 deunhealthID="$(docker run -d -v /var/run/docker.sock:/var/run/docker.sock qmcgaw/deunhealth)"
@@ -54,5 +59,6 @@ logs="$(docker logs $deunhealthID)"
 [ "$(echo $logs | grep -o $healthyName | wc -l)" = "0" ] || ( echo "Container $healthyName appears in deunhealth logs"; echo "$logs"; exit 1 )
 [ "$(echo $logs | grep -o $unhealthyName | wc -l)" = "0" ] || ( echo "Container $unhealthyName appears in deunhealth logs"; echo "$logs"; exit 1 )
 [ "$(echo $logs | grep -o $nohealthMarkedName | wc -l)" = "0" ] || ( echo "Container $nohealthMarkedName appears in deunhealth logs"; echo "$logs"; exit 1 )
-[ "$(echo $logs | grep -o $healthyMarkedName | wc -l)" = "0" ] || ( echo "Container $healthyMarkedName appears in deunhealth logs"; echo "$logs"; exit 1 )
+[ "$(echo $logs | grep -o $healthyMarkedName | wc -l)" != "0" ] || ( echo "Container $healthyMarkedName does not appears in deunhealth logs"; echo "$logs"; exit 1 )
 [ "$(echo $logs | grep -o $unhealthyMarkedName | wc -l)" != "0" ] || ( echo "Container $unhealthyMarkedName does not appear in deunhealth logs"; echo "$logs"; exit 1 )
+[ "$(echo $logs | grep -o $nohealthLinkedName | wc -l)" != "0" ] || ( echo "Container $nohealthLinkedName does not appear in deunhealth logs"; echo "$logs"; exit 1 )

internal/docker/docker.go

@@ -12,6 +12,7 @@ var _ Dockerer = (*Docker)(nil)
 type Dockerer interface {
 	UnhealthyGetter
 	UnhealthyStreamer
+	LinkedContainerGetter
 	ContainerRestarter
 	LabeledGetter
 }

internal/docker/name.go

@@ -1,6 +1,8 @@
 package docker
 
 import (
+	"strings"
+
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/events"
 )
@@ -9,7 +11,7 @@ func extractName(container types.Container) (name string) {
 	name = container.ID
 	for _, containerName := range container.Names {
 		if containerName != "" {
-			name = containerName
+			name = strings.TrimPrefix(containerName, "/")
 			break
 		}
 	}

internal/docker/unhealthy.go

@@ -82,3 +82,33 @@ func (d *Docker) StreamUnhealthy(ctx context.Context, unhealthies chan<- Contain
 		}
 	}
 }
+
+type LinkedContainerGetter interface {
+	GetLinkedContainer(ctx context.Context, unhealthy Container) (linkedContainers []Container, err error)
+}
+
+func (d *Docker) GetLinkedContainer(ctx context.Context, unhealthy Container) (linkedContainers []Container, err error) {
+	// See https://docs.docker.com/engine/reference/commandline/ps/#filtering
+	filtersArgs := filters.NewArgs()
+	filtersArgs.Add("label", "deunhealth.restart.with.unhealthy.container=" + unhealthy.Name)
+
+	containers, err := d.client.ContainerList(ctx, types.ContainerListOptions{
+		Filters: filtersArgs,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	linkedContainers = make([]Container, len(containers))
+
+	for i, container := range containers {
+		linkedContainers[i] = Container{
+			ID:    container.ID,
+			Name:  extractName(container),
+			Image: container.Image,
+		}
+	}
+
+	return linkedContainers, nil
+}

internal/loop/unhealthy.go

@@ -88,4 +88,20 @@ func (l *unhealthyLoop) restartUnhealthy(ctx context.Context, unhealthy docker.C
 	} else {
 		l.logger.Info("container " + unhealthy.Name + " restarted successfully")
 	}
+	l.restartLinked(ctx, unhealthy)
+}
+
+func (l *unhealthyLoop) restartLinked(ctx context.Context, unhealthy docker.Container) {
+	linkedContainers, _ := l.docker.GetLinkedContainer(ctx, unhealthy)
+	for _, linkedContainer := range linkedContainers {
+		l.logger.Info("container " + linkedContainer.Name +
+			" (image " + linkedContainer.Image + ") is linked to unhealthy container " + 
+			unhealthy.Name + ", restarting it...")
+		err := l.docker.RestartContainer(ctx, linkedContainer.Name)
+		if err != nil {
+			l.logger.Error("failed restarting linked container: " + err.Error())
+		} else {
+			l.logger.Info("linked container " + linkedContainer.Name + " restarted successfully")
+		}	
+	}
 }