feat: use Dockerhub API to poll for digest change (#19)

- Use Dockerhub API instead of checking manifest from Docker registry
so each poll does not count as an image pull. This is needed as we got
warning from docker support of exceeding docker rate limits.
- Mock Dockerhub server in tests.

.gitignore

@@ -1,8 +1,11 @@
 docker-registry-watcher
+registrywatcher
 *.toml
 !test.toml
 !sample.toml
 
+testutils/snakeoil
+
 # dependencies
 node_modules
 

Dockerfile-golang

@@ -1,6 +1,5 @@
-FROM golang:1.11-alpine
+FROM golang:1.12-alpine
 
 # Only install hard dependencies
-RUN apk add --no-cache gcc git musl-dev && \
-    go get -v -u golang.org/x/lint/golint && \
+RUN apk add --no-cache gcc git musl-dev make && \
     go get -v -u gotest.tools/gotestsum

Makefile

@@ -0,0 +1,12 @@
+# Setup name variables for the package/tool
+NAME := registrywatcher
+PKG := github.com/dsaidgovsg/$(NAME)
+
+GOROOT := $(shell go env | grep GOROOT | cut -b 8-)
+
+.PHONY: snakeoil
+snakeoil: ## Update snakeoil certs for testing.
+	go run $(GOROOT)/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
+	mkdir -p testutils/snakeoil
+	mv cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
+	mv key.pem $(CURDIR)/testutils/snakeoil/key.pem

README.md

@@ -115,20 +115,22 @@ You need to make one modification to your docker daemon config (usually at ~/.do
 
 Add the flag `--insecure-registry localhost:5000` to your docker daemon, documented [here](https://docs.docker.com/registry/insecure/) for testing against an insecure registry.
 
-You may need to update the certificates from time to time.
-```
-go run /usr/local/go/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
-	mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem
-	mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
+You may need to update the certificates from time to time. To do so, run
+
+```bash
+make snakeoil
 ```
 
 ## Tests
 
 Run locally:
 
-```
+```bash
 // run both integration and unit tests
 gotestsum -- -tags="integration unit" ./...
+
+// clean testcache
+go clean -testcache
 ```
 
 The tests don't test the nomad deployment capability of the service.
@@ -137,8 +139,6 @@ The tests don't test the nomad deployment capability of the service.
 
 A makefile to avoid having to change your local docker daemon config so as to run the tests as docker-in-docker for CI integration.
 
-A makefile step to regenerate testing certificates in `testutils/snakeoil`
-
 An interface to swap in the deployment agent (only Nomad supported for now).
 
 Tests take too long to run, this is mainly due each integration test case spinning up and down its own docker containers.
@@ -148,3 +148,5 @@ Nomad mock server cannot run jobs, which blocks writing of integration tests inv
 Buttons should implement debouncing
 
 Bug where only new tags/SHA changes AFTER you toggle back auto deployment will trigger an update, meaning any updates during the window where auto deployment was turned off are moot.
+
+Clean up unused registry methods.

client/client.go

@@ -18,6 +18,7 @@ type Clients struct {
 	NomadClient          *NomadClient
 	DockerRegistryClient *DockerRegistryClient
 	PostgresClient       *PostgresClient
+	DockerhubApi         *DockerhubApi
 	DockerTags           sync.Map
 	DigestMap            sync.Map
 
@@ -31,6 +32,10 @@ func SetUpClients(conf *viper.Viper) *Clients {
 		panic(fmt.Errorf("starting postgres client failed: %v", err))
 	}
 	dockerClient := InitializeDockerRegistryClient(conf)
+	dockerhubApi, err := InitializeDockerhubApi(conf)
+	if err != nil {
+		log.LogAppErr("error initializing dockerhub API client", err)
+	}
 
 	// caching fields
 	dockerTags := sync.Map{}
@@ -43,6 +48,7 @@ func SetUpClients(conf *viper.Viper) *Clients {
 		NomadClient:          InitializeNomadClient(conf),
 		PostgresClient:       postgresClient,
 		DockerRegistryClient: dockerClient,
+		DockerhubApi:         dockerhubApi,
 		DockerTags:           dockerTags,
 		DigestMap:            digestMap,
 	}
@@ -84,6 +90,7 @@ func SetUpTestClients(t *testing.T, conf *viper.Viper) *Clients {
 		NomadServer:          ns,
 		PostgresClient:       postgresClient,
 		DockerRegistryClient: InitializeDockerRegistryClient(conf),
+		DockerhubApi:         nil,
 		DockerTags:           dockerTags,
 		DigestMap:            digestMap,
 	}
@@ -145,6 +152,8 @@ func (client *Clients) DeployPinnedTag(conf *viper.Viper, repoName string) {
 	jobID := utils.GetRepoNomadJob(conf, repoName)
 	taskName := utils.GetRepoNomadTaskName(conf, repoName)
 	client.NomadClient.UpdateNomadJobTag(jobID, repoName, taskName, pinnedTag)
+	// update after deploying new sha, so it will not trigger autodeployment
+	client.updateCaches(repoName)
 }
 
 func (client *Clients) PopulateCaches(repoName string) {
@@ -162,11 +171,12 @@ func (client *Clients) PopulateCaches(repoName string) {
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while populating cache for %s", repoName), err)
 	}
-	tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag)
-	client.updateDigestCache(repoName, tagDigest)
+	tagDigest, err := client.DockerhubApi.GetTagDigestFromApi(repoName, pinnedTag)
 	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while populating cache for %s", repoName), err)
+		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from Dockerhub while populating cache for %s", repoName), err)
+		return
 	}
+	client.updateDigestCache(repoName, *tagDigest)
 }
 
 func (client *Clients) isNewReleaseTagAvailable(repoName string) bool {
@@ -249,21 +259,21 @@ func (client *Clients) isTagDigestChanged(repoName string) (bool, error) {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while checking if it was changed for %s", repoName), err)
 		return false, err
 	}
-	tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag)
+	cachedTagDigest, err := client.GetCachedTagDigest(repoName)
 	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while checking if it was changed for %s", repoName), err)
+		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while checking if it was changed for %s", repoName), err)
 		return false, err
 	}
-	cachedTagDigest, err := client.GetCachedTagDigest(repoName)
+	digestIsCurrent, err := client.DockerhubApi.CheckImageIsCurrent(repoName, cachedTagDigest, pinnedTag)
 	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while checking if it was changed for %s", repoName), err)
+		log.LogAppErr("Couldn't check if tag currently points to cached image digest", err)
 		return false, err
 	}
 
-	return cachedTagDigest != tagDigest, nil
+	return !*digestIsCurrent, nil
 }
 
-func (client *Clients) UpdateCaches(repoName string) {
+func (client *Clients) updateCaches(repoName string) {
 	validTags, err := client.getSHATags(repoName)
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch tags from registry while updating cache for %s", repoName), err)
@@ -282,7 +292,7 @@ func (client *Clients) UpdateCaches(repoName string) {
 			log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while updating cache for %s", repoName), err)
 			return
 		}
-		tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag)
+		tagDigest, err := client.DockerhubApi.GetTagDigestFromApi(repoName, pinnedTag)
 		if err != nil {
 			log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while updating cache for %s", repoName), err)
 			return
@@ -294,21 +304,22 @@ func (client *Clients) UpdateCaches(repoName string) {
 			log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while updating cache for %s", repoName), err)
 			return
 		}
-		log.LogAppInfo(fmt.Sprintf("cached digest: %s, new digest: %s", cachedTagDigest, tagDigest))
+		log.LogAppInfo(fmt.Sprintf("cached digest: %s, new digest: %s", cachedTagDigest, *tagDigest))
 
-		client.updateDigestCache(repoName, tagDigest)
+		client.updateDigestCache(repoName, *tagDigest)
 	}
 }
 
 // this function compares cached values with the actual values,
-// so only update the cache after calling, not before
+// so only update the cache before returning non-error cases
 func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 	autoDeploy, err := client.PostgresClient.GetAutoDeployFlag(repoName)
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch whether to deploy flag while checking whether to deploy for %s", repoName), err)
 		return false, err
 	}
 	if !autoDeploy {
+		client.updateCaches(repoName)
 		return false, nil
 	}
 
@@ -324,8 +335,9 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 	}
 
 	if (pinnedTag == "" && client.isNewReleaseTagAvailable(repoName)) || isDigestChanged {
+		client.updateCaches(repoName)
 		return true, nil
 	}
-
+	client.updateCaches(repoName)
 	return false, nil
 }

client/client_test.go

@@ -8,7 +8,9 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-// test ShouldDeploy for a multitude of conditions
+/*
+	Test ShouldDeploy for a multitude of conditions.
+*/
 func TestAutoDeployLatestTag(t *testing.T) {
 	te := SetUpClientTest(t)
 	defer te.TearDown()
@@ -31,7 +33,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "latest")
 
 	shouldDeploy, _ := te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// v0.0.2 is not a new release version
 	assert.False(t, shouldDeploy)
@@ -42,7 +43,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "latest")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// v0.2.0 is a new release version,
 	assert.True(t, shouldDeploy)
@@ -53,7 +53,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "latest")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// v0.0.9 is not a new release version
 	assert.False(t, shouldDeploy)
@@ -68,16 +67,16 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.UpdatePinnedTag(newTag)
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
+	// should not deploy as all tags are based on the digest "latest", even though
+	// the pinned tag is changed
 	assert.False(t, shouldDeploy)
 	assert.Equal(t, "test", tagToDeploy)
 
 	// "test" is now based on "alpine", rather than the original "latest"
 	te.PushNewTag(newTag, "alpine")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	assert.True(t, shouldDeploy)
 	assert.Equal(t, "test", tagToDeploy)

client/docker_registry_client.go

@@ -62,13 +62,3 @@ func (e *DockerRegistryClient) GetAllTags(repoName string) ([]string, error) {
 	tags, err := repoRegistry.Tags(fmt.Sprintf("%s/%s", registryPrefix, repoName))
 	return tags, err
 }
-
-func (e *DockerRegistryClient) GetTagDigest(repoName, tag string) (string, error) {
-	_, _, registryPrefix, _ := utils.ExtractRegistryInfo(e.conf, repoName)
-	repoRegistry := e.Hubs[repoName]
-	deserializedManifest, err := repoRegistry.ManifestV2(fmt.Sprintf("%s/%s", registryPrefix, repoName), tag)
-	if err != nil {
-		return "", err
-	}
-	return string(deserializedManifest.Manifest.Config.Digest), nil
-}