From bb6d093baac15dc31c69acf22b97bf08ca959042 Mon Sep 17 00:00:00 2001 From: Becca Qiu Date: Tue, 3 Aug 2021 18:42:44 +0800 Subject: [PATCH] feat: use Dockerhub API to poll for digest change (#19) - Use Dockerhub API instead of checking manifest from Docker registry so each poll does not count as an image pull. This is needed as we got warning from docker support of exceeding docker rate limits. - Mock Dockerhub server in tests. --- .gitignore | 3 + Dockerfile-golang | 5 +- Makefile | 12 ++ README.md | 18 +-- client/client.go | 40 +++--- client/client_test.go | 11 +- client/docker_registry_client.go | 10 -- client/dockerhub_api_client.go | 206 +++++++++++++++++++++++++++++++ client/setup.go | 93 +++++++++++++- config/sample.toml | 10 +- go.mod | 16 ++- go.sum | 77 ++++++++++++ main.go | 6 - test.sh | 6 +- testutils/snakeoil/cert.pem | 19 --- testutils/snakeoil/key.pem | 27 ---- worker/watcher_worker.go | 3 - 17 files changed, 452 insertions(+), 110 deletions(-) create mode 100644 Makefile create mode 100644 client/dockerhub_api_client.go delete mode 100644 testutils/snakeoil/cert.pem delete mode 100644 testutils/snakeoil/key.pem diff --git a/.gitignore b/.gitignore index a9e092f..4f4cde6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ docker-registry-watcher +registrywatcher *.toml !test.toml !sample.toml +testutils/snakeoil + # dependencies node_modules diff --git a/Dockerfile-golang b/Dockerfile-golang index 94e78e6..7195f3c 100644 --- a/Dockerfile-golang +++ b/Dockerfile-golang @@ -1,6 +1,5 @@ -FROM golang:1.11-alpine +FROM golang:1.12-alpine # Only install hard dependencies -RUN apk add --no-cache gcc git musl-dev && \ - go get -v -u golang.org/x/lint/golint && \ +RUN apk add --no-cache gcc git musl-dev make && \ go get -v -u gotest.tools/gotestsum diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..157b2b7 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +# Setup name variables for the package/tool +NAME := registrywatcher +PKG := github.com/dsaidgovsg/$(NAME) + +GOROOT := $(shell go env | grep GOROOT | cut -b 8-) + +.PHONY: snakeoil +snakeoil: ## Update snakeoil certs for testing. + go run $(GOROOT)/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca + mkdir -p testutils/snakeoil + mv cert.pem $(CURDIR)/testutils/snakeoil/cert.pem + mv key.pem $(CURDIR)/testutils/snakeoil/key.pem diff --git a/README.md b/README.md index c1f0ef6..4de1bf6 100644 --- a/README.md +++ b/README.md @@ -115,20 +115,22 @@ You need to make one modification to your docker daemon config (usually at ~/.do Add the flag `--insecure-registry localhost:5000` to your docker daemon, documented [here](https://docs.docker.com/registry/insecure/) for testing against an insecure registry. -You may need to update the certificates from time to time. -``` -go run /usr/local/go/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca - mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem - mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem +You may need to update the certificates from time to time. To do so, run + +```bash +make snakeoil ``` ## Tests Run locally: -``` +```bash // run both integration and unit tests gotestsum -- -tags="integration unit" ./... + +// clean testcache +go clean -testcache ``` The tests don't test the nomad deployment capability of the service. @@ -137,8 +139,6 @@ The tests don't test the nomad deployment capability of the service. A makefile to avoid having to change your local docker daemon config so as to run the tests as docker-in-docker for CI integration. -A makefile step to regenerate testing certificates in `testutils/snakeoil` - An interface to swap in the deployment agent (only Nomad supported for now). Tests take too long to run, this is mainly due each integration test case spinning up and down its own docker containers. @@ -148,3 +148,5 @@ Nomad mock server cannot run jobs, which blocks writing of integration tests inv Buttons should implement debouncing Bug where only new tags/SHA changes AFTER you toggle back auto deployment will trigger an update, meaning any updates during the window where auto deployment was turned off are moot. + +Clean up unused registry methods. diff --git a/client/client.go b/client/client.go index 08dc83a..4024c6b 100644 --- a/client/client.go +++ b/client/client.go @@ -18,6 +18,7 @@ type Clients struct { NomadClient *NomadClient DockerRegistryClient *DockerRegistryClient PostgresClient *PostgresClient + DockerhubApi *DockerhubApi DockerTags sync.Map DigestMap sync.Map @@ -31,6 +32,10 @@ func SetUpClients(conf *viper.Viper) *Clients { panic(fmt.Errorf("starting postgres client failed: %v", err)) } dockerClient := InitializeDockerRegistryClient(conf) + dockerhubApi, err := InitializeDockerhubApi(conf) + if err != nil { + log.LogAppErr("error initializing dockerhub API client", err) + } // caching fields dockerTags := sync.Map{} @@ -43,6 +48,7 @@ func SetUpClients(conf *viper.Viper) *Clients { NomadClient: InitializeNomadClient(conf), PostgresClient: postgresClient, DockerRegistryClient: dockerClient, + DockerhubApi: dockerhubApi, DockerTags: dockerTags, DigestMap: digestMap, } @@ -84,6 +90,7 @@ func SetUpTestClients(t *testing.T, conf *viper.Viper) *Clients { NomadServer: ns, PostgresClient: postgresClient, DockerRegistryClient: InitializeDockerRegistryClient(conf), + DockerhubApi: nil, DockerTags: dockerTags, DigestMap: digestMap, } @@ -145,6 +152,8 @@ func (client *Clients) DeployPinnedTag(conf *viper.Viper, repoName string) { jobID := utils.GetRepoNomadJob(conf, repoName) taskName := utils.GetRepoNomadTaskName(conf, repoName) client.NomadClient.UpdateNomadJobTag(jobID, repoName, taskName, pinnedTag) + // update after deploying new sha, so it will not trigger autodeployment + client.updateCaches(repoName) } func (client *Clients) PopulateCaches(repoName string) { @@ -162,11 +171,12 @@ func (client *Clients) PopulateCaches(repoName string) { if err != nil { log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while populating cache for %s", repoName), err) } - tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag) - client.updateDigestCache(repoName, tagDigest) + tagDigest, err := client.DockerhubApi.GetTagDigestFromApi(repoName, pinnedTag) if err != nil { - log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while populating cache for %s", repoName), err) + log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from Dockerhub while populating cache for %s", repoName), err) + return } + client.updateDigestCache(repoName, *tagDigest) } func (client *Clients) isNewReleaseTagAvailable(repoName string) bool { @@ -249,21 +259,21 @@ func (client *Clients) isTagDigestChanged(repoName string) (bool, error) { log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while checking if it was changed for %s", repoName), err) return false, err } - tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag) + cachedTagDigest, err := client.GetCachedTagDigest(repoName) if err != nil { - log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while checking if it was changed for %s", repoName), err) + log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while checking if it was changed for %s", repoName), err) return false, err } - cachedTagDigest, err := client.GetCachedTagDigest(repoName) + digestIsCurrent, err := client.DockerhubApi.CheckImageIsCurrent(repoName, cachedTagDigest, pinnedTag) if err != nil { - log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while checking if it was changed for %s", repoName), err) + log.LogAppErr("Couldn't check if tag currently points to cached image digest", err) return false, err } - return cachedTagDigest != tagDigest, nil + return !*digestIsCurrent, nil } -func (client *Clients) UpdateCaches(repoName string) { +func (client *Clients) updateCaches(repoName string) { validTags, err := client.getSHATags(repoName) if err != nil { log.LogAppErr(fmt.Sprintf("Couldn't fetch tags from registry while updating cache for %s", repoName), err) @@ -282,7 +292,7 @@ func (client *Clients) UpdateCaches(repoName string) { log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while updating cache for %s", repoName), err) return } - tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag) + tagDigest, err := client.DockerhubApi.GetTagDigestFromApi(repoName, pinnedTag) if err != nil { log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while updating cache for %s", repoName), err) return @@ -294,14 +304,14 @@ func (client *Clients) UpdateCaches(repoName string) { log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while updating cache for %s", repoName), err) return } - log.LogAppInfo(fmt.Sprintf("cached digest: %s, new digest: %s", cachedTagDigest, tagDigest)) + log.LogAppInfo(fmt.Sprintf("cached digest: %s, new digest: %s", cachedTagDigest, *tagDigest)) - client.updateDigestCache(repoName, tagDigest) + client.updateDigestCache(repoName, *tagDigest) } } // this function compares cached values with the actual values, -// so only update the cache after calling, not before +// so only update the cache before returning non-error cases func (client *Clients) ShouldDeploy(repoName string) (bool, error) { autoDeploy, err := client.PostgresClient.GetAutoDeployFlag(repoName) if err != nil { @@ -309,6 +319,7 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) { return false, err } if !autoDeploy { + client.updateCaches(repoName) return false, nil } @@ -324,8 +335,9 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) { } if (pinnedTag == "" && client.isNewReleaseTagAvailable(repoName)) || isDigestChanged { + client.updateCaches(repoName) return true, nil } - + client.updateCaches(repoName) return false, nil } diff --git a/client/client_test.go b/client/client_test.go index 5712f0c..64dcdb8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -8,7 +8,9 @@ import ( "github.com/stretchr/testify/assert" ) -// test ShouldDeploy for a multitude of conditions +/* + Test ShouldDeploy for a multitude of conditions. +*/ func TestAutoDeployLatestTag(t *testing.T) { te := SetUpClientTest(t) defer te.TearDown() @@ -31,7 +33,6 @@ func TestAutoDeployLatestTag(t *testing.T) { te.PushNewTag(newTag, "latest") shouldDeploy, _ := te.Clients.ShouldDeploy(te.TestRepoName) - te.Clients.UpdateCaches(te.TestRepoName) tagToDeploy, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName) // v0.0.2 is not a new release version assert.False(t, shouldDeploy) @@ -42,7 +43,6 @@ func TestAutoDeployLatestTag(t *testing.T) { te.PushNewTag(newTag, "latest") shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - te.Clients.UpdateCaches(te.TestRepoName) tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) // v0.2.0 is a new release version, assert.True(t, shouldDeploy) @@ -53,7 +53,6 @@ func TestAutoDeployLatestTag(t *testing.T) { te.PushNewTag(newTag, "latest") shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - te.Clients.UpdateCaches(te.TestRepoName) tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) // v0.0.9 is not a new release version assert.False(t, shouldDeploy) @@ -68,8 +67,9 @@ func TestAutoDeployLatestTag(t *testing.T) { te.UpdatePinnedTag(newTag) shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - te.Clients.UpdateCaches(te.TestRepoName) tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) + // should not deploy as all tags are based on the digest "latest", even though + // the pinned tag is changed assert.False(t, shouldDeploy) assert.Equal(t, "test", tagToDeploy) @@ -77,7 +77,6 @@ func TestAutoDeployLatestTag(t *testing.T) { te.PushNewTag(newTag, "alpine") shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - te.Clients.UpdateCaches(te.TestRepoName) tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) assert.True(t, shouldDeploy) assert.Equal(t, "test", tagToDeploy) diff --git a/client/docker_registry_client.go b/client/docker_registry_client.go index 3562d60..37d4104 100644 --- a/client/docker_registry_client.go +++ b/client/docker_registry_client.go @@ -62,13 +62,3 @@ func (e *DockerRegistryClient) GetAllTags(repoName string) ([]string, error) { tags, err := repoRegistry.Tags(fmt.Sprintf("%s/%s", registryPrefix, repoName)) return tags, err } - -func (e *DockerRegistryClient) GetTagDigest(repoName, tag string) (string, error) { - _, _, registryPrefix, _ := utils.ExtractRegistryInfo(e.conf, repoName) - repoRegistry := e.Hubs[repoName] - deserializedManifest, err := repoRegistry.ManifestV2(fmt.Sprintf("%s/%s", registryPrefix, repoName), tag) - if err != nil { - return "", err - } - return string(deserializedManifest.Manifest.Config.Digest), nil -} diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go new file mode 100644 index 0000000..41bd7f0 --- /dev/null +++ b/client/dockerhub_api_client.go @@ -0,0 +1,206 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/dsaidgovsg/registrywatcher/log" + "github.com/pkg/errors" + "github.com/spf13/viper" +) + +type DockerhubApi struct { + url string + namespace string + token string + username string + secret string +} + +type AuthenticateResp struct { + Token string `json:"token"` +} + +type CheckImageResp struct { + Results []TagWithStatus `json:"results"` +} + +type TagWithStatus struct { + Tag string `json:"tag"` + IsCurrent bool `json:"is_current"` +} + +type GetTagDigestResp struct { + Results []GetTagDigestResult `json:"results"` +} + +type GetTagDigestResult struct { + Digest string `json:"digest"` + Tags []TagWithStatus `json:"tags"` +} + +func InitializeDockerhubApi(conf *viper.Viper) (*DockerhubApi, error) { + client := DockerhubApi{ + url: conf.GetString("dockerhub_url"), + namespace: conf.GetString("dockerhub_namespace"), + username: conf.GetString("dockerhub_username"), + secret: conf.GetString("dockerhub_secret"), + token: "", + } + + jwt, err := client.Authenticate() + if err != nil { + return &client, err + } + + client.token = *jwt + return &client, nil +} + +func (api *DockerhubApi) Authenticate() (*string, error) { + addr := fmt.Sprintf("%s%s", api.url, "/v2/users/login") + log.LogAppInfo(fmt.Sprintf("dockerhub.users.login url=%s", addr)) + + data := map[string]string{"username": api.username, "password": api.secret} + jsonData, err := json.Marshal(data) + + req, err := http.NewRequest("POST", addr, bytes.NewBuffer(jsonData)) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + log.LogAppInfo("Error obtaining JWT") + errMsg := fmt.Sprintf("Response status %d message %s", resp.StatusCode, string(body)) + return nil, errors.New(errMsg) + } + + var deserialized AuthenticateResp + json.Unmarshal(body, &deserialized) + + return &deserialized.Token, nil +} + +func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag string) ( + *bool, error) { + endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images/%s/tags", + api.namespace, repository, digest) + addr := fmt.Sprintf("%s%s", api.url, endpoint) + log.LogAppInfo(fmt.Sprintf("dockerhub check if image is current, url=%s", addr)) + + req, err := http.NewRequest("GET", addr, nil) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token)) + req.Header.Set("Accept", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + // Obtain JWT if it has expired + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + log.LogAppInfo("Obtaining new JWT") + jwt, err := api.Authenticate() + if err != nil { + return nil, err + } + + api.token = *jwt + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token)) + resp, err = http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("Response status %d message %s", resp.StatusCode, string(body)) + return nil, errors.New(errMsg) + } + + var deserialized CheckImageResp + json.Unmarshal(body, &deserialized) + + for _, item := range deserialized.Results { + if item.Tag == checkTag { + return &item.IsCurrent, nil + } + } + // image tag does not match the tag to be checked + // this means the cached digest belongs to a previous tag + // return false (not current) because we want the cache to be updated + log.LogAppInfo(fmt.Sprintf(fmt.Sprintf("Digest %s does not have tag %s", digest, checkTag))) + isCurrent := false + return &isCurrent, nil +} + +// Note that the digest returned from the Dockerhub API does not match +// the digest from docker registry manifest V2 API +func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string) ( + *string, error) { + endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images?", api.namespace, + repository) + queryParams := fmt.Sprintf("currently_tagged=%s&page_size=%s&ordering=%s", "true", + "100", "-last_activity") + + addr := fmt.Sprintf("%s%s%s", api.url, endpoint, queryParams) + log.LogAppInfo(fmt.Sprintf("dockerhub get tag digest url=%s", addr)) + + req, err := http.NewRequest("GET", addr, nil) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token)) + req.Header.Set("Accept", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + // Obtain JWT if it has expired + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + log.LogAppInfo("Obtaining new JWT") + jwt, err := api.Authenticate() + if err != nil { + return nil, err + } + + api.token = *jwt + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token)) + resp, err = http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("Response status %d message %s", resp.StatusCode, string(body)) + return nil, errors.New(errMsg) + } + + var deserialized GetTagDigestResp + json.Unmarshal(body, &deserialized) + + for _, item := range deserialized.Results { + for _, tag := range item.Tags { + if tag.Tag == checkTag && tag.IsCurrent { + return &item.Digest, nil + } + } + } + + // image tag not found in repository's 100 last active tags + return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", checkTag, repository)) +} diff --git a/client/setup.go b/client/setup.go index e0165d3..d27df88 100644 --- a/client/setup.go +++ b/client/setup.go @@ -10,6 +10,12 @@ import ( "github.com/dsaidgovsg/registrywatcher/utils" nomad "github.com/hashicorp/nomad/api" "github.com/spf13/viper" + + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/gorilla/mux" ) type testEngine struct { @@ -17,6 +23,8 @@ type testEngine struct { Conf *viper.Viper helper *testutils.TestHelper Clients *Clients + ImageTagMap map[string][]TagWithStatus + Ts *httptest.Server TestRepoName string } @@ -28,8 +36,8 @@ func (te *testEngine) printState() { fmt.Println("cached tags", cachedTags) pinnedTag, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName) - registryTagDigest, _ := te.Clients.DockerRegistryClient.GetTagDigest(te.TestRepoName, pinnedTag) - fmt.Println("registry tag digest", registryTagDigest) + tagDigest, _ := te.Clients.DockerhubApi.GetTagDigestFromApi(te.TestRepoName, pinnedTag) + fmt.Println("new tag digest", *tagDigest) cachedTagDigest, _ := te.Clients.GetCachedTagDigest(te.TestRepoName) fmt.Println("cached tag digest", cachedTagDigest) @@ -67,6 +75,60 @@ func SetUpClientTest(t *testing.T) *testEngine { // we use this so much might as well keep it in the struct te.TestRepoName = te.Conf.GetStringSlice("watched_repositories")[0] + // initialize mock imageTag store + te.ImageTagMap = make(map[string][]TagWithStatus) + + // initialize mock Dockerhub server + router := mux.NewRouter() + + router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) { + res.Write([]byte(`{"token": "test token"}`)) + }).Methods("GET") + + router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images/{digest}/tags", + func(res http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + digest := vars["digest"] + + res.Header().Set("Content-Type", "application/json") + res.WriteHeader(http.StatusOK) + + if tags, ok := te.ImageTagMap[digest]; ok { + results := CheckImageResp{Results: tags} + data, _ := json.Marshal(results) + res.Write(data) + } else { + results := CheckImageResp{Results: nil} + data, _ := json.Marshal(results) + res.Write(data) + } + }).Methods("GET") + + router.HandleFunc("/v2/namespaces/namespace/repositories/testrepo/images", + func(res http.ResponseWriter, req *http.Request) { + var resSlice []GetTagDigestResult + for image, tags := range te.ImageTagMap { + resSlice = append(resSlice, GetTagDigestResult{Digest: image, Tags: tags}) + } + res.Header().Set("Content-Type", "application/json") + res.WriteHeader(http.StatusOK) + results := GetTagDigestResp{Results: resSlice} + data, _ := json.Marshal(results) + res.Write(data) + }) + + ts := httptest.NewServer(router) + te.Ts = ts + + dockerhubApi := DockerhubApi{ + url: ts.URL, + namespace: "namespace", + username: "username", + secret: "secret", + token: "fake token", + } + + te.Clients.DockerhubApi = &dockerhubApi return &te } @@ -120,15 +182,38 @@ func (te *testEngine) TearDown() { log.LogAppErr("Couldn't remove container", err) } } + te.Ts.Close() } -// named tag is what it will appear as in the docker registryDomain -// actual tag is what the tag is based on (this is for testing purposes only) +/* + For tests, note that the base image tags "latest" and "alpine" are taken to be equivalent + to image digest strings in the mock image-tag store. The mock image tag store is used + because we are mocking the Dockerhub API server. While in the real world, image digests + are SHA hashes, this still allows us to test the auto-deploy behaviour as we use the + digest string to check if the underlying image is changed. +*/ func (te *testEngine) PushNewTag(namedTag, actualTag string) { _, registryDomain, registryPrefix, _ := utils.ExtractRegistryInfo(te.Conf, te.TestRepoName) mockImageName := utils.ConstructImageName(registryDomain, registryPrefix, te.TestRepoName, namedTag) publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag) err := te.helper.AddImageToRegistry(publicImageName, mockImageName) + + imageDigest := actualTag + if _, ok := te.ImageTagMap[imageDigest]; ok { + te.ImageTagMap[imageDigest] = append(te.ImageTagMap[imageDigest], TagWithStatus{namedTag, true}) + } else { + te.ImageTagMap[imageDigest] = []TagWithStatus{{namedTag, true}} + } + + // make is_current false for older images holding the same tag + for image, tags := range te.ImageTagMap { + for i, tag := range tags { + if tag.Tag == namedTag && image != imageDigest { + tags[i].IsCurrent = false + } + } + } + if err != nil { panic(fmt.Errorf("couldn't add image to registry: %v", err)) } diff --git a/config/sample.toml b/config/sample.toml index f1c9e0c..092c00f 100644 --- a/config/sample.toml +++ b/config/sample.toml @@ -16,13 +16,19 @@ webhook_url = "$YOUR_SLACK_URL_HERE" database_url = "postgresql://registry-watcher:registry-watcher@localhost:5432/registry-watcher?sslmode=disable" create_database_schema = true +# Dockerhub API Client +dockerhub_url = "https://hub.docker.com" +dockerhub_namespace = "namespace" +dockerhub_username = "user" +dockerhub_secret = "secret" + # Tests is_test = false # Docker Registry information (to be interpolated by Nomad) -[registry_map.codefresh] +[registry_map.dockerhub] registry_scheme = "https" -registry_domain = "r.cfcr.io" +registry_domain = "registry-1.docker.io" registry_prefix = "some_prefix" registry_auth = "$YOUR_AUTH_STRING_HERE" diff --git a/go.mod b/go.mod index 2ea83a9..954ee87 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,10 @@ require ( github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect + github.com/fatih/color v1.12.0 // indirect github.com/gin-contrib/cors v1.3.0 github.com/gin-gonic/gin v1.4.0 - github.com/gorilla/mux v1.7.1 // indirect + github.com/gorilla/mux v1.7.1 github.com/gorilla/websocket v1.4.0 // indirect github.com/hashicorp/consul v1.4.4 // indirect github.com/hashicorp/go-hclog v0.9.2 // indirect @@ -25,21 +26,30 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.1.0 github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 // indirect + github.com/mattn/go-isatty v0.0.13 // indirect github.com/mitchellh/hashstructure v1.0.0 // indirect github.com/nlopes/slack v0.5.0 + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.4.3 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.1 // indirect github.com/spf13/viper v1.3.2 github.com/stretchr/testify v1.3.0 go.uber.org/atomic v1.4.0 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/tools v0.1.5 // indirect google.golang.org/appengine v1.5.0 // indirect google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect + gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect + gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect gotest.tools v2.2.0+incompatible // indirect - gotest.tools/gotestsum v0.3.4 // indirect + gotest.tools/gotestsum v1.6.4 // indirect ) replace github.com/ugorji/go v1.1.4 => github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 diff --git a/go.sum b/go.sum index 6634482..a5f065e 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f h1:W4fbqg0JUwy6lLesoJaV/rE0fwAmtdtinMa64X1CEh0= @@ -35,9 +37,15 @@ github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNE github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gin-contrib/cors v1.3.0 h1:PolezCc89peu+NgkIWt9OB01Kbzt6IP0J/JvkG6xxlg= github.com/gin-contrib/cors v1.3.0/go.mod h1:artPvLlhkF7oG06nK8v3U8TNz6IeX+w1uzCSEId5/Vc= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g= @@ -64,6 +72,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= @@ -136,6 +148,8 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -156,9 +170,15 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= @@ -204,6 +224,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -240,6 +262,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMVXab7UayvSlIpiyGwRvuX3EKYY7UA= github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= @@ -251,9 +275,18 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -261,17 +294,28 @@ golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU= golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -285,18 +329,48 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpbl golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= @@ -332,4 +406,7 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v0.3.4 h1:LdVJDg3RHrci4MbupUgSkwPCikz4kTzDHWtUahDAleY= gotest.tools/gotestsum v0.3.4/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY= +gotest.tools/gotestsum v1.6.4 h1:HFkapG0hK/HWiOxWS78SbR/JK5EpbH8hFzUuCvvfbfQ= +gotest.tools/gotestsum v1.6.4/go.mod h1:fTR9ZhxC/TLAAx2/WMk/m3TkMB9eEI89gdEzhiRVJT8= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 6adc928..fe0d71e 100644 --- a/main.go +++ b/main.go @@ -152,8 +152,6 @@ func (h *Handler) ResetTagHandler(c *gin.Context) { } else { log.LogAppInfo(fmt.Sprintf("Updated pinned_tag for repo %s from %s to %s succesfully, deployment of pinned_tag will happen shortly", repoName, originalTag, pinnedTag)) h.clients.DeployPinnedTag(h.conf, repoName) - // update after deploying new sha, so it will not trigger autodeployment - h.clients.UpdateCaches(repoName) c.JSON(200, gin.H{ "message": fmt.Sprintf("Deploying to %s", pinnedTag), }) @@ -236,8 +234,6 @@ func (h *Handler) DeployTagHandler(c *gin.Context) { originalTag, err := h.clients.PostgresClient.GetPinnedTag(repoName) if originalTag == pinnedTag { h.clients.DeployPinnedTag(h.conf, repoName) - // update after deploying new sha, so it will not trigger autodeployment - h.clients.UpdateCaches(repoName) c.JSON(200, gin.H{ "message": fmt.Sprintf("Deploying to %s", pinnedTag), }) @@ -255,8 +251,6 @@ func (h *Handler) DeployTagHandler(c *gin.Context) { } else { log.LogAppInfo(fmt.Sprintf("Updated pinned_tag for repo %s from %s to %s succesfully, deployment of pinned_tag will happen shortly", repoName, originalTag, pinnedTag)) h.clients.DeployPinnedTag(h.conf, repoName) - // update after deploying new sha, so it will not trigger autodeployment - h.clients.UpdateCaches(repoName) c.JSON(200, gin.H{ "message": fmt.Sprintf("Deploying to %s", pinnedTag), }) diff --git a/test.sh b/test.sh index 841d269..2439d35 100755 --- a/test.sh +++ b/test.sh @@ -8,7 +8,7 @@ assert_each_go_file () { failed_files=$(find . -type f -iname '*.go' -not -path "./.*/*" -exec $@ {} \;) printf "${failed_files}" - if [[ ${failed_files} ]]; then + if [ -n "${failed_files}" ]; then printf "\nFAILED!\n" exit 1 fi @@ -18,10 +18,6 @@ echo "-- gofmt check --" assert_each_go_file gofmt -l echo "OK" -echo "-- golint check --" -assert_each_go_file golint -echo "OK" - echo "-- go unit tests --" gotestsum -- -tags=unit ./... echo "OK" diff --git a/testutils/snakeoil/cert.pem b/testutils/snakeoil/cert.pem deleted file mode 100644 index 19d7724..0000000 --- a/testutils/snakeoil/cert.pem +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDAjCCAeqgAwIBAgIQXPx/2Iddx4oM41yA7X8HzDANBgkqhkiG9w0BAQsFADAS -MRAwDgYDVQQKEwdBY21lIENvMB4XDTIwMDYwNzExNDg0MFoXDTIxMDYwNzExNDg0 -MFowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC -AQoCggEBAND3WGSWGaZEkGfJ8lZabPLyMUgGHhivTPTbjwGl+3O2BW59/wdJ3q5R -nD4ilmoKipJwGnlwhwvrHJA17fGgZVWslXqCo1Gq3WYoUZ/bJD6EpZYPqgH6RY5l -pVsmiY8hCRDNcgfdfl1NbxwkQ3zJgUEUyEg2/BTLQ3Dvr4byUx8yGptBg6tek4t2 -oXk1WYF6N4RVTtfcgys3S0H/F7rpO3e+eK5trMTRQ7m4Y/XFt22J3BujvvQffblJ -uGHKt6Am/RFiSHyU0k2NHwitCZtBesWvg8zvh/TFLrsC4Ib+YeNNihn535y6HLVq -GLu9TOgdShjHbsrez9WsCeQfub30NQcCAwEAAaNUMFIwDgYDVR0PAQH/BAQDAgKk -MBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wGgYDVR0RBBMw -EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBU747cFtp1Rg/z -iB7QM66AzWFO2spWtXEKFkQZXmmsTbI+OltwtyD3GJRiGK4nfQYJErGAed2f8SG4 -N6829rffDdc5EftqhVoqtVrCgLtkdHH9tyKzCTDnPkwAZvDHjfv+gs61G6AoaxXa -Q0UEy/OFA7eP7FodXms1SQeC0rnm+O/+wRZwzS6cllNpsf2gI/VTJbzJyi+GG0OW -xoaiwpqYnfN8ESphdrp+W/ECA5IXofenlBuDWHTU9l7agqv8wZsYHaxI5yYNnZi6 -nHPjlT/M5EsZCb/k+/fok0/G6FMc+OXC0uDlbTq8/uKslkTsIbCkZQyjgF+h9Q6h -7krNzH1J ------END CERTIFICATE----- diff --git a/testutils/snakeoil/key.pem b/testutils/snakeoil/key.pem deleted file mode 100644 index 0ce2090..0000000 --- a/testutils/snakeoil/key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEA0PdYZJYZpkSQZ8nyVlps8vIxSAYeGK9M9NuPAaX7c7YFbn3/ -B0nerlGcPiKWagqKknAaeXCHC+sckDXt8aBlVayVeoKjUardZihRn9skPoSllg+q -AfpFjmWlWyaJjyEJEM1yB91+XU1vHCRDfMmBQRTISDb8FMtDcO+vhvJTHzIam0GD -q16Ti3aheTVZgXo3hFVO19yDKzdLQf8Xuuk7d754rm2sxNFDubhj9cW3bYncG6O+ -9B99uUm4Ycq3oCb9EWJIfJTSTY0fCK0Jm0F6xa+DzO+H9MUuuwLghv5h402KGfnf -nLoctWoYu71M6B1KGMduyt7P1awJ5B+5vfQ1BwIDAQABAoIBACXuP+VCObWsjGnK -hmqWT2BqBfgE5EGm0FlB4VtzPYlraOD9jeVMy1bpCWtd69+jujOKlcOlYV4Qd0bW -AzLw4nQCt79dJzgTw8xfVTGnHZUbjtMGVN23CBko6jL5c86/OF4MbL5wvhdBjwGU -s4tbKFMf6prxICDGJJ/DYiEkq2Hl6le/KX2/MvxD/QpqEWjD11PIKrB5mXle37Rl -96l+gPwTAZjuSTAFRlkW0N80oaj7ePVngjEIPK2okrDJyvbSvTA5dceoPCyeDrna -RAadkHY2UBGf8bM5mX7UNesFr43x1ze3CSzCwyHHXGHRBSiU29ThBs68+Ch4jdf0 -XvNg/AECgYEA8CKBMn4nUWX3IMOsM8mWP1jlRZXXJ+wJx1C360tckcq3PowiyFcj -VcUoTM/qEdKFYbiIk51MhEnPXkqxQamfEQtPwWQ7DqJ3MDs+Gjzat0kLTMCwxmy7 -qCMUVmDWcnPQYuIBvypfKBdupNsP2lQBwPHeQPeXEM35KLrDi6MYDsECgYEA3sWs -fgEg3Kuhu/dRzVcGOh1U3jRT/bNgn5bx3h14H2CA/k9mkHOQ3qscs67HJDGLwMf7 -KA53F0Oi+U7ISxUNbki6FMH4PXPGUtQgTd1rmpqfmo6RqjjF8B4XwW2yS6Jq9yci -oXvpVUXIiILoLP3xFvkvKxNqB5Bgn7r1bMas/ccCgYA8/xbAQ6krFPJ0IR/xu6ft -iWAbj9cqLdvCBC/jla363nozahc/ulqgY8c706hKRKZazyvqCw+vnqwM8H4cBsD/ -7Fai8hnKOtx5pOK+eWy5F9XF9kAw9ten+tkko851HhP1uiDldRbcsS8KsxZQx/v2 -Dbn92KZHv3olrWMUaiYnQQKBgF+nKQWBnoah+3QmIa21rCrP5VzzwwywNNDRk1D3 -/lgaq8iGd1eXB6FUf6J3bXKRA5znQWRVsbT/ib7aoQ38ICQ2fg/jqZ9ZUtLF2okU -hduAgP9wMAOTFQqhITogy+nF93/gAfKsree7avuSmfaXoylmHWa1k1VkHNj5AeCO -/kCJAoGAYTZGxQ97WpRupawLlir3kaPEwAlyltGvISHNweOcnbqIrTFeRXq9vnpj -jUPVRxI8pS7DULHyOFrp5mpCiyptNCGOOR5DTRU7059XWr2nXloq83U2/zyfvhWC -mc5mZ0Tpp0QVq3qvBMqH/UpqDSWPtHi9DjL4oy+Ve3oyfr3Mc+Q= ------END RSA PRIVATE KEY----- diff --git a/worker/watcher_worker.go b/worker/watcher_worker.go index e15a00c..db95142 100644 --- a/worker/watcher_worker.go +++ b/worker/watcher_worker.go @@ -44,12 +44,9 @@ func (ww *WatcherWorker) initialize() { func (ww *WatcherWorker) runOnce() { shouldDeploy, err := ww.clients.ShouldDeploy(ww.repoName) if err != nil { - ww.clients.UpdateCaches(ww.repoName) return } originalTag, err := ww.clients.GetFormattedPinnedTag(ww.repoName) - // update AFTER checking for changes - ww.clients.UpdateCaches(ww.repoName) if err != nil || !shouldDeploy { return }