From b7d5e766eefc91d562469db58c97518946613545 Mon Sep 17 00:00:00 2001 From: Jakub Coufal Date: Sun, 20 May 2018 21:37:00 +0200 Subject: [PATCH 1/3] Test upload pkg --- Gopkg.lock | 33 +++++++++++++++- pkg/upload/upload.go | 19 +++++---- pkg/upload/upload_test.go | 81 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 pkg/upload/upload_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 22094dc..6ce3322 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -501,39 +501,70 @@ name = "k8s.io/client-go" packages = [ "discovery", + "discovery/fake", "kubernetes", + "kubernetes/fake", "kubernetes/scheme", "kubernetes/typed/admissionregistration/v1alpha1", + "kubernetes/typed/admissionregistration/v1alpha1/fake", "kubernetes/typed/admissionregistration/v1beta1", + "kubernetes/typed/admissionregistration/v1beta1/fake", "kubernetes/typed/apps/v1", + "kubernetes/typed/apps/v1/fake", "kubernetes/typed/apps/v1beta1", + "kubernetes/typed/apps/v1beta1/fake", "kubernetes/typed/apps/v1beta2", + "kubernetes/typed/apps/v1beta2/fake", "kubernetes/typed/authentication/v1", + "kubernetes/typed/authentication/v1/fake", "kubernetes/typed/authentication/v1beta1", + "kubernetes/typed/authentication/v1beta1/fake", "kubernetes/typed/authorization/v1", + "kubernetes/typed/authorization/v1/fake", "kubernetes/typed/authorization/v1beta1", + "kubernetes/typed/authorization/v1beta1/fake", "kubernetes/typed/autoscaling/v1", + "kubernetes/typed/autoscaling/v1/fake", "kubernetes/typed/autoscaling/v2beta1", + "kubernetes/typed/autoscaling/v2beta1/fake", "kubernetes/typed/batch/v1", + "kubernetes/typed/batch/v1/fake", "kubernetes/typed/batch/v1beta1", + "kubernetes/typed/batch/v1beta1/fake", "kubernetes/typed/batch/v2alpha1", + "kubernetes/typed/batch/v2alpha1/fake", "kubernetes/typed/certificates/v1beta1", + "kubernetes/typed/certificates/v1beta1/fake", "kubernetes/typed/core/v1", + "kubernetes/typed/core/v1/fake", "kubernetes/typed/events/v1beta1", + "kubernetes/typed/events/v1beta1/fake", "kubernetes/typed/extensions/v1beta1", + "kubernetes/typed/extensions/v1beta1/fake", "kubernetes/typed/networking/v1", + "kubernetes/typed/networking/v1/fake", "kubernetes/typed/policy/v1beta1", + "kubernetes/typed/policy/v1beta1/fake", "kubernetes/typed/rbac/v1", + "kubernetes/typed/rbac/v1/fake", "kubernetes/typed/rbac/v1alpha1", + "kubernetes/typed/rbac/v1alpha1/fake", "kubernetes/typed/rbac/v1beta1", + "kubernetes/typed/rbac/v1beta1/fake", "kubernetes/typed/scheduling/v1alpha1", + "kubernetes/typed/scheduling/v1alpha1/fake", "kubernetes/typed/settings/v1alpha1", + "kubernetes/typed/settings/v1alpha1/fake", "kubernetes/typed/storage/v1", + "kubernetes/typed/storage/v1/fake", "kubernetes/typed/storage/v1alpha1", + "kubernetes/typed/storage/v1alpha1/fake", "kubernetes/typed/storage/v1beta1", + "kubernetes/typed/storage/v1beta1/fake", "pkg/version", "rest", "rest/watch", + "testing", "tools/auth", "tools/clientcmd", "tools/clientcmd/api", @@ -562,6 +593,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "0a0b58e4aff27b5d7d52a353fa93c2dca9301cafb6ff1ead0861e129c42c4bf2" + inputs-digest = "8d62002ccf4dad7deb18c6519fec8befe50a4362463dd4583d6fce79d10a4d58" solver-name = "gps-cdcl" solver-version = 1 diff --git a/pkg/upload/upload.go b/pkg/upload/upload.go index db0f733..02ebc15 100644 --- a/pkg/upload/upload.go +++ b/pkg/upload/upload.go @@ -34,15 +34,20 @@ const ( Upsert MergeType = "upsert" ) +// FileIter provides an iterator for the files in a tree. +type FileIter interface { + ForEach(cb func(*object.File) error) error +} + // Uploader uploading data to target type Uploader interface { // Upload files into config map tagged by commitID - Upload(commitID string, iter *object.FileIter) error + Upload(commitID string, iter FileIter) error } type uploader struct { restconfig *rest.Config - clientset *kubernetes.Clientset + clientset kubernetes.Interface namespace string name string mergeType MergeType @@ -121,7 +126,7 @@ func NewConfigMapUploader(o *UploaderOptions) (Uploader, error) { }, nil } -func (u *configmapUploader) Upload(commitID string, iter *object.FileIter) error { +func (u *configmapUploader) Upload(commitID string, iter FileIter) error { configMaps := u.clientset.CoreV1().ConfigMaps(u.namespace) data, err := u.iterToConfigMapData(iter) @@ -220,7 +225,7 @@ func (u *configmapUploader) createConfigMap(configMaps typedcore.ConfigMapInterf return nil } -func (u *configmapUploader) iterToConfigMapData(iter *object.FileIter) (map[string]string, error) { +func (u *configmapUploader) iterToConfigMapData(iter FileIter) (map[string]string, error) { var data = make(map[string]string) err := iter.ForEach(func(file *object.File) error { if filterFile(file, u.includes, u.excludes) { @@ -283,7 +288,7 @@ func NewSecretUploader(o *UploaderOptions) (Uploader, error) { }, nil } -func (u *secretUploader) Upload(commitID string, iter *object.FileIter) error { +func (u *secretUploader) Upload(commitID string, iter FileIter) error { secrets := u.clientset.CoreV1().Secrets(u.namespace) data, err := u.iterToSecretData(iter) @@ -382,7 +387,7 @@ func (u *secretUploader) createSecret(secrets typedcore.SecretInterface, data ma return nil } -func (u *secretUploader) iterToSecretData(iter *object.FileIter) (map[string][]byte, error) { +func (u *secretUploader) iterToSecretData(iter FileIter) (map[string][]byte, error) { var data = make(map[string][]byte) err := iter.ForEach(func(file *object.File) error { if filterFile(file, u.includes, u.excludes) { @@ -434,7 +439,7 @@ func NewFolderUploader(o *UploaderOptions) (Uploader, error) { }, nil } -func (u *folderUploader) Upload(commitID string, iter *object.FileIter) error { +func (u *folderUploader) Upload(commitID string, iter FileIter) error { err := iter.ForEach(func(file *object.File) error { if filterFile(file, u.includes, u.excludes) { src := path.Join(u.sourcePath, file.Name) diff --git a/pkg/upload/upload_test.go b/pkg/upload/upload_test.go new file mode 100644 index 0000000..4ca9b24 --- /dev/null +++ b/pkg/upload/upload_test.go @@ -0,0 +1,81 @@ +package upload + +import ( + "testing" + testclient "k8s.io/client-go/kubernetes/fake" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing/filemode" +) + +type mockFileIter struct { + files []*object.File +} + +func (m *mockFileIter) ForEach(cb func(*object.File) error) error { + for _, f := range m.files { + cb(f) + } + return nil +} + +func TestConfigmapUploader_Upload(t *testing.T) { + cases := []struct { + name string + namespace string + mapname string + labels map[string]string + annotations map[string]string + iter *mockFileIter + }{ + { + name: "Empty JSON in default namespace", + namespace: "default", + mapname: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + iter: &mockFileIter{ + files: []*object.File{ + object.NewFile("test.json", filemode.Regular, &object.Blob{}), + }, + }, + }, + { + name: "No files in config namespace", + namespace: "config", + mapname: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + iter: &mockFileIter{ + files: []*object.File{}, + }, + }, + } + + for _, c := range cases { + fakeclient := testclient.NewSimpleClientset() + cu := &configmapUploader{ + clientset: fakeclient, + namespace: c.namespace, + name: c.mapname, + labels: c.labels, + annotations: c.annotations, + } + err := cu.Upload("id", c.iter) + if err != nil { + t.Errorf("%s case failed: %v", c.name, err) + } + + firsta := fakeclient.Actions()[0] + if firsta.GetNamespace() != c.namespace { + t.Errorf("%s case failed expected %s namespace but got %s instead", c.name, c.namespace, firsta.GetNamespace()) + } + } +} + +func TestSecretUploader_Upload(t *testing.T) { + +} + +func TestFolderUploader_Upload(t *testing.T) { + +} From fb07aab7d2ef2bd906030fc641690318a3edae60 Mon Sep 17 00:00:00 2001 From: "jakub.coufal" Date: Mon, 21 May 2018 15:35:43 +0200 Subject: [PATCH 2/3] Test upload pkg - test uploaded content --- pkg/upload/testdata/test.json | 3 + pkg/upload/testdata/test.yaml | 2 + pkg/upload/upload_test.go | 218 ++++++++++++++++++++++++++++------ 3 files changed, 186 insertions(+), 37 deletions(-) create mode 100644 pkg/upload/testdata/test.json create mode 100644 pkg/upload/testdata/test.yaml diff --git a/pkg/upload/testdata/test.json b/pkg/upload/testdata/test.json new file mode 100644 index 0000000..3297e9c --- /dev/null +++ b/pkg/upload/testdata/test.json @@ -0,0 +1,3 @@ +{ + "test": 1 +} \ No newline at end of file diff --git a/pkg/upload/testdata/test.yaml b/pkg/upload/testdata/test.yaml new file mode 100644 index 0000000..c98d4ff --- /dev/null +++ b/pkg/upload/testdata/test.yaml @@ -0,0 +1,2 @@ +test: + some: 1 \ No newline at end of file diff --git a/pkg/upload/upload_test.go b/pkg/upload/upload_test.go index 4ca9b24..0221bc2 100644 --- a/pkg/upload/upload_test.go +++ b/pkg/upload/upload_test.go @@ -1,10 +1,18 @@ package upload import ( - "testing" - testclient "k8s.io/client-go/kubernetes/fake" - "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "io/ioutil" + "k8s.io/apimachinery/pkg/apis/meta/v1" + testclient "k8s.io/client-go/kubernetes/fake" + testing2 "k8s.io/client-go/testing" + "os" + "path/filepath" + "reflect" + "regexp" + "testing" ) type mockFileIter struct { @@ -13,69 +21,205 @@ type mockFileIter struct { func (m *mockFileIter) ForEach(cb func(*object.File) error) error { for _, f := range m.files { - cb(f) + obj := &plumbing.MemoryObject{} + content, err := ioutil.ReadFile(filepath.Join("testdata", f.Name)) + if err != nil { + panic(err) + } + obj.Write(content) + obj.SetType(plumbing.BlobObject) + blob := &object.Blob{} + err = blob.Decode(obj) + if err != nil { + panic(err) + } + + cb(object.NewFile(f.Name, f.Mode, blob)) } return nil } -func TestConfigmapUploader_Upload(t *testing.T) { - cases := []struct { - name string - namespace string - mapname string - labels map[string]string - annotations map[string]string - iter *mockFileIter - }{ - { - name: "Empty JSON in default namespace", - namespace: "default", - mapname: "git2kube", - labels: map[string]string{}, - annotations: map[string]string{}, - iter: &mockFileIter{ - files: []*object.File{ - object.NewFile("test.json", filemode.Regular, &object.Blob{}), - }, +var basicCases = []struct { + name string + namespace string + target string + includes []*regexp.Regexp + excludes []*regexp.Regexp + labels map[string]string + annotations map[string]string + iter *mockFileIter + contains []string +}{ + { + name: "Empty JSON in default namespace include all", + namespace: "default", + target: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + includes: []*regexp.Regexp{ + regexp.MustCompile(".*"), + }, + iter: &mockFileIter{ + files: []*object.File{ + object.NewFile("test.json", filemode.Regular, &object.Blob{}), + object.NewFile("test.yaml", filemode.Regular, &object.Blob{}), }, }, - { - name: "No files in config namespace", - namespace: "config", - mapname: "git2kube", - labels: map[string]string{}, - annotations: map[string]string{}, - iter: &mockFileIter{ - files: []*object.File{}, + contains: []string{"test.json", "test.yaml"}, + }, + { + name: "Empty JSON in default namespace include json", + namespace: "default", + target: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + includes: []*regexp.Regexp{ + regexp.MustCompile(".*\\.json"), + }, + iter: &mockFileIter{ + files: []*object.File{ + object.NewFile("test.json", filemode.Regular, &object.Blob{}), + object.NewFile("test.yaml", filemode.Regular, &object.Blob{}), }, }, - } + contains: []string{"test.json"}, + }, + { + name: "No files in config namespace", + namespace: "config", + target: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + iter: &mockFileIter{ + files: []*object.File{}, + }, + }, + { + name: "No files in config namespace with annotations and labels", + namespace: "config", + target: "git2kube", + labels: map[string]string{ + "test1": "value1", + }, + annotations: map[string]string{ + "test2": "value2", + }, + iter: &mockFileIter{ + files: []*object.File{}, + }, + }, +} - for _, c := range cases { +func TestConfigmapUploader_Upload(t *testing.T) { + for _, c := range basicCases { fakeclient := testclient.NewSimpleClientset() cu := &configmapUploader{ clientset: fakeclient, namespace: c.namespace, - name: c.mapname, + name: c.target, labels: c.labels, annotations: c.annotations, + includes: c.includes, + excludes: c.excludes, } err := cu.Upload("id", c.iter) if err != nil { t.Errorf("%s case failed: %v", c.name, err) } - firsta := fakeclient.Actions()[0] - if firsta.GetNamespace() != c.namespace { - t.Errorf("%s case failed expected %s namespace but got %s instead", c.name, c.namespace, firsta.GetNamespace()) + assertAction(fakeclient.Actions()[0], t, c.name, c.namespace, "get", "configmaps") + assertAction(fakeclient.Actions()[1], t, c.name, c.namespace, "create", "configmaps") + res, err := fakeclient.CoreV1().ConfigMaps(c.namespace).Get(c.target, v1.GetOptions{}) + if err != nil { + t.Errorf("%s case failed: %v", c.name, err) } + assertAnnotationsAndLabels(res.Annotations, res.Labels, t, c.name, c.annotations, c.labels) + assertData(res.Data, t, c.name, c.contains) } } func TestSecretUploader_Upload(t *testing.T) { + for _, c := range basicCases { + fakeclient := testclient.NewSimpleClientset() + cu := &secretUploader{ + clientset: fakeclient, + namespace: c.namespace, + name: c.target, + labels: c.labels, + annotations: c.annotations, + includes: c.includes, + excludes: c.excludes, + } + err := cu.Upload("id", c.iter) + if err != nil { + t.Errorf("%s case failed: %v", c.name, err) + } + + assertAction(fakeclient.Actions()[0], t, c.name, c.namespace, "get", "secrets") + assertAction(fakeclient.Actions()[1], t, c.name, c.namespace, "create", "secrets") + + res, err := fakeclient.CoreV1().Secrets(c.namespace).Get(c.target, v1.GetOptions{}) + if err != nil { + t.Errorf("%s case failed: %v", c.name, err) + } + assertAnnotationsAndLabels(res.Annotations, res.Labels, t, c.name, c.annotations, c.labels) + data := make(map[string]string) + for k, v := range res.Data { + data[k] = string(v[:]) + } + assertData(data, t, c.name, c.contains) + } } func TestFolderUploader_Upload(t *testing.T) { + ex, err := os.Executable() + if err != nil { + panic(err) + } + exPath := filepath.Dir(ex) + for _, c := range basicCases { + cu := &folderUploader{ + sourcePath: exPath, + name: c.target, + includes: c.includes, + excludes: c.excludes, + } + err := cu.Upload("id", c.iter) + if err != nil { + t.Errorf("%s case failed: %v", c.name, err) + } + } +} + +func assertAction(action testing2.Action, t *testing.T, name string, namespace string, verb string, resource string) { + if action.GetNamespace() != namespace { + t.Errorf("%s case failed: expected '%s' namespace but got '%s' instead", name, namespace, action.GetNamespace()) + } + if !action.Matches(verb, resource) { + t.Errorf("%s case failed: expected action '[%s]%s' namespace but got '[%s]%s' instead", name, verb, resource, action.GetVerb(), action.GetResource().Resource) + } +} + +func assertAnnotationsAndLabels(annotations map[string]string, labels map[string]string, t *testing.T, name string, exannotations map[string]string, exlabels map[string]string) { + if !reflect.DeepEqual(annotations, exannotations) { + t.Errorf("%s case failed: expected annotations '%s' but got '%s' instead", name, exannotations, annotations) + } + + if !reflect.DeepEqual(labels, exlabels) { + t.Errorf("%s case failed: expected labels '%s' but got '%s' instead", name, exlabels, labels) + } +} + +func assertData(data map[string]string, t *testing.T, name string, contains []string) { + if len(contains) != len(data) { + t.Errorf("%s case failed: expected data '%s' but got '%s' instead", name, contains, reflect.ValueOf(data).MapKeys()) + } + + for _, k := range contains { + if _, ok := data[k]; !ok { + t.Errorf("%s case failed: expected data with key '%s' in '%s'", name, k, data) + } + } } From d9e2814ebc4a7555654a3ce77a9f546993031301 Mon Sep 17 00:00:00 2001 From: "jakub.coufal" Date: Mon, 21 May 2018 15:43:39 +0200 Subject: [PATCH 3/3] Test upload pkg - added more cases and checked content --- pkg/upload/upload_test.go | 41 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/pkg/upload/upload_test.go b/pkg/upload/upload_test.go index 0221bc2..a9413bb 100644 --- a/pkg/upload/upload_test.go +++ b/pkg/upload/upload_test.go @@ -1,6 +1,7 @@ package upload import ( + "encoding/base64" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -50,6 +51,18 @@ var basicCases = []struct { iter *mockFileIter contains []string }{ + { + name: "Empty JSON in default namespace no include", + namespace: "default", + target: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + iter: &mockFileIter{ + files: []*object.File{ + object.NewFile("test.json", filemode.Regular, &object.Blob{}), + }, + }, + }, { name: "Empty JSON in default namespace include all", namespace: "default", @@ -68,13 +81,16 @@ var basicCases = []struct { contains: []string{"test.json", "test.yaml"}, }, { - name: "Empty JSON in default namespace include json", + name: "Empty JSON in default namespace include all exclude yaml", namespace: "default", target: "git2kube", labels: map[string]string{}, annotations: map[string]string{}, includes: []*regexp.Regexp{ - regexp.MustCompile(".*\\.json"), + regexp.MustCompile(".*"), + }, + excludes: []*regexp.Regexp{ + regexp.MustCompile(".*\\.yaml"), }, iter: &mockFileIter{ files: []*object.File{ @@ -84,6 +100,18 @@ var basicCases = []struct { }, contains: []string{"test.json"}, }, + { + name: "Empty JSON in default namespace no include", + namespace: "default", + target: "git2kube", + labels: map[string]string{}, + annotations: map[string]string{}, + iter: &mockFileIter{ + files: []*object.File{ + object.NewFile("test.json", filemode.Regular, &object.Blob{}), + }, + }, + }, { name: "No files in config namespace", namespace: "config", @@ -218,7 +246,14 @@ func assertData(data map[string]string, t *testing.T, name string, contains []st } for _, k := range contains { - if _, ok := data[k]; !ok { + if v, ok := data[k]; ok { + content, _ := ioutil.ReadFile(filepath.Join("testdata", k)) + base64content := make([]byte, base64.StdEncoding.EncodedLen(len(content))) + base64.StdEncoding.Encode(base64content, content) + if v != string(content) && v != string(base64content) { + t.Errorf("%s case failed: content mismatch expected '%s' but got '%s(%s)' instead", name, content, base64content, v) + } + } else { t.Errorf("%s case failed: expected data with key '%s' in '%s'", name, k, data) } }