Merge pull request #168 from liangchenye/tuf

 add snapshot service to implement scan feature

Author: Quanyi Ma

handler/appv1.go

@@ -283,7 +283,9 @@ func AppDeleteFileV1Handler(ctx *macaron.Context) (int, []byte) {
 	return httpRet("AppV1 Delete data", nil, err)
 }
 
-func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) {
+// AppRegistScanHooksV1Handler adds a scan plugin to a user repo
+// TODO: to make it easier as a start, we assume each repo could only have one scan plugin
+func AppRegistScanHooksV1Handler(ctx *macaron.Context) (int, []byte) {
 	data, err := ctx.Req.Body().Bytes()
 	if err != nil {
 		log.Errorf("[%s] Req.Body.Bytes error: %s", ctx.Req.RequestURI, err.Error())
@@ -292,18 +294,22 @@ func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) {
 		return http.StatusBadRequest, result
 	}
 
-	var reg models.ScanHookRegist
-	err = json.Unmarshal(data, &reg)
+	type scanPlugin struct {
+		Name string
+	}
+	var n scanPlugin
+	err = json.Unmarshal(data, &n)
 	if err != nil {
 		log.Errorf("[%s] Invalid body data: %s", ctx.Req.RequestURI, err.Error())
 
 		result, _ := json.Marshal(map[string]string{"Error": "Parse Req.Body.Bytes Error"})
 		return http.StatusBadRequest, result
 	}
 
+	var reg models.ScanHookRegist
 	namespace := ctx.Params(":namespace")
 	repository := ctx.Params(":repository")
-	err = reg.Regist(namespace, repository, reg.ImageName)
+	err = reg.Regist("appv1", namespace, repository, n.Name)
 	if err != nil {
 		log.Errorf("[%s] scan hook regist error: %s", ctx.Req.RequestURI, err.Error())
 
@@ -314,8 +320,8 @@ func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) {
 	return httpRet("AppV1 Scan Hook Regist", nil, err)
 }
 
-// AppCallbackScanHooksHandler gets callback from container and save the scan result.
-func AppCallbackScanHooksHandler(ctx *macaron.Context) (int, []byte) {
+// AppCallbackScanHooksV1Handler gets callback from container and save the scan result.
+func AppCallbackScanHooksV1Handler(ctx *macaron.Context) (int, []byte) {
 	data, err := ctx.Req.Body().Bytes()
 	if err != nil {
 		log.Errorf("[%s] Req.Body.Bytes error: %s", ctx.Req.RequestURI, err.Error())
@@ -336,3 +342,50 @@ func AppCallbackScanHooksHandler(ctx *macaron.Context) (int, []byte) {
 
 	return httpRet("AppV1 Scan Hook Callback", nil, err)
 }
+
+// AppActiveScanHooksTaskV1Handler actives a scan task
+func AppActiveScanHooksTaskV1Handler(ctx *macaron.Context) (int, []byte) {
+	namespace := ctx.Params(":namespace")
+	repository := ctx.Params(":repository")
+
+	var r models.ScanHookRegist
+	rID, err := r.FindID("appv1", namespace, repository)
+	if err != nil {
+		log.Errorf("[%s] scan hook callback error: %s", ctx.Req.RequestURI, err.Error())
+
+		result, _ := json.Marshal(map[string]string{"Error": "Donnot have registed scan plugin"})
+		return http.StatusBadRequest, result
+	}
+
+	a := models.ArtifactV1{
+		OS:   ctx.Params(":os"),
+		Arch: ctx.Params(":arch"),
+		App:  ctx.Params(":app"),
+		Tag:  ctx.Params(":tag"),
+	}
+	a, err = a.Get()
+	if err != nil {
+		log.Errorf("[%s] scan hook callback error: %s", ctx.Req.RequestURI, err.Error())
+
+		result, _ := json.Marshal(map[string]string{"Error": "Cannot find artifactv1"})
+		return http.StatusBadRequest, result
+	}
+
+	// create a task
+	var t models.ScanHookTask
+	tID, err := t.Put(rID, a.Path)
+	if err != nil {
+		log.Errorf("[%s] scan hook callback error: %s", ctx.Req.RequestURI, err.Error())
+
+		result, _ := json.Marshal(map[string]string{"Error": "Fail to create a scan task"})
+		return http.StatusBadRequest, result
+	}
+
+	idBytes, err := utils.TokenMarshal(tID, setting.ScanKey)
+
+	val := struct {
+		TaskID string
+	}{TaskID: string(idBytes)}
+
+	return httpRet("AppV1 Active Scan Hook Task", val, nil)
+}

models/appv1.go

@@ -99,6 +99,12 @@ func (app *AppV1) Delete(artifact ArtifactV1) error {
 	return nil
 }
 
+// Get gets full info by os/arch/app/tag
+func (a *ArtifactV1) Get() (ArtifactV1, error) {
+	// TODO
+	return *a, nil
+}
+
 func (a *ArtifactV1) GetName() string {
 	if ok, _ := a.isValid(); !ok {
 		return ""

models/hook.go

@@ -18,48 +18,66 @@ package models
 
 import (
 	"errors"
+	"strconv"
 	"time"
 
 	"github.com/containerops/dockyard/setting"
+	"github.com/containerops/dockyard/updateservice/snapshot"
 	"github.com/containerops/dockyard/utils"
 )
 
 // ScanHookRegist:
 //   Namespace/Repository contains images/apps/vms to be scaned
-//   ImageName is used to scan the images/apps/vms within a repository
-//   NOTE: no need to record the type of a repository ("docker/v1", "app/v1"),
-//         a user should regist his/her repository with the right ImageName
+//   ScanPluginName is a plugin name of 'snapshot'
 type ScanHookRegist struct {
-	ID         int64  `json:"id" gorm:"primary_key"`
-	Namespace  string `json:"namespace" sql:"not null;type:varchar(255)"`
-	Repository string `json:"repository" sql:"not null;type:varchar(255)"`
-	ImageName  string `json:"ImageName" sql:"not null;type:varchar(255)"`
+	ID             int64  `json:"id" gorm:"primary_key"`
+	Proto          string `json:"proto" sql:"not null;type:varchar(255)"`
+	Namespace      string `json:"namespace" sql:"not null;type:varchar(255)"`
+	Repository     string `json:"repository" sql:"not null;type:varchar(255)"`
+	ScanPluginName string `json:"scanPluginName" sql:"not null;type:varchar(255)"`
 }
 
 // Regist regists a repository with a scan image
-//   A namespace/repository could have multiple ImageNames,
-//   but should not have multiple records with same Namespace&Repository&ImageName.
-func (s *ScanHookRegist) Regist(n, r, image string) error {
-	if n == "" || r == "" || image == "" {
-		return errors.New("'Namespace', 'Repository' and 'ImageName' should not be empty")
+//   A namespace/repository could have multiple ScanPluginName,
+//   but now we only support one.
+func (s *ScanHookRegist) Regist(p, n, r, name string) error {
+	if p == "" || n == "" || r == "" || name == "" {
+		return errors.New("'Proto', 'Namespace', 'Repository' and 'ScanPluginName' should not be empty")
 	}
-	s.Namespace, s.Repository, s.ImageName = n, r, image
+
+	if ok, err := snapshot.IsSnapshotSupported(p, name); !ok {
+		return err
+	}
+
+	s.Proto, s.Namespace, s.Repository, s.ScanPluginName = p, n, r, name
 	//TODO: add to db
 	return nil
 }
 
 // UnRegist unregists a repository with a scan image
 //   if ImageName is nil, unregist all the scan images.
-func (s *ScanHookRegist) UnRegist(n, r string) error {
-	if n == "" || r == "" {
-		return errors.New("'Namespace', 'Repository' should not be empty")
+func (s *ScanHookRegist) UnRegist(p, n, r string) error {
+	if p == "" || n == "" || r == "" {
+		return errors.New("'Proto', 'Namespace', 'Repository' should not be empty")
 	}
-	s.Namespace, s.Repository = n, r
+	s.Proto, s.Namespace, s.Repository = p, n, r
 
 	//TODO: remove from db
 	return nil
 }
 
+// FindByID finds content by id
+func (s *ScanHookRegist) FindByID(id int64) (ScanHookRegist, error) {
+	//TODO: query db
+	return *s, nil
+}
+
+// FindID finds id by Proto, Namespace and Repository
+func (s *ScanHookRegist) FindID(p, n, r string) (int64, error) {
+	//TODO: query db
+	return 0, nil
+}
+
 // ListScanHooks returns a list of registed scan hooks of a repository
 func (s *ScanHookRegist) List(n, r string) ([]ScanHookRegist, error) {
 	if n == "" || r == "" {
@@ -70,7 +88,8 @@ func (s *ScanHookRegist) List(n, r string) ([]ScanHookRegist, error) {
 
 // ScanHookTask is the scan task
 type ScanHookTask struct {
-	ID       int64  `json:"id" gorm:"primary_key"`
+	ID int64 `json:"id" gorm:"primary_key"`
+	//Path is image url now
 	Path     string `json:"path" sql:"not null;type:varchar(255)"`
 	Callback string `json:"callback" sql:"not null;type:varchar(255)"`
 	// ID of ScanHookRegist
@@ -82,12 +101,31 @@ type ScanHookTask struct {
 	UpdatedAt time.Time `json:"update_at" sql:""`
 }
 
-func (t *ScanHookTask) Put(p, c string, rID int64) error {
-	if p == "" || c == "" || rID == 0 {
-		return errors.New("'Namespace', 'Repository' and RegistID should not be empty")
+// Put returns task id
+func (t *ScanHookTask) Put(rID int64, url string) (int64, error) {
+	if url == "" || rID == 0 {
+		return 0, errors.New("'URL' and 'RegistID' should not be empty")
 	}
+	//TODO: add to db and get task ID
 
-	return nil
+	var reg ScanHookRegist
+	reg, err := reg.FindByID(rID)
+	if err != nil {
+		return 0, err
+	}
+
+	// Do the real scan work
+	s, err := snapshot.NewUpdateServiceSnapshot(reg.ScanPluginName, strconv.FormatInt(rID, 10), url, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	err = s.Process()
+	if err != nil {
+		return 0, err
+	}
+
+	return 0, nil
 }
 
 func (t *ScanHookTask) Update(status string) error {
@@ -97,14 +135,14 @@ func (t *ScanHookTask) Update(status string) error {
 
 func (t *ScanHookTask) Find(encodedCallbackID string) error {
 	//TODO: update status and updatedAt
-	var id int
+	var id int64
 	err := utils.TokenUnmarshal(encodedCallbackID, setting.ScanKey, &id)
 
 	return err
 }
 
 func (t *ScanHookTask) UpdateResult(encodedCallbackID string, data []byte) error {
-	var id int
+	var id int64
 	err := utils.TokenUnmarshal(encodedCallbackID, setting.ScanKey, &id)
 	if err != nil {
 		return err

router/router.go

@@ -140,8 +140,9 @@ func SetRouters(m *macaron.Macaron) {
 				m.Delete("/:os/:arch/:type/:app/?:tag", handler.AppDeleteFileV1Handler)
 
 				//Content Scan
-				m.Post("/shook", handler.AppRegistScanHooksHandler)
-				m.Post("/shook/:callbackID", handler.AppCallbackScanHooksHandler)
+				m.Post("/shook", handler.AppRegistScanHooksV1Handler)
+				m.Post("/shook/:callbackID", handler.AppCallbackScanHooksV1Handler)
+				m.Post("/:os/:arch/:app/shook/?:tag", handler.AppActiveScanHooksTaskV1Handler)
 			})
 		})
 	})

tests/unit/testdata/testmd5

@@ -0,0 +1 @@
+this is test md5.

tests/unit/updateservice_snapshot_appv1_test.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2016 The ContainerOps Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package unittest
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/containerops/dockyard/updateservice/snapshot"
+)
+
+func TestAppv1New(t *testing.T) {
+	cases := []struct {
+		id       string
+		url      string
+		expected bool
+	}{
+		{"id", "url", true},
+		{"", "url", false},
+		{"id", "", false},
+	}
+
+	var appv1 snapshot.UpdateServiceSnapshotAppv1
+	for _, c := range cases {
+		_, err := appv1.New(c.id, c.url, nil)
+		assert.Equal(t, c.expected, err == nil, "Fail to create new snapshot appv1")
+	}
+}
+
+func TestAppv1Supported(t *testing.T) {
+	cases := []struct {
+		proto    string
+		expected bool
+	}{
+		{"appv1", true},
+		{"invalid", false},
+	}
+
+	var appv1 snapshot.UpdateServiceSnapshotAppv1
+	for _, c := range cases {
+		assert.Equal(t, c.expected, appv1.Supported(c.proto), "Fail to get supported status")
+	}
+}
+
+var (
+	cbMap = make(map[string]snapshot.UpdateServiceSnapshotOutput)
+)
+
+func testCB(id string, data snapshot.UpdateServiceSnapshotOutput) error {
+	if id != "1" && id != "2" {
+		return errors.New("invalid id")
+	}
+
+	cbMap[id] = data
+	return nil
+}
+
+func TestAppv1Process(t *testing.T) {
+	for n, _ := range cbMap {
+		delete(cbMap, n)
+	}
+
+	cases := []struct {
+		id         string
+		url        string
+		cb         snapshot.Callback
+		pExpected  bool
+		idExpected bool
+		md5        string
+	}{
+		{"1", "testmd5", testCB, true, true, "ffe7c736f2aa54531ac6430e3cbf2545"},
+		{"2", "invalid", testCB, true, true, ""},
+		{"3", "testmd5", testCB, false, false, ""},
+		{"4", "testmd5", nil, true, false, ""},
+	}
+
+	var appv1 snapshot.UpdateServiceSnapshotAppv1
+	_, path, _, _ := runtime.Caller(0)
+	dir := filepath.Join(filepath.Dir(path), "testdata")
+
+	for _, c := range cases {
+		a, _ := appv1.New(c.id, filepath.Join(dir, c.url), c.cb)
+		err := a.Process()
+		assert.Equal(t, c.pExpected, err == nil, "Fail to get correct process output")
+
+		data, ok := cbMap[c.id]
+		assert.Equal(t, c.idExpected, ok, "Fail to call cb")
+		if ok {
+			assert.Equal(t, c.md5, fmt.Sprintf("%x", data.Data), "Fail to call cb md5")
+		}
+	}
+}