Merge pull request #139 from kthcloud/dev

snapshot support, typo fixes and bug fixes

main.go

@@ -24,6 +24,7 @@ func main() {
 	_ = flag.Bool("job-executor", false, "start job executor")
 	_ = flag.Bool("repairer", false, "start repairer")
 	_ = flag.Bool("pinger", false, "start pinger")
+	_ = flag.Bool("snapshotter", false, "start snapshotter")
 
 	flag.Parse()
 
@@ -33,16 +34,18 @@ func main() {
 	jobExecutor := isFlagPassed("job-executor")
 	repairer := isFlagPassed("repairer")
 	pinger := isFlagPassed("pinger")
+	snapshotter := isFlagPassed("snapshotter")
 
 	var options *app.StartOptions
-	if confirmer || statusUpdater || jobExecutor || repairer || api || pinger {
+	if confirmer || statusUpdater || jobExecutor || repairer || api || pinger || snapshotter {
 		options = &app.StartOptions{
 			API:           api,
 			Confirmer:     confirmer,
 			StatusUpdater: statusUpdater,
 			JobExecutor:   jobExecutor,
 			Repairer:      repairer,
 			Pinger:        pinger,
+			Snapshotter:   snapshotter,
 		}
 
 		log.Println("api: ", options.API)
@@ -51,6 +54,7 @@ func main() {
 		log.Println("job-executor: ", options.JobExecutor)
 		log.Println("repairer: ", options.Repairer)
 		log.Println("pinger: ", options.Pinger)
+		log.Println("snapshotter: ", options.Snapshotter)
 	} else {
 		log.Println("no workers specified, starting all")
 	}

models/dto/body/vm.go

@@ -43,9 +43,10 @@ type VmCreate struct {
 }
 
 type VmUpdate struct {
-	Ports    *[]Port `json:"ports" bson:"ports" binding:"omitempty,port_list_names,port_list_numbers,dive,min=0,max=1000"`
-	CpuCores *int    `json:"cpuCores" binding:"omitempty,min=1"`
-	RAM      *int    `json:"ram" binding:"omitempty,min=1"`
+	SnapshotID *string `json:"snapshotId" binding:"omitempty,uuid4"`
+	Ports      *[]Port `json:"ports" bson:"ports" binding:"omitempty,port_list_names,port_list_numbers,dive,min=0,max=1000"`
+	CpuCores   *int    `json:"cpuCores" binding:"omitempty,min=1"`
+	RAM        *int    `json:"ram" binding:"omitempty,min=1"`
 }
 
 type VmCreated struct {
@@ -88,3 +89,13 @@ type GpuRead struct {
 type VmCommand struct {
 	Command string `json:"command" binding:"required,oneof=start stop reboot"`
 }
+
+type VmSnapshotRead struct {
+	ID         string    `json:"id"`
+	VmID       string    `json:"vmId"`
+	Name       string    `json:"displayname"`
+	ParentName *string   `json:"parentName,omitempty"`
+	CreatedAt  time.Time `json:"created"`
+	State      string    `json:"state"`
+	Current    bool      `json:"current"`
+}

models/sys/job/helpers.go

@@ -25,6 +25,10 @@ func (job *Job) ToDTO(statusMessage string) body.JobRead {
 }
 
 func CreateJob(id, userID, jobType string, args map[string]interface{}) error {
+	return CreateScheduledJob(id, userID, jobType, time.Now(), args)
+}
+
+func CreateScheduledJob(id, userID, jobType string, runAfter time.Time, args map[string]interface{}) error {
 	currentJob, err := GetByID(id)
 	if err != nil {
 		return err
@@ -40,6 +44,7 @@ func CreateJob(id, userID, jobType string, args map[string]interface{}) error {
 		Type:      jobType,
 		Args:      args,
 		CreatedAt: time.Now(),
+		RunAfter:  runAfter,
 		Status:    StatusPending,
 		ErrorLogs: make([]string, 0),
 	}
@@ -52,6 +57,20 @@ func CreateJob(id, userID, jobType string, args map[string]interface{}) error {
 	return nil
 }
 
+func Exists(jobType string, args map[string]interface{}) (bool, error) {
+	filter := bson.D{
+		{"type", jobType},
+		{"args", args},
+		{"status", bson.D{{"$ne", StatusTerminated}}},
+	}
+	count, err := models.JobCollection.CountDocuments(context.TODO(), filter)
+	if err != nil {
+		return false, err
+	}
+
+	return count > 0, nil
+}
+
 func GetByID(id string) (*Job, error) {
 	var job Job
 	err := models.JobCollection.FindOne(context.TODO(), bson.D{{"id", id}}).Decode(&job)
@@ -68,7 +87,11 @@ func GetByID(id string) (*Job, error) {
 }
 
 func GetNext() (*Job, error) {
-	filter := bson.D{{"status", StatusPending}}
+	now := time.Now()
+	filter := bson.D{
+		{"status", StatusPending},
+		{"runAfter", bson.D{{"$lte", now}}},
+	}
 	opts := options.FindOneAndUpdate().SetSort(bson.D{{"createdAt", -1}})
 	update := bson.D{{"$set", bson.D{{"status", StatusRunning}, {"lastRunAt", time.Now()}}}}
 
@@ -86,7 +109,11 @@ func GetNext() (*Job, error) {
 }
 
 func GetNextFailed() (*Job, error) {
-	filter := bson.D{{"status", StatusFailed}}
+	now := time.Now()
+	filter := bson.D{
+		{"status", StatusFailed},
+		{"runAfter", bson.D{{"$lte", now}}},
+	}
 	opts := options.FindOneAndUpdate().SetSort(bson.D{{"createdAt", -1}})
 	update := bson.D{{"$set", bson.D{{"status", StatusRunning}, {"lastRunAt", time.Now()}}}}
 

models/sys/job/models.go

@@ -15,6 +15,8 @@ const (
 	TypeRepairVM         = "repairVm"
 	TypeRepairDeployment = "repairDeployment"
 	TypeRepairGPUs       = "repairGpus"
+	TypeCreateSnapshot   = "createSnapshot"
+	TypeApplySnapshot    = "applySnapshot"
 )
 
 const (
@@ -33,6 +35,7 @@ type Job struct {
 	CreatedAt  time.Time              `bson:"createdAt" json:"createdAt"`
 	LastRunAt  time.Time              `bson:"lastRunAt" json:"lastRunAt"`
 	FinishedAt time.Time              `bson:"finishedAt" json:"finishedAt"`
+	RunAfter   time.Time              `bson:"runAfter" json:"runAfter"`
 	Status     string                 `bson:"status" json:"status"`
 	ErrorLogs  []string               `bson:"errorLogs" json:"errorLogs"`
 }

models/sys/vm/converters.go

@@ -52,6 +52,7 @@ func (vm *VM) ToDTO(status string, connectionString *string, gpu *body.GpuRead,
 }
 
 func (p *UpdateParams) FromDTO(dto *body.VmUpdate) {
+	p.SnapshotID = dto.SnapshotID
 	if dto.Ports != nil {
 		ports := make([]Port, len(*dto.Ports))
 		if dto.Ports != nil {

models/sys/vm/models.go

@@ -2,15 +2,18 @@ package vm
 
 import (
 	csModels "go-deploy/pkg/subsystems/cs/models"
+	"time"
 )
 
 const (
-	ActivityBeingCreated = "beingCreated"
-	ActivityBeingDeleted = "beingDeleted"
-	ActivityBeingUpdated = "beingUpdated"
-	ActivityAttachingGPU = "attachingGpu"
-	ActivityDetachingGPU = "detachingGpu"
-	ActivityRepairing    = "repairing"
+	ActivityBeingCreated     = "beingCreated"
+	ActivityBeingDeleted     = "beingDeleted"
+	ActivityBeingUpdated     = "beingUpdated"
+	ActivityAttachingGPU     = "attachingGpu"
+	ActivityDetachingGPU     = "detachingGpu"
+	ActivityRepairing        = "repairing"
+	ActivityCreatingSnapshot = "creatingSnapshot"
+	ActivityApplyingSnapshot = "applyingSnapshot"
 )
 
 type Port struct {
@@ -27,6 +30,7 @@ type CS struct {
 	ServiceOffering       csModels.ServiceOfferingPublic               `bson:"serviceOffering"`
 	VM                    csModels.VmPublic                            `bson:"vm"`
 	PortForwardingRuleMap map[string]csModels.PortForwardingRulePublic `bson:"portForwardingRuleMap"`
+	SnapshotMap           map[string]csModels.SnapshotPublic           `bson:"snapshotMap"`
 }
 
 type Usage struct {
@@ -45,7 +49,18 @@ type CreateParams struct {
 }
 
 type UpdateParams struct {
-	Ports    *[]Port `json:"ports"`
-	CpuCores *int    `json:"cpuCores"`
-	RAM      *int    `json:"ram"`
+	SnapshotID *string `json:"snapshotId"`
+	Ports      *[]Port `json:"ports"`
+	CpuCores   *int    `json:"cpuCores"`
+	RAM        *int    `json:"ram"`
+}
+
+type Snapshot struct {
+	ID         string    `json:"id"`
+	VmID       string    `json:"vmId"`
+	Name       string    `json:"displayname"`
+	ParentName *string   `json:"parentName,omitempty"`
+	CreatedAt  time.Time `json:"created"`
+	State      string    `json:"state"`
+	Current    bool      `json:"current"`
 }

pkg/app/server.go

@@ -11,9 +11,10 @@ import (
 	"go-deploy/pkg/sys"
 	"go-deploy/pkg/workers/confirm"
 	"go-deploy/pkg/workers/job_execute"
+	"go-deploy/pkg/workers/migrate"
 	"go-deploy/pkg/workers/ping"
-	"go-deploy/pkg/workers/migrator"
 	"go-deploy/pkg/workers/repair"
+	"go-deploy/pkg/workers/snapshot"
 	"go-deploy/pkg/workers/status_update"
 	"go-deploy/routers"
 	"log"
@@ -29,6 +30,7 @@ type StartOptions struct {
 	JobExecutor   bool
 	Repairer      bool
 	Pinger        bool
+	Snapshotter   bool
 }
 
 func shutdown() {
@@ -59,6 +61,7 @@ func Start(options *StartOptions) *http.Server {
 			JobExecutor:   true,
 			Repairer:      true,
 			Pinger:        true,
+			Snapshotter:   true,
 		}
 	}
 
@@ -77,6 +80,9 @@ func Start(options *StartOptions) *http.Server {
 	if options.Pinger {
 		ping.Setup(c)
 	}
+	if options.Snapshotter {
+		snapshot.Setup(c)
+	}
 	if options.API {
 		ginMode, exists := os.LookupEnv("GIN_MODE")
 		if exists {

pkg/status_codes/code.go

@@ -12,13 +12,15 @@ const (
 	ResourceUpdated = 10011
 	ResourceDeleted = 10012
 
-	ResourceNotCreated   = 10020
-	ResourceNotFound     = 10021
-	ResourceNotUpdated   = 10022
-	ResourceNotReady     = 10023
-	ResourceNotAvailable = 10024
-	ResourceBeingCreated = 10025
-	ResourceBeingDeleted = 10026
+	ResourceNotCreated       = 10020
+	ResourceNotFound         = 10021
+	ResourceNotUpdated       = 10022
+	ResourceNotReady         = 10023
+	ResourceNotAvailable     = 10024
+	ResourceBeingCreated     = 10025
+	ResourceBeingDeleted     = 10026
+	ResourceCreatingSnapshot = 10027
+	ResourceApplyingSnapshot = 10028
 
 	ResourceStarting   = 10031
 	ResourceRunning    = 10032

pkg/status_codes/msg.go

@@ -11,13 +11,15 @@ var MsgFlags = map[int]string{
 	ResourceCreated: "resourceCreated",
 	ResourceUpdated: "resourceUpdated",
 
-	ResourceNotCreated:   "resourceNotCreated",
-	ResourceNotFound:     "resourceNotFound",
-	ResourceNotUpdated:   "resourceNotModified",
-	ResourceNotReady:     "resourceNotReady",
-	ResourceNotAvailable: "resourceNotAvailable",
-	ResourceBeingCreated: "resourceBeingCreated",
-	ResourceBeingDeleted: "resourceBeingDeleted",
+	ResourceNotCreated:       "resourceNotCreated",
+	ResourceNotFound:         "resourceNotFound",
+	ResourceNotUpdated:       "resourceNotModified",
+	ResourceNotReady:         "resourceNotReady",
+	ResourceNotAvailable:     "resourceNotAvailable",
+	ResourceBeingCreated:     "resourceBeingCreated",
+	ResourceBeingDeleted:     "resourceBeingDeleted",
+	ResourceCreatingSnapshot: "resourceCreatingSnapshot",
+	ResourceApplyingSnapshot: "resourceApplyingSnapshot",
 
 	ResourceStarting:   "resourceStarting",
 	ResourceRunning:    "resourceRunning",

pkg/subsystems/cs/models/snapshot_public.go

@@ -0,0 +1,38 @@
+package models
+
+import (
+	"go-deploy/pkg/imp/cloudstack"
+	"time"
+)
+
+type SnapshotPublic struct {
+	ID         string    `json:"id"`
+	VmID       string    `json:"vmId"`
+	Name       string    `json:"displayname"`
+	ParentName *string   `json:"parentName"`
+	CreatedAt  time.Time `json:"created"`
+	State      string    `json:"state"`
+	Current    bool      `json:"current"`
+}
+
+func CreateSnapshotPublicFromGet(snapshot *cloudstack.VMSnapshot) *SnapshotPublic {
+	createdAt, err := time.Parse(time.RFC3339, snapshot.Created)
+	if err != nil {
+		createdAt = time.Now()
+	}
+
+	var parentName *string
+	if snapshot.ParentName != "" {
+		parentName = &snapshot.ParentName
+	}
+
+	return &SnapshotPublic{
+		ID:         snapshot.Id,
+		VmID:       snapshot.Virtualmachineid,
+		Name:       snapshot.Displayname,
+		ParentName: parentName,
+		CreatedAt:  createdAt,
+		State:      snapshot.State,
+		Current:    snapshot.Current,
+	}
+}