Skip to content

Commit

Permalink
Started migrating according to Shuffle/Shuffle#324
Browse files Browse the repository at this point in the history
  • Loading branch information
@frikky
frikky committed Apr 3, 2021
1 parent 5bd697e commit 6a623b5
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 45 deletions.
63 changes: 54 additions & 9 deletions db-connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,13 @@ func GetWorkflow(ctx context.Context, id string) (*Workflow, error) {

key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
if err := project.Dbclient.Get(ctx, key, workflow); err != nil {
return &Workflow{}, err

if strings.Contains(err.Error(), `cannot load field`) {
log.Printf("[INFO] Error in workflow loading. Migrating workflow to new workflow handler.")
err = nil
} else {
return &Workflow{}, err
}
}

if project.CacheDb {
Expand Down Expand Up @@ -495,9 +501,12 @@ func GetAllWorkflows(ctx context.Context, orgId string) ([]Workflow, error) {

// ListBooks returns a list of books, ordered by title.
func GetOrg(ctx context.Context, id string) (*Org, error) {
nameKey := "Organizations"
cacheKey := fmt.Sprintf("%s_%s", nameKey, id)

curOrg := &Org{}
if project.CacheDb {
cache, err := GetCache(ctx, id)
cache, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(cache.([]uint8))
//log.Printf("CACHEDATA: %#v", cacheData)
Expand All @@ -510,9 +519,28 @@ func GetOrg(ctx context.Context, id string) (*Org, error) {
}
}

key := datastore.NameKey("Organizations", id, nil)
key := datastore.NameKey(nameKey, id, nil)
if err := project.Dbclient.Get(ctx, key, curOrg); err != nil {
return &Org{}, err
if strings.Contains(err.Error(), `cannot load field`) {
log.Printf("[INFO] Error in org loading. Migrating org to new org and user handler.")
err = nil

//for _, user := range curOrg.Users {
// log.Printf("USER: %#v", user)
//}
//SetOrg(ctx, *curOrg, curOrg.Id)

//curUser.ActiveOrg = OrgMini{
// Name: curUser.ActiveOrg.Name,
// Id: curUser.ActiveOrg.Id,
// Role: "user",
//}

// Updating the user and their org
//SetUser(ctx, curUser)
} else {
return &Org{}, err
}
}

newUsers := []User{}
Expand All @@ -535,7 +563,7 @@ func GetOrg(ctx context.Context, id string) (*Org, error) {
return curOrg, nil
}

err = SetCache(ctx, id, neworg)
err = SetCache(ctx, cacheKey, neworg)
if err != nil {
log.Printf("[WARNING] Failed updating cache: %s", err)
}
Expand All @@ -545,6 +573,7 @@ func GetOrg(ctx context.Context, id string) (*Org, error) {
}

func SetOrg(ctx context.Context, data Org, id string) error {
nameKey := "Organizations"
timeNow := int64(time.Now().Unix())
if data.Created == 0 {
data.Created = timeNow
Expand All @@ -553,13 +582,14 @@ func SetOrg(ctx context.Context, data Org, id string) error {
data.Edited = timeNow

// clear session_token and API_token for user
k := datastore.NameKey("Organizations", id, nil)
k := datastore.NameKey(nameKey, id, nil)
if _, err := project.Dbclient.Put(ctx, k, &data); err != nil {
log.Println(err)
return err
}

if project.CacheDb {

newUsers := []User{}
for _, user := range data.Users {
user.Password = ""
Expand All @@ -580,7 +610,8 @@ func SetOrg(ctx context.Context, data Org, id string) error {
return nil
}

err = SetCache(ctx, id, neworg)
cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
err = SetCache(ctx, cacheKey, neworg)
if err != nil {
log.Printf("[WARNING] Failed setting cache for org: %s", err)
}
Expand Down Expand Up @@ -733,7 +764,21 @@ func GetUser(ctx context.Context, username string) (*User, error) {

key := datastore.NameKey("Users", parsedKey, nil)
if err := project.Dbclient.Get(ctx, key, curUser); err != nil {
return &User{}, err
// Handles migration of the user
if strings.Contains(err.Error(), `cannot load field`) {
log.Printf("[INFO] Error in user. Migrating to new org and user handler.")
curUser.ActiveOrg = OrgMini{
Name: curUser.ActiveOrg.Name,
Id: curUser.ActiveOrg.Id,
Role: "user",
}

// Updating the user and their org
SetUser(ctx, curUser)
} else {
log.Printf("[WARNING] Error in Get User: %s", err)
return &User{}, err
}
}

if project.CacheDb {
Expand Down Expand Up @@ -828,7 +873,7 @@ func fixUserOrg(ctx context.Context, user *User) *User {

org, err := GetOrg(ctx, orgId)
if err != nil {
log.Printf("Error getting org %s", orgId)
log.Printf("[WARNING] Error getting org %s in fixUserOrg: %s", orgId, err)
continue
}

Expand Down
39 changes: 19 additions & 20 deletions shared.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ func GetSpecificApps(resp http.ResponseWriter, request *http.Request) {
var tmpBody tmpStruct
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("Error with unmarshal tmpBody: %s", err)
log.Printf("[WARNING] Error with unmarshal tmpBody specific apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
Expand All @@ -324,7 +324,7 @@ func GetSpecificApps(resp http.ResponseWriter, request *http.Request) {
ctx := getContext(request)
workflowapps, err := GetPrioritizedApps(ctx, user)
if err != nil {
log.Printf("Error: Failed getting workflowapps: %s", err)
log.Printf("[WARNING] Error: Failed getting workflowapps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
Expand Down Expand Up @@ -815,8 +815,6 @@ func HandleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U
return User{}, errors.New("Couldn't find the user")
}

log.Printf("USER: %#v", userdata)

// Caching both bad and good apikeys :)
b, err := json.Marshal(userdata)
if err != nil {
Expand Down Expand Up @@ -1265,11 +1263,11 @@ func GetWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
var workflowExecutions []WorkflowExecution
_, err = project.Dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
if err != nil && len(workflowExecutions) == 0 {
if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(15)
_, err = project.Dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
if err != nil && len(workflowExecutions) == 0 {
log.Printf("[WARNING] Error getting workflowexec (2): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId)))
Expand All @@ -1280,14 +1278,14 @@ func GetWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {

q = datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Limit(25)
_, err = project.Dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
if err != nil && len(workflowExecutions) == 0 {
log.Printf("[WARNING] Error getting workflowexec (3): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId)))
return
}
} else {
log.Printf("[WARNING] Error getting workflowexec: %s", err)
log.Printf("[WARNING] Error getting workflowexec (4): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions for %s"}`, fileId)))
return
Expand Down Expand Up @@ -1353,12 +1351,12 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) {
q = q.Order("-edited")

_, err = project.Dbclient.GetAll(ctx, q, &workflows)
if err != nil {
if err != nil && len(workflows) == 0 {
if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
q = q.Limit(36)
_, err = project.Dbclient.GetAll(ctx, q, &workflows)
if err != nil {
log.Printf("Failed getting workflows for user %s: %s (0)", user.Username, err)
if err != nil && len(workflows) == 0 {
log.Printf("[WARNING] Failed getting workflows for user %s (0): %s", user.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
Expand All @@ -1373,15 +1371,15 @@ func GetWorkflows(resp http.ResponseWriter, request *http.Request) {

q.Limit(30)
_, err = project.Dbclient.GetAll(ctx, q, &workflows)
if err != nil {
log.Printf("Failed getting workflows for user %s: %s (0)", user.Username, err)
if err != nil && len(workflows) == 0 {
log.Printf("[WARNING] Failed getting workflows for user %s (2): %s", user.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}

} else {
log.Printf("Failed getting workflows for user %s: %s (1)", user.Username, err)
log.Printf("[WARNING] Failed getting workflows for user %s (1): %s ", user.Username, err)
//DeleteKey(ctx, "workflow", "5694357e-8063-4580-8529-301cc72df951")

//log.Printf("Workflows: %#v", workflows)
Expand Down Expand Up @@ -1917,7 +1915,7 @@ func SetNewWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.ID = uuid.NewV4().String()
workflow.Owner = user.Id
workflow.Sharing = "private"
user.ActiveOrg.Users = []User{}
user.ActiveOrg.Users = []UserMini{}
workflow.ExecutingOrg = user.ActiveOrg
workflow.OrgId = user.ActiveOrg.Id
//log.Printf("TRIGGERS: %d", len(workflow.Triggers))
Expand Down Expand Up @@ -2164,7 +2162,7 @@ func SaveWorkflow(resp http.ResponseWriter, request *http.Request) {
ctx := getContext(request)
tmpworkflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("Failed getting the workflow locally (save workflow): %s", err)
log.Printf("[WARNING] Failed getting the workflow locally (save workflow): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
Expand All @@ -2182,7 +2180,7 @@ func SaveWorkflow(resp http.ResponseWriter, request *http.Request) {
workflow.ID = uuid.NewV4().String()
workflow.Public = false
workflow.Owner = user.Id
workflow.Org = []Org{
workflow.Org = []OrgMini{
user.ActiveOrg,
}
workflow.ExecutingOrg = user.ActiveOrg
Expand Down Expand Up @@ -2249,7 +2247,7 @@ func SaveWorkflow(resp http.ResponseWriter, request *http.Request) {

if len(workflow.ExecutingOrg.Id) == 0 {
log.Printf("[INFO] Setting executing org for workflow")
user.ActiveOrg.Users = []User{}
user.ActiveOrg.Users = []UserMini{}
workflow.ExecutingOrg = user.ActiveOrg
}

Expand Down Expand Up @@ -3180,6 +3178,7 @@ func HandleGetUsers(resp http.ResponseWriter, request *http.Request) {
ctx := getContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org in get users: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`))
return
Expand Down Expand Up @@ -4529,9 +4528,9 @@ func SanitizeWorkflow(workflow Workflow) Workflow {
}

workflow.Owner = ""
workflow.Org = []Org{}
workflow.Org = []OrgMini{}
workflow.OrgId = ""
workflow.ExecutingOrg = Org{}
workflow.ExecutingOrg = OrgMini{}
workflow.PreviouslySaved = false

// Add Gitguardian or similar secrets discovery
Expand Down
47 changes: 31 additions & 16 deletions structs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,13 @@ type UserAuthField struct {
Value string `json:"value" datastore:"value"`
}

// Used to contain users in miniOrg
type UserMini struct {
Username string `datastore:"Username" json:"username"`
Id string `datastore:"id" json:"id"`
Role string `datastore:"role" json:"role"`
}

type User struct {
Username string `datastore:"Username" json:"username"`
Password string `datastore:"password,noindex" password:"password,omitempty"`
Expand All @@ -250,7 +257,7 @@ type User struct {
Id string `datastore:"id" json:"id"`
Orgs []string `datastore:"orgs" json:"orgs"`
CreationTime int64 `datastore:"creation_time" json:"creation_time"`
ActiveOrg Org `json:"active_org" datastore:"active_org"`
ActiveOrg OrgMini `json:"active_org" datastore:"active_org"`
Active bool `datastore:"active" json:"active"`
}

Expand Down Expand Up @@ -448,6 +455,14 @@ type Hook struct {
Environment string `json:"environment" datastore:"environment"`
}

// Used within a user
type OrgMini struct {
Name string `json:"name" datastore:"name"`
Id string `json:"id" datastore:"id"`
Users []UserMini `json:"users" datastore:"users"`
Role string `json:"role" datastore:"role"`
}

type Org struct {
Name string `json:"name" datastore:"name"`
Description string `json:"description" datastore:"description"`
Expand Down Expand Up @@ -668,21 +683,21 @@ type Workflow struct {
ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"`
StartFromTop bool `json:"start_from_top" datastore:"start_from_top"`
} `json:"configuration,omitempty" datastore:"configuration"`
Created int64 `json:"created" datastore:"created"`
Edited int64 `json:"edited" datastore:"edited"`
LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"`
Errors []string `json:"errors,omitempty" datastore:"errors"`
Tags []string `json:"tags,omitempty" datastore:"tags"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
Name string `json:"name" datastore:"name"`
Description string `json:"description" datastore:"description,noindex"`
Start string `json:"start" datastore:"start"`
Owner string `json:"owner" datastore:"owner"`
Sharing string `json:"sharing" datastore:"sharing"`
Org []Org `json:"org,omitempty" datastore:"org"`
ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"`
OrgId string `json:"org_id,omitempty" datastore:"org_id"`
Created int64 `json:"created" datastore:"created"`
Edited int64 `json:"edited" datastore:"edited"`
LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"`
Errors []string `json:"errors,omitempty" datastore:"errors"`
Tags []string `json:"tags,omitempty" datastore:"tags"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
Name string `json:"name" datastore:"name"`
Description string `json:"description" datastore:"description,noindex"`
Start string `json:"start" datastore:"start"`
Owner string `json:"owner" datastore:"owner"`
Sharing string `json:"sharing" datastore:"sharing"`
Org []OrgMini `json:"org,omitempty" datastore:"org"`
ExecutingOrg OrgMini `json:"execution_org,omitempty" datastore:"execution_org"`
OrgId string `json:"org_id,omitempty" datastore:"org_id"`
WorkflowVariables []struct {
Description string `json:"description" datastore:"description,noindex"`
ID string `json:"id" datastore:"id"`
Expand Down

0 comments on commit 6a623b5

Please sign in to comment.