chore: remove unused code around node operations (#118)

* chore: remove unused code around node operations

* chore: set the static cache to be expired after 7 days

shibuya/api/main.go

@@ -241,10 +241,6 @@ func (s *ShibuyaAPI) collectionAdminGetHandler(w http.ResponseWriter, r *http.Re
 	}
 	acr := new(AdminCollectionResponse)
 	acr.RunningCollections = collections
-	if config.SC.ExecutorConfig.Cluster.OnDemand {
-		// we ignore errors here for simplicity
-		acr.NodePools, _ = s.ctr.Scheduler.GetAllNodesInfo()
-	}
 	s.jsonise(w, http.StatusOK, acr)
 }
 
@@ -298,6 +294,7 @@ func (s *ShibuyaAPI) planDeleteHandler(w http.ResponseWriter, r *http.Request, p
 	if err != nil {
 		s.handleErrors(w, err)
 		return
+
 	}
 	if using {
 		s.handleErrors(w, makeInvalidRequestError("plan is being used"))
@@ -432,14 +429,6 @@ func (s *ShibuyaAPI) collectionDeleteHandler(w http.ResponseWriter, r *http.Requ
 		s.handleErrors(w, err)
 		return
 	}
-	if config.SC.ExecutorConfig.Cluster.OnDemand {
-		operator := controller.NewGCPOperator(collection.ID, 0)
-		pool := operator.GetNodePool()
-		if pool != nil {
-			s.handleErrors(w, makeInvalidRequestError("You cannot delete collection when you have nodes launched"))
-			return
-		}
-	}
 	if s.ctr.Scheduler.PodReadyCount(collection.ID) > 0 {
 		s.handleErrors(w, makeInvalidRequestError("You cannot launch engines when there are engines already deployed"))
 		return
@@ -518,10 +507,6 @@ func (s *ShibuyaAPI) collectionUploadHandler(w http.ResponseWriter, r *http.Requ
 		s.handleErrors(w, makeInvalidRequestError(err.Error()))
 		return
 	}
-	if e == nil {
-		s.handleErrors(w, makeInvalidResourceError("YAML file"))
-		return
-	}
 	if e.Content.CollectionID != collection.ID {
 		s.handleErrors(w, makeInvalidRequestError("collection ID mismatch"))
 		return

shibuya/config/init.go

@@ -43,8 +43,6 @@ type ClusterConfig struct {
 	ClusterID   string  `json:"cluster_id"`
 	Kind        string  `json:"kind"`
 	APIEndpoint string  `json:"api_endpoint"`
-	NodeCPUSpec int     `json:"node_cpu_spec"`
-	OnDemand    bool    `json:"on_demand"`
 	GCDuration  float64 `json:"gc_duration"` // in minutes
 	ServiceType string  `json:"service_type"`
 }

shibuya/controller/main.go

@@ -1,7 +1,6 @@
 package controller
 
 import (
-	"math"
 	"net/http"
 	"strconv"
 	"sync"
@@ -182,13 +181,6 @@ func (c *Controller) readConnectedEngines() {
 	}
 }
 
-func (c *Controller) calNodesRequired(enginesNum int) int64 {
-	masterCPU, _ := strconv.ParseFloat(config.SC.ExecutorConfig.JmeterContainer.CPU, 64)
-	enginePerNode := math.Floor(float64(config.SC.ExecutorConfig.Cluster.NodeCPUSpec) / masterCPU)
-	nodesRequired := math.Ceil(float64(enginesNum) / enginePerNode)
-	return int64(nodesRequired)
-}
-
 func (c *Controller) DeployCollection(collection *model.Collection) error {
 	eps, err := collection.GetExecutionPlans()
 	if err != nil {
@@ -201,14 +193,6 @@ func (c *Controller) DeployCollection(collection *model.Collection) error {
 		enginesCount += e.Engines
 		vu += e.Engines * e.Concurrency
 	}
-	if config.SC.ExecutorConfig.Cluster.OnDemand {
-		nodesCount = c.calNodesRequired(enginesCount)
-		operator := NewGCPOperator(collection.ID, nodesCount)
-		err := operator.prepareNodes()
-		if err != nil {
-			return err
-		}
-	}
 	sid := ""
 	if project, err := model.GetProject(collection.ProjectID); err == nil {
 		sid = project.SID
@@ -255,15 +239,6 @@ func (c *Controller) CollectionStatus(collection *model.Collection) (*smodel.Col
 	if err != nil {
 		return nil, err
 	}
-	if config.SC.ExecutorConfig.Cluster.OnDemand {
-		operator := NewGCPOperator(collection.ID, 0)
-		info := operator.GCPNodesInfo()
-		cs.PoolStatus = "LAUNCHED"
-		if info != nil {
-			cs.PoolSize = info.Size
-			cs.PoolStatus = info.Status
-		}
-	}
 	if config.SC.DevMode {
 		cs.PoolSize = 100
 		cs.PoolStatus = "running"

shibuya/install/shibuya/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: v0.1.2
+version: v0.1.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

shibuya/install/shibuya/templates/configmap.yaml

@@ -29,7 +29,6 @@ data:
         },
         "executors": {
             "cluster": {
-                "on_demand": {{ .Values.runtime.executors.cluster.on_demand }},
                 "service_type": {{ .Values.runtime.executors.cluster.service_type | quote }},
                 "gc_duration": {{ .Values.runtime.executors.cluster.gc_duration }}
             },

shibuya/install/shibuya/values.yaml

@@ -67,7 +67,6 @@ runtime:
     keypairs: ""
   executors:
     cluster:
-      on_demand: false
       project: ""
       zone: ""
       cluster_id: ""

shibuya/main.go

@@ -24,6 +24,13 @@ func main() {
 		r.Handle(route.Method, route.Path, route.HandlerFunc)
 	}
 	r.Handler("GET", "/metrics", promhttp.Handler())
-	r.ServeFiles("/static/*filepath", http.Dir("/static"))
+
+	fileServer := http.FileServer(http.Dir("/static"))
+	r.GET("/static/*filepath", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
+		req.URL.Path = ps.ByName("filepath")
+		// Set the cache expiration time to 7 days
+		w.Header().Set("Cache-Control", "public, max-age=604800")
+		fileServer.ServeHTTP(w, req)
+	})
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", 8080), context.ClearHandler(r)))
 }

shibuya/scheduler/cloudrun.go

@@ -339,17 +339,12 @@ func (cr *CloudRun) GetDeployedCollections() (map[int64]time.Time, error) {
 	return deployCollections, nil
 }
 
-func (cr *CloudRun) GetAllNodesInfo() (smodel.AllNodesInfo, error) {
-	// For cloud run, nodes info is not needed
-	return nil, nil
-}
-
 func (cr *CloudRun) GetPodsMetrics(collectionID, planID int64) (map[string]apiv1.ResourceList, error) {
 	// For cloud run, pod metrics is not supported
 	return nil, FeatureUnavailable
 }
 
-//TODO: what we need is actually get the deployed engines account, not only ready ones.
+// TODO: what we need is actually get the deployed engines account, not only ready ones.
 // We also need to change this in k8s.go
 func (cr *CloudRun) PodReadyCount(collectionID int64) int {
 	items, err := cr.getEnginesByCollection(collectionID)

shibuya/scheduler/k8s.go

@@ -108,10 +108,6 @@ func collectionPodAffinity(collectionID int64) *apiv1.PodAffinity {
 
 func prepareAffinity(collectionID int64) *apiv1.Affinity {
 	affinity := &apiv1.Affinity{}
-	if config.SC.ExecutorConfig.Cluster.OnDemand {
-		affinity.NodeAffinity = collectionNodeAffinity(collectionID)
-		return affinity
-	}
 	affinity.PodAffinity = collectionPodAffinity(collectionID)
 	na := config.SC.ExecutorConfig.NodeAffinity
 	if len(na) > 0 {
@@ -900,42 +896,6 @@ func (kcm *K8sClientManager) CreateIngress(ingressClass, ingressName, serviceNam
 	return nil
 }
 
-func (kcm *K8sClientManager) GetNodesByCollection(collectionID string) ([]apiv1.Node, error) {
-	opts := metav1.ListOptions{
-		LabelSelector: fmt.Sprintf("collection_id=%s", collectionID),
-	}
-	return kcm.getNodes(opts)
-}
-
-func (kcm *K8sClientManager) getNodes(opts metav1.ListOptions) ([]apiv1.Node, error) {
-	nodeList, err := kcm.client.CoreV1().Nodes().List(context.TODO(), opts)
-	if err != nil {
-		return nil, err
-	}
-	return nodeList.Items, nil
-}
-
-func (kcm *K8sClientManager) GetAllNodesInfo() (smodel.AllNodesInfo, error) {
-	opts := metav1.ListOptions{}
-	nodes, err := kcm.getNodes(opts)
-	if err != nil {
-		return nil, err
-	}
-	r := make(smodel.AllNodesInfo)
-	for _, node := range nodes {
-		nodeInfo := r[node.ObjectMeta.Labels["collection_id"]]
-		if nodeInfo == nil {
-			nodeInfo = &smodel.NodesInfo{}
-			r[node.ObjectMeta.Labels["collection_id"]] = nodeInfo
-		}
-		nodeInfo.Size++
-		if nodeInfo.LaunchTime.IsZero() || nodeInfo.LaunchTime.After(node.ObjectMeta.CreationTimestamp.Time) {
-			nodeInfo.LaunchTime = node.ObjectMeta.CreationTimestamp.Time
-		}
-	}
-	return r, nil
-}
-
 func (kcm *K8sClientManager) GetDeployedCollections() (map[int64]time.Time, error) {
 	labelSelector := fmt.Sprintf("kind=executor")
 	pods, err := kcm.GetPods(labelSelector, "")

shibuya/scheduler/types.go

@@ -18,7 +18,6 @@ type EngineScheduler interface {
 	FetchEngineUrlsByPlan(collectionID, planID int64, opts *smodel.EngineOwnerRef) ([]string, error)
 	PurgeCollection(collectionID int64) error
 	GetDeployedCollections() (map[int64]time.Time, error)
-	GetAllNodesInfo() (smodel.AllNodesInfo, error)
 	GetPodsMetrics(collectionID, planID int64) (map[string]apiv1.ResourceList, error)
 	PodReadyCount(collectionID int64) int
 	DownloadPodLog(collectionID, planID int64) (string, error)

shibuya/ui/handler.go

@@ -29,7 +29,6 @@ type HomeResp struct {
 	Account               string
 	BackgroundColour      string
 	Context               string
-	OnDemandCluster       bool
 	IsAdmin               bool
 	ResultDashboard       string
 	EnableSid             bool
@@ -56,7 +55,7 @@ func (u *UI) homeHandler(w http.ResponseWriter, r *http.Request, params httprout
 	sc := config.SC
 	gcDuration := config.SC.ExecutorConfig.Cluster.GCDuration
 	template.Execute(w, &HomeResp{account.Name, sc.BackgroundColour, sc.Context,
-		config.SC.ExecutorConfig.Cluster.OnDemand, IsAdmin, resultDashboardURL, enableSid,
+		IsAdmin, resultDashboardURL, enableSid,
 		engineHealthDashboardURL, sc.ProjectHome, sc.UploadFileHelp, gcDuration})
 }