Setup DB and Add Pagination to Conform to OGC Specs

.github/workflows/e2e-tests.yml

@@ -23,7 +23,6 @@ jobs:
           echo S3_BUCKET=${{ secrets.S3_BUCKET }} >> .env
           echo S3_RESULTS_DIR='tests/results' >> .env
           echo S3_META_DIR='tests/metadata' >> .env
-          echo S3_LOGS_DIR='tests/logs' >> .env
           echo BATCH_LOG_STREAM_GROUP='/aws/batch/job' >> .env
           echo CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE='YES' >> .env
           echo EXPIRY_DAYS='1' >> .env

README.md

@@ -21,8 +21,6 @@ https://developer.ogc.org/api/processes/index.html
 3. Update swagger documents and compile the server: `swag init && go build main.go`.
 4. Run the server: `./main`, with the following available flags:
    ```
-      `-c [type int] specify the path of the max cache size for storing jobs (default 11073741824 (1GB))`
-      `-o [type bool] specify if cache should be overwritten`
       `-d [type string] specify the path of the processes directory to load (default "plugins" assuming program called from inside repo)`
       `-e [type string] specify the path of the dot env file to load (default ".env")`
       `-p [type string] specify the port to run the api on (default "5050")`
@@ -68,8 +66,6 @@ When a job is submitted, a local container is fired up immediately for sync jobs
 
 The API responds to all GET requests (except `/jobs/<jobID>/results`) as HTML or JSON depending upon if the request is being originated from Browser or not or if it specifies the format using query parameter ‘f’.
 
-The app maintains a cache of all jobs submitted, a snapshot of this cache is saved to disk at `./.data/snapshot.gob` every 60 minutes and at the graceful shutdown of the server. If during the restart the snapshot is found at the above location, the cache is repopulated with the snapshot data.
-
 ## Example .env file
 
 For AWS services, an env file should be located at the root of this repository (`./.env`) and be formatted like so:
@@ -83,7 +79,7 @@ AWS_DEFAULT_REGION='us-east-1'
 # S3
 S3_BUCKET='********'
 S3_RESULTS_DIR='results'
-S3_LOGS_DIR='logs'
+S3_META_DIR='metadata'
 
 # BATCH
 BATCH_LOG_STREAM_GROUP='/aws/batch/job'

go.mod

@@ -28,12 +28,13 @@ require (
 	github.com/go-openapi/swag v0.19.15 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect

go.sum

@@ -41,8 +41,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -80,6 +80,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
 github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=

handlers/config.go

@@ -4,12 +4,12 @@ import (
 	"app/jobs"
 	pr "app/processes"
 	"io"
+	"log"
 	"text/template"
 
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/labstack/echo/v4"
-	"github.com/labstack/gommon/log"
 )
 
 // Store for templates and a receiver function to render them
@@ -29,13 +29,15 @@ type RESTHandler struct {
 	ConformsTo  []string
 	T           Template
 	S3Svc       *s3.S3
-	JobsCache   *jobs.JobsCache
+	DB          *jobs.DB
+	ActiveJobs  *jobs.ActiveJobs
 	ProcessList *pr.ProcessList
 }
 
 // Initializes resources and return a new handler
-func NewRESTHander(pluginsDir string, cacheSize uint64, o bool) (*RESTHandler, error) {
-	// working with pointers here so as not to copy large templates, yamls, and job cache
+// errors are fatal
+func NewRESTHander(pluginsDir string) *RESTHandler {
+	// working with pointers here so as not to copy large templates, yamls, and ActiveJobs
 	config := RESTHandler{
 		Title:       "process-api",
 		Description: "ogc process api written in Golang for use with cloud service controllers to manage asynchronous requests",
@@ -59,34 +61,19 @@ func NewRESTHander(pluginsDir string, cacheSize uint64, o bool) (*RESTHandler, e
 	}))
 	config.S3Svc = s3.New(sess)
 
-	// Setup Job Cache that will store all jobs
-	jc := jobs.JobsCache{MaxSizeBytes: uint64(cacheSize),
-		CurrentSizeBytes: 0, TrimThreshold: 0.80}
-	jc.Jobs = make(map[string]*jobs.Job)
-	if !o {
-		// load from previous snapshot if it exist
-		err := jc.LoadCacheFromFile()
-		if err != nil {
-			if err.Error() == "not found" {
-				log.Info("Starting with clean Jobs cache.")
-			} else {
-				// if file exists and there is an error in loading, we do not want to start from clean cache
-				// as this will overwrite the old file when the server is shutdown
-				log.Fatalf("Error loading snapshot: %s\nSet -o flag to true to overwrite snapshot", err.Error())
-			}
-		} else {
-			log.Info("Starting from snapshot saved at .data/snapshot.gob")
-		}
-	} else {
-		log.Info("Starting with clean Jobs cache.")
-	}
-	config.JobsCache = &jc
+	// Setup Active Jobs that will store all jobs currently in process
+	ac := jobs.ActiveJobs{}
+	ac.Jobs = make(map[string]*jobs.Job)
+	config.ActiveJobs = &ac
+
+	adb := jobs.InitDB(".data/db.sqlite")
+	config.DB = adb
 
 	processList, err := pr.LoadProcesses(pluginsDir)
 	if err != nil {
-		return nil, err
+		log.Fatal(err)
 	}
 	config.ProcessList = &processList
 
-	return &config, nil
+	return &config
 }