Update docs

.github/workflows/e2e-tests.yml

@@ -21,9 +21,9 @@ jobs:
           echo API_NAME='github-testing-process-api' >> .env
           echo STORAGE_SERVICE='minio' >> .env
           echo STORAGE_BUCKET='api-storage' >> .env
-          echo STORAGE_METADATA_DIR='metadata' >> .env
-          echo STORAGE_RESULTS_DIR='results' >> .env
-          echo STORAGE_LOGS_DIR='logs' >> .env
+          echo STORAGE_METADATA_PREFIX='metadata' >> .env
+          echo STORAGE_RESULTS_PREFIX='results' >> .env
+          echo STORAGE_LOGS_PREFIX='logs' >> .env
 
           echo LOCAL_LOGS_DIR='../.data/logs' >> .env
 
@@ -33,6 +33,8 @@ jobs:
           echo MINIO_S3_REGION='us-east-1' >> .env
           echo MINIO_S3_DISABLE_SSL=true >> .env
           echo MINIO_S3_FORCE_PATH_STYLE=true >> .env
+          echo MINIO_ROOT_USER=user >> .env
+          echo MINIO_ROOT_PASSWORD=password >> .env
 
           echo AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} >> .env
           echo AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} >> .env

.gitignore

@@ -20,3 +20,6 @@
 .data/*
 *main
 settings.json
+
+
+.miniodata/

api/docs/swagger.json

@@ -266,7 +266,7 @@
             "post": {
                 "description": "[Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job)",
                 "consumes": [
-                    "*/*"
+                    "application/json"
                 ],
                 "produces": [
                     "application/json"
@@ -284,8 +284,8 @@
                         "required": true
                     },
                     {
-                        "description": "example: {“text”:“Hello World!”}",
-                        "name": "input",
+                        "description": "example: {inputs: {text:Hello World!}} (add double quotes for all strings in the payload)",
+                        "name": "inputs",
                         "in": "body",
                         "required": true,
                         "schema": {
@@ -333,23 +333,23 @@
         "jobs.JobLogs": {
             "type": "object",
             "properties": {
-                "api_logs": {
-                    "type": "array",
-                    "items": {
-                        "type": "string"
-                    }
-                },
                 "container_logs": {
                     "type": "array",
                     "items": {
-                        "type": "string"
+                        "$ref": "#/definitions/jobs.LogEntry"
                     }
                 },
                 "jobID": {
                     "type": "string"
                 },
                 "processID": {
                     "type": "string"
+                },
+                "server_logs": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/jobs.LogEntry"
+                    }
                 }
             }
         },
@@ -380,6 +380,20 @@
                 }
             }
         },
+        "jobs.LogEntry": {
+            "type": "object",
+            "properties": {
+                "level": {
+                    "type": "string"
+                },
+                "msg": {
+                    "type": "string"
+                },
+                "time": {
+                    "type": "string"
+                }
+            }
+        },
         "processes.Info": {
             "type": "object",
             "properties": {
@@ -557,5 +571,9 @@
                 }
             }
         }
+    },
+    "externalDocs": {
+        "description": "Schemas",
+        "url": "http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/"
     }
 }

api/docs/swagger.yaml

@@ -19,18 +19,18 @@ definitions:
     type: object
   jobs.JobLogs:
     properties:
-      api_logs:
-        items:
-          type: string
-        type: array
       container_logs:
         items:
-          type: string
+          $ref: '#/definitions/jobs.LogEntry'
         type: array
       jobID:
         type: string
       processID:
         type: string
+      server_logs:
+        items:
+          $ref: '#/definitions/jobs.LogEntry'
+        type: array
     type: object
   jobs.JobRecord:
     properties:
@@ -50,6 +50,15 @@ definitions:
       updated:
         type: string
     type: object
+  jobs.LogEntry:
+    properties:
+      level:
+        type: string
+      msg:
+        type: string
+      time:
+        type: string
+    type: object
   processes.Info:
     properties:
       description:
@@ -165,6 +174,9 @@ definitions:
           $ref: '#/definitions/processes.Outputs'
         type: array
     type: object
+externalDocs:
+  description: Schemas
+  url: http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/
 host: localhost:5050
 info:
   contact:
@@ -340,17 +352,18 @@ paths:
   /processes/{processID}/execution:
     post:
       consumes:
-      - '*/*'
+      - application/json
       description: '[Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job)'
       parameters:
       - description: pyecho
         in: path
         name: processID
         required: true
         type: string
-      - description: 'example: {“text”:“Hello World!”}'
+      - description: 'example: {inputs: {text:Hello World!}} (add double quotes for
+          all strings in the payload)'
         in: body
-        name: input
+        name: inputs
         required: true
         schema:
           type: string

api/handlers/handlers.go

@@ -241,10 +241,10 @@ func (rh *RESTHandler) ProcessDescribeHandler(c echo.Context) error {
 // @Summary Execute Process
 // @Description [Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job)
 // @Tags processes
-// @Accept */*
+// @Accept json
 // @Produce json
 // @Param processID path string true "pyecho"
-// @Param input body string true "example: {“text”:“Hello World!”}"
+// @Param inputs body string true "example: {inputs: {text:Hello World!}} (add double quotes for all strings in the payload)"
 // @Success 200 {object} jobResponse
 // @Router /processes/{processID}/execution [post]
 // Does not produce HTML

api/jobs/aws_batch_jobs.go

@@ -405,7 +405,7 @@ func (j *AWSBatchJob) WriteMetaData() {
 		return
 	}
 
-	metadataDir := os.Getenv("STORAGE_METADATA_DIR")
+	metadataDir := os.Getenv("STORAGE_METADATA_PREFIX")
 	mdLocation := fmt.Sprintf("%s/%s.json", metadataDir, j.UUID)
 	// TODO: Determine if batch metadata should be put on aws...currently this is the case
 	utils.WriteToS3(j.StorageSvc, jsonBytes, mdLocation, "application/json", 0)
@@ -415,7 +415,7 @@ func (j *AWSBatchJob) WriteMetaData() {
 // 	j.logger.Info("Starting results writing routine.")
 // 	defer j.logger.Info("Finished results writing routine.")
 
-// 	resultsDir := os.Getenv("STORAGE_RESULTS_DIR")
+// 	resultsDir := os.Getenv("STORAGE_RESULTS_PREFIX")
 // 	resultsLocation := fmt.Sprintf("%s/%s.json", resultsDir, j.UUID)
 // 	err = utils.WriteToS3(j.StorageSvc, data, resultsLocation, "application/json", 0)
 // 	if err != nil {

api/jobs/docker_jobs.go

@@ -358,7 +358,7 @@ func (j *DockerJob) WriteMetaData() {
 		return
 	}
 
-	metadataDir := os.Getenv("STORAGE_METADATA_DIR")
+	metadataDir := os.Getenv("STORAGE_METADATA_PREFIX")
 	mdLocation := fmt.Sprintf("%s/%s.json", metadataDir, j.UUID)
 	err = utils.WriteToS3(j.StorageSvc, jsonBytes, mdLocation, "application/json", 0)
 	if err != nil {
@@ -370,7 +370,7 @@ func (j *DockerJob) WriteMetaData() {
 // 	j.logger.Info("Starting results writing routine.")
 // 	defer j.logger.Info("Finished results writing routine.")
 
-// 	resultsDir := os.Getenv("STORAGE_RESULTS_DIR")
+// 	resultsDir := os.Getenv("STORAGE_RESULTS_PREFIX")
 // 	resultsLocation := fmt.Sprintf("%s/%s.json", resultsDir, j.UUID)
 // 	fmt.Println(resultsLocation)
 // 	err = utils.WriteToS3(j.StorageSvc, data, resultsLocation, "application/json", 0)

api/jobs/jobs.go

@@ -192,7 +192,7 @@ func FetchResults(svc *s3.S3, jid string) (interface{}, error) {
 // // If JobID exists but results file doesn't then it raises an error
 // // Assumes jobID is valid
 // func FetchResults(svc *s3.S3, jid string) (interface{}, error) {
-// 	key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_RESULTS_DIR"), jid)
+// 	key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_RESULTS_PREFIX"), jid)
 
 // 	exist, err := utils.KeyExists(key, svc)
 // 	if err != nil {
@@ -214,7 +214,7 @@ func FetchResults(svc *s3.S3, jid string) (interface{}, error) {
 // If JobID exists but metadata file doesn't then it raises an error
 // Assumes jobID is valid
 func FetchMeta(svc *s3.S3, jid string) (interface{}, error) {
-	key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_METADATA_DIR"), jid)
+	key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_METADATA_PREFIX"), jid)
 
 	exist, err := utils.KeyExists(key, svc)
 	if err != nil {
@@ -272,7 +272,7 @@ func FetchLogs(svc *s3.S3, jid, pid string, onlyContainer bool) (JobLogs, error)
 		}
 
 		// If not found locally, check storage
-		storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_DIR"), jid, k.key)
+		storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_PREFIX"), jid, k.key)
 		exists, err := utils.KeyExists(storageKey, svc)
 		if err != nil {
 			return JobLogs{}, err
@@ -309,7 +309,7 @@ func UploadLogsToStorage(svc *s3.S3, jid, pid string) {
 			log.Error(err.Error())
 		}
 
-		storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_DIR"), jid, k)
+		storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_PREFIX"), jid, k)
 		err = utils.WriteToS3(svc, bytes, storageKey, "text/plain", 0)
 		if err != nil {
 			log.Error(err.Error())

docker-compose.yml

@@ -12,20 +12,19 @@ services:
       - .env
     volumes:
       - ./api:/app
-      - ./.data:/.data
+      - ./.data/api:/.data
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - process_api_net
-    depends_on:
-      - minio
+    # depends_on:
+    #   - minio
 
   minio:
     container_name: minio
     image: minio/minio:RELEASE.2023-08-23T10-07-06Z.fips
-    environment:
-      - MINIO_ROOT_USER=user
-      - MINIO_ROOT_PASSWORD=password
     command: server /data --console-address ":9001"
+    env_file:
+      - .env
     ports:
       - '9000:9000'
       - '9001:9001'