From 823045fc2c90c581d3e93f96a637f147a3ecde9e Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Wed, 11 Oct 2023 17:12:04 -0400 Subject: [PATCH 1/3] update swagger docs --- .gitignore | 3 +++ api/docs/docs.go | 41 +++++++++++++++++++++++++++++----------- api/docs/swagger.json | 38 +++++++++++++++++++++++++++---------- api/docs/swagger.yaml | 29 ++++++++++++++++++++-------- api/handlers/handlers.go | 4 ++-- docker-compose.yml | 2 +- 6 files changed, 85 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 0239475..b9d26f4 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ .data/* *main settings.json + + +.miniodata/ \ No newline at end of file diff --git a/api/docs/docs.go b/api/docs/docs.go index 20b0a56..931cb55 100644 --- a/api/docs/docs.go +++ b/api/docs/docs.go @@ -1,4 +1,5 @@ -// Code generated by swaggo/swag. DO NOT EDIT +// Code generated by swaggo/swag. DO NOT EDIT. + package docs import "github.com/swaggo/swag" @@ -269,7 +270,7 @@ const docTemplate = `{ "post": { "description": "[Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job)", "consumes": [ - "*/*" + "application/json" ], "produces": [ "application/json" @@ -287,8 +288,8 @@ const docTemplate = `{ "required": true }, { - "description": "example: {“text”:“Hello World!”}", - "name": "input", + "description": "example: {inputs: {text:Hello World!}} (add double quotes for all strings in the payload)", + "name": "inputs", "in": "body", "required": true, "schema": { @@ -336,16 +337,10 @@ const docTemplate = `{ "jobs.JobLogs": { "type": "object", "properties": { - "api_logs": { - "type": "array", - "items": { - "type": "string" - } - }, "container_logs": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/jobs.LogEntry" } }, "jobID": { @@ -353,6 +348,12 @@ const docTemplate = `{ }, "processID": { "type": "string" + }, + "server_logs": { + "type": "array", + "items": { + "$ref": "#/definitions/jobs.LogEntry" + } } } }, @@ -383,6 +384,20 @@ const docTemplate = `{ } } }, + "jobs.LogEntry": { + "type": "object", + "properties": { + "level": { + "type": "string" + }, + "msg": { + "type": "string" + }, + "time": { + "type": "string" + } + } + }, "processes.Info": { "type": "object", "properties": { @@ -560,6 +575,10 @@ const docTemplate = `{ } } } + }, + "externalDocs": { + "description": "Schemas", + "url": "http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/" } }` diff --git a/api/docs/swagger.json b/api/docs/swagger.json index ddafdf1..0f5a71d 100644 --- a/api/docs/swagger.json +++ b/api/docs/swagger.json @@ -266,7 +266,7 @@ "post": { "description": "[Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job)", "consumes": [ - "*/*" + "application/json" ], "produces": [ "application/json" @@ -284,8 +284,8 @@ "required": true }, { - "description": "example: {“text”:“Hello World!”}", - "name": "input", + "description": "example: {inputs: {text:Hello World!}} (add double quotes for all strings in the payload)", + "name": "inputs", "in": "body", "required": true, "schema": { @@ -333,16 +333,10 @@ "jobs.JobLogs": { "type": "object", "properties": { - "api_logs": { - "type": "array", - "items": { - "type": "string" - } - }, "container_logs": { "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/jobs.LogEntry" } }, "jobID": { @@ -350,6 +344,12 @@ }, "processID": { "type": "string" + }, + "server_logs": { + "type": "array", + "items": { + "$ref": "#/definitions/jobs.LogEntry" + } } } }, @@ -380,6 +380,20 @@ } } }, + "jobs.LogEntry": { + "type": "object", + "properties": { + "level": { + "type": "string" + }, + "msg": { + "type": "string" + }, + "time": { + "type": "string" + } + } + }, "processes.Info": { "type": "object", "properties": { @@ -557,5 +571,9 @@ } } } + }, + "externalDocs": { + "description": "Schemas", + "url": "http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/" } } \ No newline at end of file diff --git a/api/docs/swagger.yaml b/api/docs/swagger.yaml index 65cc4a1..2d0f80a 100644 --- a/api/docs/swagger.yaml +++ b/api/docs/swagger.yaml @@ -19,18 +19,18 @@ definitions: type: object jobs.JobLogs: properties: - api_logs: - items: - type: string - type: array container_logs: items: - type: string + $ref: '#/definitions/jobs.LogEntry' type: array jobID: type: string processID: type: string + server_logs: + items: + $ref: '#/definitions/jobs.LogEntry' + type: array type: object jobs.JobRecord: properties: @@ -50,6 +50,15 @@ definitions: updated: type: string type: object + jobs.LogEntry: + properties: + level: + type: string + msg: + type: string + time: + type: string + type: object processes.Info: properties: description: @@ -165,6 +174,9 @@ definitions: $ref: '#/definitions/processes.Outputs' type: array type: object +externalDocs: + description: Schemas + url: http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/ host: localhost:5050 info: contact: @@ -340,7 +352,7 @@ paths: /processes/{processID}/execution: post: consumes: - - '*/*' + - application/json description: '[Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job)' parameters: - description: pyecho @@ -348,9 +360,10 @@ paths: name: processID required: true type: string - - description: 'example: {“text”:“Hello World!”}' + - description: 'example: {inputs: {text:Hello World!}} (add double quotes for + all strings in the payload)' in: body - name: input + name: inputs required: true schema: type: string diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 15b0673..f8c78ff 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -241,10 +241,10 @@ func (rh *RESTHandler) ProcessDescribeHandler(c echo.Context) error { // @Summary Execute Process // @Description [Execute Process Specification](https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_create_job) // @Tags processes -// @Accept */* +// @Accept json // @Produce json // @Param processID path string true "pyecho" -// @Param input body string true "example: {“text”:“Hello World!”}" +// @Param inputs body string true "example: {inputs: {text:Hello World!}} (add double quotes for all strings in the payload)" // @Success 200 {object} jobResponse // @Router /processes/{processID}/execution [post] // Does not produce HTML diff --git a/docker-compose.yml b/docker-compose.yml index fe07b97..6a45943 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: - .env volumes: - ./api:/app - - ./.data:/.data + - ./.data/api:/.data - /var/run/docker.sock:/var/run/docker.sock networks: - process_api_net From 366c151f12cd2191acd8d2bf003d6f9b5e1e40a9 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Wed, 11 Oct 2023 17:33:30 -0400 Subject: [PATCH 2/3] update build args and env vars --- .github/workflows/e2e-tests.yml | 6 +++--- api/jobs/aws_batch_jobs.go | 4 ++-- api/jobs/docker_jobs.go | 4 ++-- api/jobs/jobs.go | 8 ++++---- docker-compose.yml | 9 ++++----- example.env | 29 ++++++++++++++--------------- 6 files changed, 29 insertions(+), 31 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a37bc74..bc028f9 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -21,9 +21,9 @@ jobs: echo API_NAME='github-testing-process-api' >> .env echo STORAGE_SERVICE='minio' >> .env echo STORAGE_BUCKET='api-storage' >> .env - echo STORAGE_METADATA_DIR='metadata' >> .env - echo STORAGE_RESULTS_DIR='results' >> .env - echo STORAGE_LOGS_DIR='logs' >> .env + echo STORAGE_METADATA_PREFIX='metadata' >> .env + echo STORAGE_RESULTS_PREFIX='results' >> .env + echo STORAGE_LOGS_PREFIX='logs' >> .env echo LOCAL_LOGS_DIR='../.data/logs' >> .env diff --git a/api/jobs/aws_batch_jobs.go b/api/jobs/aws_batch_jobs.go index 29571ef..507e8e5 100644 --- a/api/jobs/aws_batch_jobs.go +++ b/api/jobs/aws_batch_jobs.go @@ -405,7 +405,7 @@ func (j *AWSBatchJob) WriteMetaData() { return } - metadataDir := os.Getenv("STORAGE_METADATA_DIR") + metadataDir := os.Getenv("STORAGE_METADATA_PREFIX") mdLocation := fmt.Sprintf("%s/%s.json", metadataDir, j.UUID) // TODO: Determine if batch metadata should be put on aws...currently this is the case utils.WriteToS3(j.StorageSvc, jsonBytes, mdLocation, "application/json", 0) @@ -415,7 +415,7 @@ func (j *AWSBatchJob) WriteMetaData() { // j.logger.Info("Starting results writing routine.") // defer j.logger.Info("Finished results writing routine.") -// resultsDir := os.Getenv("STORAGE_RESULTS_DIR") +// resultsDir := os.Getenv("STORAGE_RESULTS_PREFIX") // resultsLocation := fmt.Sprintf("%s/%s.json", resultsDir, j.UUID) // err = utils.WriteToS3(j.StorageSvc, data, resultsLocation, "application/json", 0) // if err != nil { diff --git a/api/jobs/docker_jobs.go b/api/jobs/docker_jobs.go index ca96ac9..109a574 100644 --- a/api/jobs/docker_jobs.go +++ b/api/jobs/docker_jobs.go @@ -358,7 +358,7 @@ func (j *DockerJob) WriteMetaData() { return } - metadataDir := os.Getenv("STORAGE_METADATA_DIR") + metadataDir := os.Getenv("STORAGE_METADATA_PREFIX") mdLocation := fmt.Sprintf("%s/%s.json", metadataDir, j.UUID) err = utils.WriteToS3(j.StorageSvc, jsonBytes, mdLocation, "application/json", 0) if err != nil { @@ -370,7 +370,7 @@ func (j *DockerJob) WriteMetaData() { // j.logger.Info("Starting results writing routine.") // defer j.logger.Info("Finished results writing routine.") -// resultsDir := os.Getenv("STORAGE_RESULTS_DIR") +// resultsDir := os.Getenv("STORAGE_RESULTS_PREFIX") // resultsLocation := fmt.Sprintf("%s/%s.json", resultsDir, j.UUID) // fmt.Println(resultsLocation) // err = utils.WriteToS3(j.StorageSvc, data, resultsLocation, "application/json", 0) diff --git a/api/jobs/jobs.go b/api/jobs/jobs.go index c9b2bc8..de58e22 100644 --- a/api/jobs/jobs.go +++ b/api/jobs/jobs.go @@ -192,7 +192,7 @@ func FetchResults(svc *s3.S3, jid string) (interface{}, error) { // // If JobID exists but results file doesn't then it raises an error // // Assumes jobID is valid // func FetchResults(svc *s3.S3, jid string) (interface{}, error) { -// key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_RESULTS_DIR"), jid) +// key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_RESULTS_PREFIX"), jid) // exist, err := utils.KeyExists(key, svc) // if err != nil { @@ -214,7 +214,7 @@ func FetchResults(svc *s3.S3, jid string) (interface{}, error) { // If JobID exists but metadata file doesn't then it raises an error // Assumes jobID is valid func FetchMeta(svc *s3.S3, jid string) (interface{}, error) { - key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_METADATA_DIR"), jid) + key := fmt.Sprintf("%s/%s.json", os.Getenv("STORAGE_METADATA_PREFIX"), jid) exist, err := utils.KeyExists(key, svc) if err != nil { @@ -272,7 +272,7 @@ func FetchLogs(svc *s3.S3, jid, pid string, onlyContainer bool) (JobLogs, error) } // If not found locally, check storage - storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_DIR"), jid, k.key) + storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_PREFIX"), jid, k.key) exists, err := utils.KeyExists(storageKey, svc) if err != nil { return JobLogs{}, err @@ -309,7 +309,7 @@ func UploadLogsToStorage(svc *s3.S3, jid, pid string) { log.Error(err.Error()) } - storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_DIR"), jid, k) + storageKey := fmt.Sprintf("%s/%s.%s.jsonl", os.Getenv("STORAGE_LOGS_PREFIX"), jid, k) err = utils.WriteToS3(svc, bytes, storageKey, "text/plain", 0) if err != nil { log.Error(err.Error()) diff --git a/docker-compose.yml b/docker-compose.yml index 6a45943..525239c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,16 +16,15 @@ services: - /var/run/docker.sock:/var/run/docker.sock networks: - process_api_net - depends_on: - - minio + # depends_on: + # - minio minio: container_name: minio image: minio/minio:RELEASE.2023-08-23T10-07-06Z.fips - environment: - - MINIO_ROOT_USER=user - - MINIO_ROOT_PASSWORD=password command: server /data --console-address ":9001" + env_file: + - .env ports: - '9000:9000' - '9001:9001' diff --git a/example.env b/example.env index f61595f..1d54c50 100644 --- a/example.env +++ b/example.env @@ -1,18 +1,6 @@ -API_NAME='process-api' # The api will launch all jobs on cloud with name starting with this prefix - -# Define the provider for API Storage such as metadata, results etc -# One of 'minio', 'aws-s3' -STORAGE_SERVICE='minio' -STORAGE_BUCKET='api-storage' -STORAGE_METADATA_DIR='metadata' -STORAGE_RESULTS_DIR='results' -STORAGE_LOGS_DIR='logs' - -# MINIO +# MINIO (An option for storage (logs and metadata) and useful for dev) MINIO_ACCESS_KEY_ID=user MINIO_SECRET_ACCESS_KEY=password -MINIO_S3_BUCKET=local-data - MINIO_S3_ENDPOINT=http://minio:9000 MINIO_S3_REGION=us-east-1 MINIO_S3_DISABLE_SSL=true @@ -23,12 +11,23 @@ AWS_ACCESS_KEY_ID=user AWS_SECRET_ACCESS_KEY=password AWS_REGION=us-east-1 +API_NAME='process-api' # The api will launch all jobs on cloud with name starting with this prefix + +# Define the provider for API Storage of logs and metadata +# One of 'minio', 'aws-s3' +STORAGE_SERVICE='minio' +STORAGE_BUCKET='api-storage' +STORAGE_METADATA_PREFIX='metadata' +STORAGE_RESULTS_PREFIX='results' +STORAGE_LOGS_PREFIX='logs' + # BATCH BATCH_LOG_STREAM_GROUP='/aws/batch/job' -# LOCAL -LOCAL_LOGS_DIR='../.data/logs' +# LOCAL (temporary storage of logs) +LOCAL_LOGS_DIR='../.data/api/logs' # Policies EXPIRY_DAYS='7' +# Additional environment variables for containers running locally should go here \ No newline at end of file From 6e52f3b9d2d7ec5c714cadc5a28d4ffd30da0270 Mon Sep 17 00:00:00 2001 From: Seth Lawler Date: Wed, 11 Oct 2023 17:37:49 -0400 Subject: [PATCH 3/3] update args in actions --- .github/workflows/e2e-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index bc028f9..6ec82c2 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -33,6 +33,8 @@ jobs: echo MINIO_S3_REGION='us-east-1' >> .env echo MINIO_S3_DISABLE_SSL=true >> .env echo MINIO_S3_FORCE_PATH_STYLE=true >> .env + echo MINIO_ROOT_USER=user >> .env + echo MINIO_ROOT_PASSWORD=password >> .env echo AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} >> .env echo AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} >> .env