From 6fddc7fc94abf705d05226b32d7af356a88119b2 Mon Sep 17 00:00:00 2001 From: Alex Buchanan Date: Mon, 8 May 2017 13:25:50 -0700 Subject: [PATCH 1/2] Update readme --- README.md | 190 +++++++++++++++--------------------------------------- 1 file changed, 52 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 0e5acf6..8a7c525 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,63 @@ -# task-execution-schemas - GA4GH Task Execution Schema (TES) ================================= -The Task Execution Schema proposal can be found at https://github.com/ga4gh/task-execution-schemas -The Protocol Buffer Based Schema can be found at https://github.com/ga4gh/task-execution-schemas/blob/master/proto/task_execution.proto -The swagger translation can be viewed at http://editor.swagger.io/#/?import=https://github.com/ga4gh/task-execution-schemas/raw/master/swagger/proto/task_execution.swagger.json +The Task Execution Schema (TES) project is an effort to define a standardized schema +and API for describing batch execution tasks. A task defines a set of input files, +a set of (Docker) containers and commands to run, a set of output files, +and some other logging and metadata. -Example Task Message -``` -{ - "name" : "TestMD5", - "projectId" : "MyProject", - "description" : "My Desc", - "inputs" : [ - { - "name" : "infile", - "description" : "File to be MD5ed", - "location" : "s3://my-bucket/input_file", - "path" : "/tmp/test_file" - } - ], - "outputs" : [ - { - "location" : "s3://my-bucket/output_file", - "path" : "/tmp/test_out" - } - ], - "resources" : { - "volumes" : [{ - "name" : "test_disk", - "sizeGb" : 5, - "mountPoint" : "/tmp" - }] - }, - "docker" : [ - { - "imageName" : "ubuntu", - "cmd" : ["md5sum", "/tmp/test_file"], - "stdout" : "/tmp/test_out" - } - ] -} -``` +The schema and APIs is defined [here](./task_execution.proto) in [protocol buffers](https://developers.google.com/protocol-buffers/). Clients may use JSON and REST to communicate +with a service implementing the TES API. -Example Task Message: -``` +Here's an example task message, defining a task which calculates +an MD5 checksum on an input file and uploads output: +```JSON { - "jobId" : "6E57CA6B-0BC7-44FB-BA2C-0CBFEC629C63", - "metadata" : { Custom service metadata }, - "task" : {Task Message Above}, - "state" : "Running", - "logs" : [ - { Job Log } - ] -} -``` - -Example Job Log Message: -``` -{ - "cmd" : ["md5sum", "/tmp/test_file"], - "startTime" : "2016-09-18T23:08:27Z", - "endTime" : "2016-09-18T23:38:00Z", - "stdout": "f6913671da6018ff8afcb1517983ab24 test_file", - "stderr": "", - "exitCode" = 0 + "name": "MD5 example", + "description": "Task which runs md5sum on the input file.", + "project": "tes-example-project-id", + "tags": { + "custom-tag": "tag-value", + }, + "inputs": [ + { + "name": "infile", + "description": "md5sum input file", + "url": "/path/to/input_file", + "path": "/container/input", + "type": "FILE", + } + ], + "outputs" : [ + { + "url" : "/path/to/output_file", + "path" : "/container/output", + } + ], + "resources" : { + "cpu_cores": 1, + "ram_gb": 1.0, + "size_gb": 100.0, + "preemptible": false, + }, + "executors" : [ + { + "image_name" : "ubuntu", + "cmd" : ["md5sum", "/container/input"], + "stdout" : "/container/output", + "stderr" : "/container/stderr", + "workdir": "/tmp", + } + ] } ``` -Example Task Conversation: - -Get meta-data about service -``` -GET /v1/jobs-service -``` -Returns (from reference server) -{"storageConfig":{"baseDir":"/var/run/task-execution-server/storage","storageType":"sharedFile"}} - - -Post Job -``` -POST /v1/jobs {JSON body message above} -Return: -{ "value" : "{job uuid}"} +This message would be submitted via HTTP Post to `/v1/tasks`. +The return value is a task ID: +```JSON +{ "id": "6E57CA6B-0BC7-44FB-BA2C-0CBFEC629C63" } ``` -Get Job Info -``` -GET /v1/jobs/{job uuid} -``` -Returns Job Body Example: -``` -{ - "jobId" : "06b170b4-6ae8-4f11-7fc6-4417f1778b74", - "logs" : [ - { - "exitCode" : -1 - } - ], - "task" : { - "projectId" : "test", - "inputs" : [ - { - "location" : "fs://README.md", - "description" : "input", - "path" : "/mnt/README.md", - "name" : "input" - } - ], - "name" : "funnel workflow", - "taskId" : "06b170b4-6ae8-4f11-7fc6-4417f1778b74", - "resources" : { - "minimumRamGb" : 1, - "minimumCpuCores" : 1, - "volumes" : [ - { - "sizeGb" : 10, - "name" : "data", - "mountPoint" : "/mnt" - } - ] - }, - "outputs" : [ - { - "location" : "fs://output/sha", - "path" : "/mnt/sha", - "name" : "stdout", - "description" : "tool stdout" - } - ], - "docker" : [ - { - "imageName" : "bmeg/openssl", - "workdir" : "/mnt/sha", - "cmd" : [ - "openssl", - "dgst", - "-sha", - "/mnt/README.md" - ] - } - ], - "description" : "CWL TES task" - }, - "state" : "Error" -} -``` \ No newline at end of file +Then, the task and logs may be retrieve with a HTTP GET. +`GET /v1/tasks/6E57CA6B-0BC7-44FB-BA2C-0CBFEC629C63 +` From e39f43ef337e70187c688954bea7262e2d5b9aa7 Mon Sep 17 00:00:00 2001 From: Alex Buchanan Date: Wed, 24 May 2017 12:03:31 -0700 Subject: [PATCH 2/2] Add get/list/cancel docs --- README.md | 103 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8a7c525..af97b10 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,12 @@ and some other logging and metadata. The schema and APIs is defined [here](./task_execution.proto) in [protocol buffers](https://developers.google.com/protocol-buffers/). Clients may use JSON and REST to communicate with a service implementing the TES API. -Here's an example task message, defining a task which calculates -an MD5 checksum on an input file and uploads output: + +Create a task +--------------------------------- + +Here's an example of a complete task message, defining a task which calculates +an MD5 checksum on an input file and uploads the output: ```JSON { "name": "MD5 example", @@ -52,12 +56,95 @@ an MD5 checksum on an input file and uploads output: } ``` -This message would be submitted via HTTP Post to `/v1/tasks`. -The return value is a task ID: +A minimal version of the same task, including only the required fields looks like: ```JSON -{ "id": "6E57CA6B-0BC7-44FB-BA2C-0CBFEC629C63" } +{ + "inputs": [ + { + "url": "/path/to/input_file", + "path": "/container/input", + } + ], + "outputs" : [ + { + "url" : "/path/to/output_file", + "path" : "/container/output", + } + ], + "executors" : [ + { + "image_name" : "ubuntu", + "cmd" : ["md5sum", "/container/input"], + "stdout" : "/container/output", + } + ] +} +``` + +To create the task, send an HTTP POST request: +```HTTP +POST /v1/tasks + +{ "id": "task-1234" } ``` -Then, the task and logs may be retrieve with a HTTP GET. -`GET /v1/tasks/6E57CA6B-0BC7-44FB-BA2C-0CBFEC629C63 -` +The return value is a task ID. + + +Get a task +-------------------------------- + +To get a task by ID: + +```HTTP +GET /v1/tasks/task-1234 + +{ "id": "task-1234", "state": "RUNNING" } +``` + +The return value will be a minimal description of the task state. + +To get more information, you can change the task view using the `view` URL query parameter. + +The `basic` view will include all task fields except a few which might be +large strings (stdout/err logging, input parameter contents). + +```HTTP +GET /v1/tasks/task-1234?view=BASIC + +{ "id": "task-1234", "state": "RUNNING", "name": "MD5 example", etc... } +``` + +The `full` view includes stdout/err logs and full input parameters: + +```HTTP +GET /v1/tasks/task-1234?view=FULL + +{ "id": "task-1234", "state": "RUNNING", "name": "MD5 example", + "logs": [{ "stdout": "stdout content..." }], etc... } +``` + +List tasks +------------------------------------ + +To list tasks: + +```HTTP +GET /v1/tasks + +{ "tasks": [{ "id": "task-1234", "state": "RUNNING"}, etc...] } +``` + +Similar to getting a task by ID, you may change the task view: +```HTTP +GET /v1/tasks?view=BASIC +``` + + +Cancel a task +------------------------------------- + +To cancel a task, send an HTTP POST to the cancel endpoint: +```HTTP +POST /v1/tasks/task-1234:cancel +```