-
Notifications
You must be signed in to change notification settings - Fork 232
/
skeleton_pipeline.json
58 lines (58 loc) · 1.58 KB
/
skeleton_pipeline.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
{
"objects": [
{
"directoryPath": "#{myS3OutputLoc}",
"name": "<fill in>",
"id": "<fill in>",
"type": "S3DataNode"
},
{
"output": "<fill in>",
"id" : "<fill in>",
"type" : "ShellCommandActivity",
"command" : "#{myShellCmd}",
"runsOn": {
"ref": "<fill in>"
}
},
{
<This is the place where you have to populate a schedule for the data pipeline>
},
{
"instanceType": "t1.micro",
"name": "EC2ResourceObj",
"id": "<fill in>",
"type": "<fill in>",
"terminateAfter": "20 Minutes"
},
{
"failureAndRerunMode": "CASCADE",
"schedule": {
"ref": "SchedulePeriod"
},
"resourceRole": "<fill in>",
"role": "<fill in>",
"pipelineLogUri": "<fill in>",
"scheduleType": "cron",
"name": "Default",
"id": "Default"
}
],
"parameters": [
{
"description": "S3 output folder",
"id": "myS3OutputLoc",
"type": "AWS::S3::ObjectKey"
},
{
"default": "echo SUCCESS!! > ${OUTPUT1_STAGING_DIR}/output_file.txt",
"description": "Shell command to run",
"id": "myShellCmd",
"type": "String"
}
],
"values": {
"myShellCmd": "echo SUCCESS!! > ${OUTPUT1_STAGING_DIR}/output_file.txt",
"myS3OutputLoc": "s3://<your bucket/dp-folder"
}
}