batch: address PR comments

cmd/aws/batch/batchsvc.go

@@ -76,9 +76,9 @@ func (b *batchsvc) CreateComputeEnvironment() (*batch.ComputeEnvironmentDetail,
 	}
 
 	var serviceRole string
-	iamres, err := iamCli.GetRole(&iam.GetRoleInput{RoleName: aws.String("AWSBatchServiceRole")})
+	grres, err := iamCli.GetRole(&iam.GetRoleInput{RoleName: aws.String("AWSBatchServiceRole")})
 	if err == nil {
-		serviceRole = *iamres.Role.Arn
+		serviceRole = *grres.Role.Arn
 	} else {
 		bsrPolicy := AssumeRolePolicy{
 			Version: "2012-10-17",
@@ -112,9 +112,11 @@ func (b *batchsvc) CreateComputeEnvironment() (*batch.ComputeEnvironmentDetail,
 	}
 
 	var instanceRole string
-	iamres, err = iamCli.GetRole(&iam.GetRoleInput{RoleName: aws.String("ecsInstanceRole")})
+	ip, err := iamCli.GetInstanceProfile(&iam.GetInstanceProfileInput{
+		InstanceProfileName: aws.String("ecsInstanceRole"),
+	})
 	if err == nil {
-		instanceRole = *iamres.Role.Arn
+		instanceRole = *ip.InstanceProfile.Arn
 	} else {
 		irPolicy := AssumeRolePolicy{
 			Version: "2012-10-17",
@@ -130,7 +132,7 @@ func (b *batchsvc) CreateComputeEnvironment() (*batch.ComputeEnvironmentDetail,
 		if err != nil {
 			return nil, fmt.Errorf("error marshaling assume role policy for ecsInstanceRole: %v", err)
 		}
-		cr, err := iamCli.CreateRole(&iam.CreateRoleInput{
+		_, err = iamCli.CreateRole(&iam.CreateRoleInput{
 			AssumeRolePolicyDocument: aws.String(string(irBinary)),
 			RoleName:                 aws.String("ecsInstanceRole"),
 		})
@@ -144,7 +146,13 @@ func (b *batchsvc) CreateComputeEnvironment() (*batch.ComputeEnvironmentDetail,
 		if err != nil {
 			return nil, fmt.Errorf("error attaching policies to ecsInstanceRole: %v", err)
 		}
-		instanceRole = *cr.Role.Arn
+		ip, err = iamCli.GetInstanceProfile(&iam.GetInstanceProfileInput{
+			InstanceProfileName: aws.String("ecsInstanceRole"),
+		})
+		if err != nil {
+			return nil, fmt.Errorf("error fetching Intance Profile ARN for ecsInstanceRole: %v", err)
+		}
+		instanceRole = *ip.InstanceProfile.Arn
 	}
 
 	input := &batch.CreateComputeEnvironmentInput{

cmd/aws/batch/config.go

@@ -89,9 +89,8 @@ type JobRoleConfig struct {
 
 // DefaultConfig returns default configuration of for AWS Batch resource creation.
 func DefaultConfig() Config {
-	region := "us-west-2"
 	c := Config{
-		Region: region,
+		Region: "",
 		ComputeEnv: ComputeEnvConfig{
 			Name:          "funnel-compute-environment",
 			InstanceTypes: []string{"optimal"},
@@ -169,10 +168,10 @@ func DefaultConfig() Config {
 	c.FunnelWorker.BufferSize = 10000
 	c.FunnelWorker.TaskReader = "dynamodb"
 	c.FunnelWorker.TaskReaders.DynamoDB.TableBasename = "funnel"
-	c.FunnelWorker.TaskReaders.DynamoDB.Region = region
+	c.FunnelWorker.TaskReaders.DynamoDB.Region = ""
 	c.FunnelWorker.ActiveEventWriters = []string{"dynamodb", "log"}
 	c.FunnelWorker.EventWriters.DynamoDB.TableBasename = "funnel"
-	c.FunnelWorker.EventWriters.DynamoDB.Region = region
+	c.FunnelWorker.EventWriters.DynamoDB.Region = ""
 
 	return c
 }

cmd/aws/batch/createall.go

@@ -12,11 +12,11 @@ func init() {
 	f.StringVar(&funnelConfigFile, "config", funnelConfigFile, "Funnel configuration file")
 	f.StringVar(&conf.Region, "region", conf.Region, "Region in which to create the Batch resources")
 	f.StringVar(&conf.ComputeEnv.Name, "ComputeEnv.Name", conf.ComputeEnv.Name, "The name of the compute environment.")
-	f.Int64Var(&conf.ComputeEnv.MinVCPUs, "ComputEnv.MinVCPUs", conf.ComputeEnv.MinVCPUs, "The minimum number of EC2 vCPUs that an environment should maintain. (default 0)")
-	f.Int64Var(&conf.ComputeEnv.MaxVCPUs, "ComputEnv.MaxVCPUs", conf.ComputeEnv.MaxVCPUs, "The maximum number of EC2 vCPUs that an environment can reach.")
+	f.Int64Var(&conf.ComputeEnv.MinVCPUs, "ComputeEnv.MinVCPUs", conf.ComputeEnv.MinVCPUs, "The minimum number of EC2 vCPUs that an environment should maintain. (default 0)")
+	f.Int64Var(&conf.ComputeEnv.MaxVCPUs, "ComputeEnv.MaxVCPUs", conf.ComputeEnv.MaxVCPUs, "The maximum number of EC2 vCPUs that an environment can reach.")
 	f.StringSliceVar(&conf.ComputeEnv.SecurityGroupIds, "ComputEnv.SecurityGroupIds", conf.ComputeEnv.SecurityGroupIds, "The EC2 security groups that are associated with instances launched in the compute environment. If none are specified all security groups will be used.")
-	f.StringSliceVar(&conf.ComputeEnv.Subnets, "ComputEnv.Subnets", conf.ComputeEnv.Subnets, "The VPC subnets into which the compute resources are launched. If none are specified all subnets will be used.")
-	f.StringSliceVar(&conf.ComputeEnv.InstanceTypes, "ComputEnv.InstanceTypes", conf.ComputeEnv.InstanceTypes, "The instances types that may be launched. You can also choose optimal to pick instance types on the fly that match the demand of your job queues.")
+	f.StringSliceVar(&conf.ComputeEnv.Subnets, "ComputeEnv.Subnets", conf.ComputeEnv.Subnets, "The VPC subnets into which the compute resources are launched. If none are specified all subnets will be used.")
+	f.StringSliceVar(&conf.ComputeEnv.InstanceTypes, "ComputeEnv.InstanceTypes", conf.ComputeEnv.InstanceTypes, "The instances types that may be launched. You can also choose optimal to pick instance types on the fly that match the demand of your job queues.")
 	f.StringVar(&conf.JobQueue.Name, "JobQueue.Name", conf.JobQueue.Name, "The name of the job queue.")
 	f.Int64Var(&conf.JobQueue.Priority, "JobQueue.Priority", conf.JobQueue.Priority, "The priority of the job queue. Priority is determined in descending order.")
 	f.StringVar(&conf.JobDef.Name, "JobDef.Name", conf.JobDef.Name, "The name of the job definition.")
@@ -32,6 +32,13 @@ var createCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		log := logger.NewLogger("batch-create-all-resources", logger.DefaultConfig())
 
+		if conf.Region == "" {
+			return fmt.Errorf("error must provide a region")
+		}
+
+		conf.FunnelWorker.TaskReaders.DynamoDB.Region = conf.Region
+		conf.FunnelWorker.EventWriters.DynamoDB.Region = conf.Region
+
 		if funnelConfigFile != "" {
 			funnelConf := config.Config{}
 			config.ParseFile(funnelConfigFile, &funnelConf)

cmd/aws/batch/createjobdef.go

@@ -27,6 +27,13 @@ var jobdefCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		log := logger.NewLogger("batch-create-job-def", logger.DefaultConfig())
 
+		if conf.Region == "" {
+			return fmt.Errorf("error must provide a region")
+		}
+
+		conf.FunnelWorker.TaskReaders.DynamoDB.Region = conf.Region
+		conf.FunnelWorker.EventWriters.DynamoDB.Region = conf.Region
+
 		if funnelConfigFile != "" {
 			funnelConf := config.Config{}
 			config.ParseFile(funnelConfigFile, &funnelConf)

config/default-config.yaml

@@ -103,7 +103,7 @@ Scheduler:
 
 
 # The name of the active compute backend
-# Available backends: local, htcondor, slurm, pbs, gridengine, manual, gce, openstack
+# Available backends: local, htcondor, slurm, pbs, gridengine, manual, gce, openstack, aws-batch
 Backend: local
 
 # Scheduler backend config
@@ -207,16 +207,10 @@ Backends:
 
   # Batch describes the configuration for the AWS Batch compute backend.
   Batch:
-    # JobDefConfig represents configuration of the AWS Batch
-    # base Job Definition.
-    JobDef:
-      Name: "funnel-job-def"
-      Image: "docker.io/ohsu-comp-bio/funnel:latest"
-      DefaultMemory: 128
-      DefaultVcpus:  1
-      JobRoleArn: ""
+    # JobDefinition can be either a name or the Amazon Resource Name (ARN).
+    JobDefinition: "funnel-job-def"
     # JobQueue can be either a name or the Amazon Resource Name (ARN).
-    JobQueue: ""
+    JobQueue: "funnel-job-queue"
     # AWS region of the specified job queue and to create the job definition in
     Region: ""
     Credentials:

website/content/docs/guides/aws.md

@@ -11,34 +11,37 @@ menu:
 
 This guide covers deploying a Funnel server that leverages [DynamoDB][0] for storage
 and [Batch][1] for task execution. You'll need to set up several resources 
-using either the Funnel CLI or through the provided Amazon web UI.
-
+using either the Funnel CLI or through the provided Amazon web console.
 
 ## Create Required AWS Batch Resources
 
 For Funnel to execute tasks on Batch, you must define a Compute Environment,
-Job Queue and Job Definition. Amazon provides a quick start guide for these steps [here][2]. 
-
-Additionally, you must define an IAM role for your Batch Job Definition. The role 
-provides the job container with permissions to call the API actions that are 
-specified in its associated policies on your behalf. For this configuration, 
-these jobs need access to S3 and DynamoDB. 
+Job Queue and Job Definition. Additionally, you must define an IAM role for your 
+Batch Job Definition. The role provides the job container with permissions to call 
+the API actions that are specified in its associated policies on your behalf. For 
+this configuration, these jobs need access to S3 and DynamoDB. 
 
-
-Note, we reccommend creating the Job Definition with Funnel via `funnel aws batch create-job-definition`. 
-Funnel expects the JobDefinition to start a `worker` process with a specific configuration. Only 
-advanced users should consider making any substantial changes to this JobDefinition. 
+Note, we recommend creating the Job Definition with Funnel by running: `funnel aws batch create-job-definition`. 
+Funnel expects the JobDefinition to start a `worker` with a specific configuration. Only 
+advanced users should consider making any substantial changes to this Job Definition. 
 
 ### Create Resources With AWS
 
-* Create Compute Environment - [link][3]
-* Create Job Queue - [link][4]
+Amazon provides a quick start guide with more information [here][2]. 
+
+* Create a Compute Environment - [link][3]
+* Create a Job Queue - [link][4]
 * Define an EC2ContainerTaskRole with policies for managing access to S3 and DynamoDB - [link][5]
 * Create a Job Definition - [link][6]
 
-
 ### Create Resources With Funnel
 
+Funnel provides a utility to create the resources you will need to get up and running.
+You will need to specify the AWS region to create these resources in using the `--region` flag.
+
+_Note:_ this command assumes your environment contains your AWS credentials. These 
+can be configured with the `aws configure` command.
+
 ```
 $ funnel aws batch create-all-resources
 
@@ -48,11 +51,11 @@ Usage:
   funnel aws batch create-all-resources [flags]
 
 Flags:
-      --ComputEnv.InstanceTypes strings      The instances types that may be launched. You can also choose optimal to pick instance types on the fly that match the demand of your job queues. (default [optimal])
-      --ComputEnv.MaxVCPUs int               The maximum number of EC2 vCPUs that an environment can reach. (default 256)
-      --ComputEnv.MinVCPUs int               The minimum number of EC2 vCPUs that an environment should maintain. (default 0)
-      --ComputEnv.SecurityGroupIds strings   The EC2 security groups that are associated with instances launched in the compute environment. If none are specified all security groups will be used.
-      --ComputEnv.Subnets strings            The VPC subnets into which the compute resources are launched. If none are specified all subnets will be used.
+      --ComputeEnv.InstanceTypes strings     The instances types that may be launched. You can also choose optimal to pick instance types on the fly that match the demand of your job queues. (default [optimal])
+      --ComputeEnv.MaxVCPUs int              The maximum number of EC2 vCPUs that an environment can reach. (default 256)
+      --ComputeEnv.MinVCPUs int              The minimum number of EC2 vCPUs that an environment should maintain. (default 0)
+      --ComputeEnv.SecurityGroupIds strings  The EC2 security groups that are associated with instances launched in the compute environment. If none are specified all security groups will be used.
+      --ComputeEnv.Subnets strings           The VPC subnets into which the compute resources are launched. If none are specified all subnets will be used.
       --ComputeEnv.Name string               The name of the compute environment. (default "funnel-compute-environment")
       --JobDef.Image string                  The docker image used to start a container. (default "docker.io/ohsucompbio/funnel:latest")
       --JobDef.JobRoleArn string             The Amazon Resource Name (ARN) of the IAM role that the container can assume for AWS permissions. A role will be created if not provided.
@@ -63,7 +66,7 @@ Flags:
       --JobQueue.Priority int                The priority of the job queue. Priority is determined in descending order. (default 1)
       --config string                        Funnel configuration file
   -h, --help                                 help for create-resources
-      --region string                        Region in which to create the Batch resources (default "us-west-2")
+      --region string                        Region in which to create the Batch resources.
 ```
 
 
@@ -79,8 +82,8 @@ Start the server:
 funnel server run --config /path/to/config.yaml
 ```
 
-
-Example configuration:
+Here is an example of the configuration you would need for the server had you 
+run `funnel aws batch create-all-resources --region us-west-2`:
 
 ```YAML
 Server:
@@ -93,16 +96,11 @@ Server:
         Key: ""
         Secret: ""
 
-Backend: "batch"
+Backend: "aws-batch"
 Backends:
   Batch:
-    JobDef:
-      Name: "funnel-job-def"
-      Image: "docker.io/ohsu-comp-bio/funnel:latest"
-      DefaultMemory: 128
-      DefaultVcpus: 1
-      JobRoleArn: "<PLACEHOLDER>"
-    JobQueue: "<PLACEHOLDER>"
+    JobDefinition: "funnel-job-def"
+    JobQueue: "funnel-job-queue" 
     Region: "us-west-2"
     Credentials:
       Key: ""

website/static/funnel-config-examples/default-config.yaml

@@ -40,6 +40,12 @@ Server:
     #     # AWS Secret Access Key
     #     Secret: ""
 
+    Elastic:
+      # Prefix to use for indexes (task, events, nodes)
+      IndexPrefix: "funnel"
+      # URL of the elasticsearch server.
+      URL: http://localhost:9200
+
   Logger:
     # Logging levels: debug, info, error
     Level: info
@@ -97,7 +103,7 @@ Scheduler:
 
 
 # The name of the active compute backend
-# Available backends: local, htcondor, slurm, pbs, gridengine, manual, gce, openstack
+# Available backends: local, htcondor, slurm, pbs, gridengine, manual, gce, openstack, aws-batch
 Backend: local
 
 # Scheduler backend config
@@ -201,16 +207,10 @@ Backends:
 
   # Batch describes the configuration for the AWS Batch compute backend.
   Batch:
-    # JobDefConfig represents configuration of the AWS Batch
-    # base Job Definition.
-    JobDef:
-      Name: "funnel-job-def"
-      Image: "docker.io/ohsu-comp-bio/funnel:latest"
-      DefaultMemory: 128
-      DefaultVcpus:  1
-      JobRoleArn: ""
+    # JobDefinition can be either a name or the Amazon Resource Name (ARN).
+    JobDefinition: "funnel-job-def"
     # JobQueue can be either a name or the Amazon Resource Name (ARN).
-    JobQueue: ""
+    JobQueue: "funnel-job-queue"
     # AWS region of the specified job queue and to create the job definition in
     Region: ""
     Credentials: