Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move to managed Nextflow AWS execution policy #30

Merged
merged 3 commits into from
May 9, 2024
Merged

Move to managed Nextflow AWS execution policy #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fcda68d
Moved from managed NF S3 bucket access policy to managed NF aws execu…
mluypaert May 9, 2024
9bafbde
Added missing NF S3 bucket permissions
mluypaert May 9, 2024
7e2104f
Python style fix
mluypaert May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 53 additions & 29 deletions pipeline/aws_infra/cdk_classes/aws_batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class PaviExecutionEnvironment:
compute_environment: aws_batch.ManagedEc2EcsComputeEnvironment
job_queue: aws_batch.JobQueue
nf_output_bucket: s3.Bucket | s3.IBucket
nf_bucket_access_policy: iam.ManagedPolicy
nf_aws_execution_policy: iam.ManagedPolicy

def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Optional[str]) -> None:
"""
Expand Down Expand Up @@ -54,40 +54,34 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option
scope=scope, id='pavi-pipeline-nf-workdir-bucket',
bucket_name=shared_work_dir_bucket)

# Create the compute environment
s3_workdir_bucket_policy_doc = iam.PolicyDocument(
statements=[
iam.PolicyStatement(
sid="S3BucketWriteAll",
effect=iam.Effect.ALLOW,
actions=['s3:Put*'],
resources=[self.nf_output_bucket.bucket_arn + '/*']
),
iam.PolicyStatement(
sid="S3BucketReadAll",
effect=iam.Effect.ALLOW,
actions=['s3:ListBucket*', 's3:Get*'],
resources=[self.nf_output_bucket.bucket_arn, self.nf_output_bucket.bucket_arn + '/*']
)
]
)
s3_workdir_bucket_policy = iam.ManagedPolicy(scope, 'pavi-s3-nextflow-bucket-policy',
managed_policy_name='agr-pavi-pipeline-nf-bucket-access',
description='Grant required access to PAVI pipeline nextflow bucket to run nextflow pipelines using it.',
document=s3_workdir_bucket_policy_doc)
cdk_tags.of(s3_workdir_bucket_policy).add("Product", "PAVI") # type: ignore
cdk_tags.of(s3_workdir_bucket_policy).add("Managed_by", "PAVI") # type: ignore

self.nf_bucket_access_policy = s3_workdir_bucket_policy
# Create the IAM policy to grant access to the nextflow output bucket
s3_bucket_access_statements = [
iam.PolicyStatement(
sid="S3BucketWriteAll",
effect=iam.Effect.ALLOW,
actions=['s3:Put*', 's3:DeleteObject'],
resources=[self.nf_output_bucket.bucket_arn + '/*']
),
iam.PolicyStatement(
sid="S3BucketReadAll",
effect=iam.Effect.ALLOW,
actions=['s3:ListBucket*', 's3:Get*'],
resources=[self.nf_output_bucket.bucket_arn, self.nf_output_bucket.bucket_arn + '/*']
)
]
s3_workdir_bucket_policy_doc = iam.PolicyDocument(statements=s3_bucket_access_statements)

# Create the compute environment
instance_role = iam.Role(scope, 'pavi-pipeline-compute-environment-instance-role',
description='Role granting permissions for Nextflow ECS execution',
assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'), # type: ignore
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEC2ContainerServiceforEC2Role"),
iam.ManagedPolicy.from_managed_policy_name(scope, "iam-ecr-read-policy", "ReadOnlyAccessECR"),
self.nf_bucket_access_policy
])
iam.ManagedPolicy.from_managed_policy_name(scope, "iam-ecr-read-policy", "ReadOnlyAccessECR")
],
inline_policies={
'nf-s3-bucket-access-policy': s3_workdir_bucket_policy_doc
})
cdk_tags.of(instance_role).add("Product", "PAVI") # type: ignore
cdk_tags.of(instance_role).add("Managed_by", "PAVI") # type: ignore

Expand Down Expand Up @@ -123,3 +117,33 @@ def __init__(self, scope: Stack, env_suffix: str, shared_work_dir_bucket: Option

cdk_tags.of(self.job_queue).add("Product", "PAVI") # type: ignore
cdk_tags.of(self.job_queue).add("Managed_by", "PAVI") # type: ignore

# Create the IAM policy to grant access to execute the nextflow pipeline using above AWS execution environment
nf_aws_execution_statements = s3_bucket_access_statements.copy()
nf_aws_execution_statements.append(
iam.PolicyStatement(
sid="AllowBatchJobMgmt",
effect=iam.Effect.ALLOW,
actions=['batch:CancelJob', 'batch:SubmitJob', 'batch:TerminateJob',
'batch:ListJobs', 'batch:DescribeJobs',
'batch:RegisterJobDefinition', 'batch:DescribeJobDefinitions'],
resources=['*']
)
)
nf_aws_execution_statements.append(
iam.PolicyStatement(
sid="AllowQueueAndComputeEnvDescribe",
effect=iam.Effect.ALLOW,
actions=['batch:DescribeJobQueues', 'batch:DescribeComputeEnvironments'],
resources=['*']
)
)

nf_aws_execution_policy = iam.ManagedPolicy(scope, 'pavi-nextflow-aws-execution-policy',
managed_policy_name='agr-pavi-pipeline-nf-aws-execution-policy',
description='Grant required access to PAVI execution environment to run nextflow pipelines using it.',
statements=nf_aws_execution_statements)
cdk_tags.of(nf_aws_execution_policy).add("Product", "PAVI") # type: ignore
cdk_tags.of(nf_aws_execution_policy).add("Managed_by", "PAVI") # type: ignore

self.nf_aws_execution_policy = nf_aws_execution_policy
4 changes: 2 additions & 2 deletions pipeline/aws_infra/tests/unit/test_cdk_infra_stack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@ def test_pipeline_nf_s3_bucket() -> None:
})


# The S3 bucket policy is assigned to the GH-actions role for PAVI, in addition to PAVI-managed resources.
# The Nextflow AWS execution policy is assigned to the GH-actions role for PAVI, in addition to PAVI-managed resources.
# Changing the name of this policy might require the gh-actions role to be updated to include the new role.
def test_pipeline_nf_s3_bucket_policy() -> None:
template.has_resource(type=ResourceType.of('AWS::IAM::ManagedPolicy').compliance_resource_type, props={
"Properties": {
"ManagedPolicyName": "agr-pavi-pipeline-nf-bucket-access"
"ManagedPolicyName": "agr-pavi-pipeline-nf-aws-execution-policy"
}
})

Expand Down