diff --git a/pulumi/AWSMegatests/.github/workflows/deploy-seqerakit.yml b/pulumi/AWSMegatests/.github/workflows/deploy-seqerakit.yml deleted file mode 100644 index 51e5d766..00000000 --- a/pulumi/AWSMegatests/.github/workflows/deploy-seqerakit.yml +++ /dev/null @@ -1,144 +0,0 @@ -# TODO We'll need to move this to the actual GitHub workflows directory, but this is just a placeholder anyways. -# We should merge our current setup and then make a PR updating it to the new setup that we want to have. -name: Deploy Seqerakit Infrastructure - -on: - push: - branches: [main] - paths: - - "seqerakit/**" - pull_request: - branches: [main] - paths: - - "seqerakit/**" - -jobs: - validate: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - defaults: - run: - working-directory: seqerakit - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Setup Seqerakit - uses: seqeralabs/setup-seqerakit@v1 - with: - token: ${{ secrets.TOWER_ACCESS_TOKEN }} - - - name: Load environment variables - run: | - # Install direnv for 1Password integration - sudo apt-get update && sudo apt-get install -y direnv - # Allow .envrc and load environment - direnv allow && eval "$(direnv export bash)" - - - name: Validate CPU environment - run: seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml --dryrun - - - name: Validate CPU ARM environment - run: seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml --dryrun - - - name: Validate GPU environment - run: seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml --dryrun - - - name: Comment PR with validation results - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - if: always() - with: - script: | - const output = ` - ## Seqerakit Validation Results ๐Ÿงช - - โœ… All compute environment configurations have been validated successfully. - - The following environments were tested: - - CPU Environment (aws_ireland_fusionv2_nvme_cpu) - - CPU ARM Environment (aws_ireland_fusionv2_nvme_cpu_ARM_snapshots) - - GPU Environment (aws_ireland_fusionv2_nvme_gpu_snapshots) - - Ready for deployment! ๐Ÿš€ - `; - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }); - - deploy: - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - defaults: - run: - working-directory: seqerakit - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Setup Seqerakit - uses: seqeralabs/setup-seqerakit@v1 - with: - token: ${{ secrets.TOWER_ACCESS_TOKEN }} - - - name: Load environment variables - run: | - # Install direnv for 1Password integration - sudo apt-get update && sudo apt-get install -y direnv - # Allow .envrc and load environment - direnv allow && eval "$(direnv export bash)" - - - name: Deploy CPU environment - run: seqerakit aws_ireland_fusionv2_nvme_cpu_current.yml - - - name: Deploy CPU ARM environment - run: seqerakit aws_ireland_fusionv2_nvme_cpu_arm_current.yml - - - name: Deploy GPU environment - run: seqerakit aws_ireland_fusionv2_nvme_gpu_current.yml - - - name: Notify deployment success - if: success() - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - const output = ` - ## Seqerakit Deployment Completed โœ… - - Successfully deployed all compute environments: - - โœ… CPU Environment (aws_ireland_fusionv2_nvme_cpu) - - โœ… CPU ARM Environment (aws_ireland_fusionv2_nvme_cpu_ARM_snapshots) - - โœ… GPU Environment (aws_ireland_fusionv2_nvme_gpu_snapshots) - - Infrastructure is now up to date! ๐ŸŽ‰ - `; - - github.rest.repos.createCommitComment({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: context.sha, - body: output - }); - - - name: Notify deployment failure - if: failure() - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - const output = ` - ## Seqerakit Deployment Failed โŒ - - Deployment failed for commit ${context.sha}. - - Please check the workflow logs for details and fix any issues. - `; - - github.rest.repos.createCommitComment({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: context.sha, - body: output - }); diff --git a/pulumi/AWSMegatests/Pulumi.dev.yaml b/pulumi/AWSMegatests/Pulumi.dev.yaml deleted file mode 100644 index 33c02c1a..00000000 --- a/pulumi/AWSMegatests/Pulumi.dev.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - aws:region: eu-west-1 diff --git a/pulumi/AWSMegatests/__main__.py b/pulumi/AWSMegatests/__main__.py index d17b86be..cf54dd59 100644 --- a/pulumi/AWSMegatests/__main__.py +++ b/pulumi/AWSMegatests/__main__.py @@ -1,135 +1,212 @@ -"""An AWS Python Pulumi program""" +"""An AWS Python Pulumi program for nf-core megatests infrastructure""" import pulumi -import pulumi_github as github -import pulumi_command as command -from pulumi_aws import s3 +import pulumi_aws as aws -# Create an AWS resource (S3 Bucket) -bucket = s3.Bucket("my-bucket") +# Import our modular components +from src.providers import ( + create_aws_provider, + create_github_provider, + create_seqera_provider, +) +from src.config import get_configuration +from src.infrastructure import create_s3_infrastructure, create_towerforge_credentials +from src.infrastructure import ( + deploy_seqera_environments_terraform, + get_compute_environment_ids_terraform, +) +from src.integrations import create_github_resources, create_github_credential +from src.integrations.workspace_participants_command import ( + create_individual_member_commands, +) -# Export the name of the bucket -pulumi.export("bucket_name", bucket.id) # type: ignore[attr-defined] +def main(): + """Main Pulumi program function""" -# Get secrets from 1Password using the CLI -def get_1password_secret(secret_ref: str) -> str: - """Get a secret from 1Password using the CLI""" - get_secret_cmd = command.local.Command( - f"get-1password-{secret_ref.replace('/', '-').replace(' ', '-')}", - create=f"op read '{secret_ref}'", - opts=pulumi.ResourceOptions(additional_secret_outputs=["stdout"]), - ) - return get_secret_cmd.stdout + # Step 1: Get configuration from ESC environment and config + config = get_configuration() + # Step 2: Create AWS, GitHub, and Seqera providers + # AWS provider uses ESC-provided credentials automatically + aws_provider = create_aws_provider() + github_provider = create_github_provider(config["github_token"]) -# Get secrets from 1Password -tower_access_token = get_1password_secret( - "op://Employee/Seqera Platform Token/credential" -) -github_token = get_1password_secret("op://Employee/Github Token nf-core/credential") - -# Get workspace ID from Tower CLI -workspace_cmd = command.local.Command( - "get-workspace-id", - create="tw -o nf-core workspaces list --format json | jq -r '.[] | select(.name==\"AWSmegatests\") | .id'", - environment={ - "TOWER_ACCESS_TOKEN": tower_access_token, - "ORGANIZATION_NAME": "nf-core", - }, - opts=pulumi.ResourceOptions(additional_secret_outputs=["stdout"]), -) -workspace_id = workspace_cmd.stdout - - -# Get compute environment IDs using Tower CLI -def get_compute_env_id(env_name: str, display_name: str) -> str: - """Get compute environment ID by name""" - get_env_cmd = command.local.Command( - f"get-compute-env-{env_name}", - create=f"tw -o nf-core -w AWSmegatests compute-envs list --format json | jq -r '.[] | select(.name==\"{display_name}\") | .id'", - environment={ - "TOWER_ACCESS_TOKEN": tower_access_token, - "ORGANIZATION_NAME": "nf-core", - "WORKSPACE_NAME": "AWSmegatests", - }, - opts=pulumi.ResourceOptions(additional_secret_outputs=["stdout"]), + # Create Seqera provider early for credential upload + seqera_provider = create_seqera_provider(config) + + # Step 3.5: Create GitHub fine-grained credential in Seqera Platform + # This allows Platform to pull pipeline repositories without hitting GitHub rate limits + github_credential, github_credential_id = create_github_credential( + seqera_provider=seqera_provider, + workspace_id=int(config["tower_workspace_id"]), + github_token=config.get("platform_github_org_token", ""), ) - return get_env_cmd.stdout + # Step 4: Set up S3 infrastructure + s3_resources = create_s3_infrastructure(aws_provider) + nf_core_awsmegatests_bucket = s3_resources["bucket"] + # Note: lifecycle_configuration is managed manually, not used in exports + + # Step 5: Create TowerForge IAM credentials and upload to Seqera Platform + ( + towerforge_access_key_id, + towerforge_access_key_secret, + seqera_credentials_id, + seqera_credential_resource, + iam_policy_hash, + ) = create_towerforge_credentials( + aws_provider, + nf_core_awsmegatests_bucket, + seqera_provider, + float(config["tower_workspace_id"]), + ) -# Get compute environment IDs for each environment -cpu_compute_env_id = get_compute_env_id("cpu", "aws_ireland_fusionv2_nvme_cpu") -gpu_compute_env_id = get_compute_env_id( - "gpu", "aws_ireland_fusionv2_nvme_gpu_snapshots" -) -arm_compute_env_id = get_compute_env_id( - "arm", "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots" -) + # Step 6: Deploy Seqera Platform compute environments using Terraform provider + # Deploy using Seqera Terraform provider with dynamic credentials ID + terraform_resources = deploy_seqera_environments_terraform( + config, + seqera_credentials_id, # Dynamic TowerForge credentials ID from Seqera Platform + seqera_provider, # Reuse existing Seqera provider + seqera_credential_resource, # Seqera credential resource for dependency + iam_policy_hash, # IAM policy hash to force CE recreation on policy changes + ) -# Create GitHub provider -github_provider = github.Provider("github", token=github_token) + # Get compute environment IDs from Terraform provider + compute_env_ids = get_compute_environment_ids_terraform(terraform_resources) + deployment_method = "terraform-provider" + + # Step 8: Create GitHub resources + # Full GitHub integration enabled - creates both variables and secrets + github_resources = create_github_resources( + github_provider, + compute_env_ids, + config["tower_workspace_id"], + tower_access_token=config["tower_access_token"], + ) -# Create org-level GitHub secrets for compute environment IDs -cpu_secret = github.ActionsOrganizationSecret( - "tower-compute-env-cpu", - visibility="private", - secret_name="TOWER_COMPUTE_ENV_CPU", - plaintext_value=cpu_compute_env_id, - opts=pulumi.ResourceOptions(provider=github_provider), -) + # Step 9: Add nf-core team members as workspace participants with role precedence + # Core team โ†’ OWNER role, Maintainers โ†’ MAINTAIN role + # Individual member tracking provides granular status per team member + + # Create team data setup and individual member tracking commands + setup_cmd, member_commands = create_individual_member_commands( + workspace_id=int(config["tower_workspace_id"]), + token=config["tower_access_token"], + github_token=config["github_token"], + opts=pulumi.ResourceOptions( + depends_on=[seqera_credential_resource] # Ensure credentials exist first + ), + ) -gpu_secret = github.ActionsOrganizationSecret( - "tower-compute-env-gpu", - visibility="private", - secret_name="TOWER_COMPUTE_ENV_GPU", - plaintext_value=gpu_compute_env_id, - opts=pulumi.ResourceOptions(provider=github_provider), -) + # Option B: Native Pulumi with HTTP calls (more integrated) + # Uncomment to use this approach instead: + # maintainer_emails = load_maintainer_emails_static() + # participants_results = create_workspace_participants_simple( + # workspace_id=pulumi.Output.from_input(config["tower_workspace_id"]), + # token=pulumi.Output.from_input(config["tower_access_token"]), + # maintainer_emails=maintainer_emails + # ) + + # Exports - All within proper Pulumi program context + pulumi.export( + "megatests_bucket", + { + "name": nf_core_awsmegatests_bucket.bucket, + "arn": nf_core_awsmegatests_bucket.arn, + "region": "eu-west-1", + "lifecycle_configuration": "managed-manually", + }, + ) -arm_secret = github.ActionsOrganizationSecret( - "tower-compute-env-arm", - visibility="private", - secret_name="TOWER_COMPUTE_ENV_ARM", - plaintext_value=arm_compute_env_id, - opts=pulumi.ResourceOptions(provider=github_provider), -) + pulumi.export( + "github_resources", + { + "variables": { + k: v.id for k, v in github_resources.get("variables", {}).items() + } + if github_resources.get("variables") + else {}, + "secrets": {k: v.id for k, v in github_resources.get("secrets", {}).items()} + if github_resources.get("secrets") + else {}, + "manual_secret_commands": github_resources.get("gh_cli_commands", []), + "note": github_resources.get("note", ""), + "workaround_info": { + "issue_url": "https://github.com/pulumi/pulumi-github/issues/250", + "workaround": "Variables via Pulumi with delete_before_replace, secrets via manual gh CLI", + "instructions": "Run the commands in 'manual_secret_commands' to set GitHub secrets", + }, + }, + ) -# Create org-level GitHub secret for Seqera Platform API token -seqera_token_secret = github.ActionsOrganizationSecret( - "tower-access-token", - visibility="private", - secret_name="TOWER_ACCESS_TOKEN", - plaintext_value=tower_access_token, - opts=pulumi.ResourceOptions(provider=github_provider), -) + pulumi.export("compute_env_ids", compute_env_ids) + pulumi.export("workspace_id", config["tower_workspace_id"]) + pulumi.export("deployment_method", deployment_method) + + # Export GitHub credential information + pulumi.export( + "github_credential", + { + "credential_id": github_credential_id, + "credential_name": "nf-core-github-finegrained", + "description": "Fine-grained GitHub token to avoid rate limits when Platform pulls pipeline repositories", + "provider_type": "github", + "base_url": "https://github.com/nf-core/", + "workspace_id": config["tower_workspace_id"], + "purpose": "Prevents GitHub API rate limiting during pipeline repository access", + }, + ) -# Create org-level GitHub secret for workspace ID -workspace_id_secret = github.ActionsOrganizationSecret( - "tower-workspace-id", - visibility="private", - secret_name="TOWER_WORKSPACE_ID", - plaintext_value=workspace_id, - opts=pulumi.ResourceOptions(provider=github_provider), -) + # Export Terraform provider resources + pulumi.export( + "terraform_resources", + { + "cpu_env_id": terraform_resources["cpu_env"].compute_env_id, + "gpu_env_id": terraform_resources["gpu_env"].compute_env_id, + "arm_env_id": terraform_resources["arm_env"].compute_env_id, + "deployment_method": "seqera-terraform-provider", + }, + ) -# Export the created secrets -pulumi.export( - "github_secrets", - { - "compute_env_cpu": cpu_secret.secret_name, - "compute_env_gpu": gpu_secret.secret_name, - "compute_env_arm": arm_secret.secret_name, - "tower_access_token": seqera_token_secret.secret_name, - "tower_workspace_id": workspace_id_secret.secret_name, - }, -) + towerforge_resources = { + "user": { + "name": "TowerForge-AWSMegatests", + "arn": f"arn:aws:iam::{aws.get_caller_identity(opts=pulumi.InvokeOptions(provider=aws_provider)).account_id}:user/TowerForge-AWSMegatests", + }, + "access_key_id": towerforge_access_key_id, + "access_key_secret": towerforge_access_key_secret, + "policies": { + "forge_policy_name": "TowerForge-Forge-Policy", + "launch_policy_name": "TowerForge-Launch-Policy", + "s3_policy_name": "TowerForge-S3-Policy", + }, + } + pulumi.export("towerforge_iam", towerforge_resources) + + # Export workspace participants management information with individual member tracking + pulumi.export( + "workspace_participants", + { + "setup_command_id": setup_cmd.id, + "setup_status": setup_cmd.stdout, + "individual_member_commands": { + username: { + "command_id": cmd.id, + "status": cmd.stdout, # Contains STATUS lines from script + "github_username": username, + } + for username, cmd in member_commands.items() + }, + "total_tracked_members": len(member_commands), + "workspace_id": config["tower_workspace_id"], + "note": "Automated team data setup with individual member sync commands and privacy protection", + "privacy": "Email data generated at runtime, never committed to git", + "todo": "Replace with seqera_workspace_participant resources when available", + }, + ) -# Export compute environment IDs for reference -pulumi.export( - "compute_env_ids", - {"cpu": cpu_compute_env_id, "gpu": gpu_compute_env_id, "arm": arm_compute_env_id}, -) -# Export workspace ID for reference -pulumi.export("workspace_id", workspace_id) +# Proper Pulumi program entry point +if __name__ == "__main__": + main() diff --git a/pulumi/AWSMegatests/seqerakit/.envrc b/pulumi/AWSMegatests/seqerakit/.envrc deleted file mode 100644 index 92331d84..00000000 --- a/pulumi/AWSMegatests/seqerakit/.envrc +++ /dev/null @@ -1,18 +0,0 @@ -export OP_ACCOUNT=nf-core - -# Load 1Password integration for direnv -source_url "https://github.com/tmatilai/direnv-1password/raw/v1.0.1/1password.sh" \ - "sha256-4dmKkmlPBNXimznxeehplDfiV+CvJiIzg7H1Pik4oqY=" - -# Load secrets from 1Password -from_op TOWER_ACCESS_TOKEN="op://Employee/Seqera Platform Token/credential" -from_op AWS_ACCESS_KEY_ID="op://Dev/AWS Tower Test Credentials/access key id" -from_op AWS_SECRET_ACCESS_KEY="op://Dev/AWS Tower Test Credentials/secret access key" - -# Static configuration variables -export ORGANIZATION_NAME="nf-core" -export WORKSPACE_NAME="AWSmegatests" -export AWS_CREDENTIALS_NAME="tower-awstest" -export AWS_REGION="eu-west-1" -export AWS_WORK_DIR="s3://nf-core-awsmegatests" -export AWS_COMPUTE_ENV_ALLOWED_BUCKETS="s3://ngi-igenomes,s3://annotation-cache" diff --git a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml b/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml deleted file mode 100644 index 2288cc70..00000000 --- a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_arm_current.yml +++ /dev/null @@ -1,7 +0,0 @@ -compute-envs: - - name: "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-cpu-arm.json" - overwrite: True diff --git a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml b/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml deleted file mode 100644 index 58131e3a..00000000 --- a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_cpu_current.yml +++ /dev/null @@ -1,7 +0,0 @@ -compute-envs: - - name: "aws_ireland_fusionv2_nvme_cpu" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-cpu.json" - overwrite: True diff --git a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml b/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml deleted file mode 100644 index e129b20e..00000000 --- a/pulumi/AWSMegatests/seqerakit/aws_ireland_fusionv2_nvme_gpu_current.yml +++ /dev/null @@ -1,7 +0,0 @@ -compute-envs: - - name: "aws_ireland_fusionv2_nvme_gpu_snapshots" - workspace: "$ORGANIZATION_NAME/$WORKSPACE_NAME" - credentials: "$AWS_CREDENTIALS_NAME" - wait: "AVAILABLE" - file-path: "./current-env-gpu.json" - overwrite: True diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-arm.config b/pulumi/AWSMegatests/seqerakit/configs/nextflow-arm.config new file mode 100644 index 00000000..6a944ba1 --- /dev/null +++ b/pulumi/AWSMegatests/seqerakit/configs/nextflow-arm.config @@ -0,0 +1,17 @@ +// Nextflow configuration for ARM CPU compute environments +// Includes base configuration and ARM-specific settings + +includeConfig 'nextflow-base.config' + +process { + publishDir = [ + path: { params.outdir }, + mode: 'copy', + tags: [ + 'compute_env': 'aws_ireland_fusionv2_nvme_cpu_ARM_snapshots', + 'architecture': 'arm64', + 'fusion': 'enabled', + 'fusionSnapshots': 'enabled' + ] + ] +} \ No newline at end of file diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-cpu.config b/pulumi/AWSMegatests/seqerakit/configs/nextflow-cpu.config new file mode 100644 index 00000000..de3a74b5 --- /dev/null +++ b/pulumi/AWSMegatests/seqerakit/configs/nextflow-cpu.config @@ -0,0 +1,17 @@ +// Nextflow configuration for CPU compute environments +// Includes base configuration and CPU-specific settings + +includeConfig 'nextflow-base.config' + +process { + publishDir = [ + path: { params.outdir }, + mode: 'copy', + tags: [ + 'compute_env': 'aws_ireland_fusionv2_nvme_cpu_snapshots', + 'architecture': 'x86_64', + 'fusion': 'enabled', + 'fusionSnapshots': 'enabled' + ] + ] +} \ No newline at end of file diff --git a/pulumi/AWSMegatests/seqerakit/configs/nextflow-gpu.config b/pulumi/AWSMegatests/seqerakit/configs/nextflow-gpu.config new file mode 100644 index 00000000..b9ca2330 --- /dev/null +++ b/pulumi/AWSMegatests/seqerakit/configs/nextflow-gpu.config @@ -0,0 +1,18 @@ +// Nextflow configuration for GPU compute environments +// Includes base configuration and GPU-specific settings + +includeConfig 'nextflow-base.config' + +process { + publishDir = [ + path: { params.outdir }, + mode: 'copy', + tags: [ + 'compute_env': 'aws_ireland_fusionv2_nvme_gpu_snapshots', + 'architecture': 'x86_64', + 'gpu': 'enabled', + 'fusion': 'enabled', + 'fusionSnapshots': 'enabled' + ] + ] +} \ No newline at end of file diff --git a/pulumi/AWSMegatests/src/__init__.py b/pulumi/AWSMegatests/src/__init__.py new file mode 100644 index 00000000..b5e4da14 --- /dev/null +++ b/pulumi/AWSMegatests/src/__init__.py @@ -0,0 +1,8 @@ +"""AWS Megatests Infrastructure Package + +This package provides modular infrastructure components for the nf-core AWS Megatests +Pulumi project, including provider configurations, infrastructure resources, +and third-party integrations. +""" + +__version__ = "1.0.0" diff --git a/pulumi/AWSMegatests/src/config/__init__.py b/pulumi/AWSMegatests/src/config/__init__.py new file mode 100644 index 00000000..0d879069 --- /dev/null +++ b/pulumi/AWSMegatests/src/config/__init__.py @@ -0,0 +1,5 @@ +"""Configuration management for AWS Megatests infrastructure.""" + +from .settings import get_configuration, ConfigurationError + +__all__ = ["get_configuration", "ConfigurationError"] diff --git a/pulumi/AWSMegatests/src/config/settings.py b/pulumi/AWSMegatests/src/config/settings.py new file mode 100644 index 00000000..335c5534 --- /dev/null +++ b/pulumi/AWSMegatests/src/config/settings.py @@ -0,0 +1,117 @@ +"""Configuration management for AWS Megatests infrastructure using Pulumi ESC.""" + +import os +from typing import Dict, Any, Optional +from dataclasses import dataclass + +from ..utils.constants import DEFAULT_ENV_VARS + + +class ConfigurationError(Exception): + """Exception raised when configuration validation fails.""" + + pass + + +@dataclass +class InfrastructureConfig: + """Typed configuration for AWS Megatests infrastructure. + + Attributes: + tower_access_token: Seqera Platform access token + tower_workspace_id: Seqera Platform workspace ID + github_token: GitHub personal access token (classic) + platform_github_org_token: GitHub fine-grained token to avoid rate limits when pulling pipelines + """ + + tower_access_token: Optional[str] + tower_workspace_id: str + github_token: Optional[str] + platform_github_org_token: Optional[str] + + def validate(self) -> None: + """Validate configuration values. + + Raises: + ConfigurationError: If required configuration is missing or invalid + """ + missing_vars = [] + + if not self.tower_access_token: + missing_vars.append("TOWER_ACCESS_TOKEN") + + if not self.github_token: + missing_vars.append("GITHUB_TOKEN") + + # Validate workspace ID is numeric + if ( + not self.tower_workspace_id + or not self.tower_workspace_id.replace(".", "").isdigit() + ): + missing_vars.append("TOWER_WORKSPACE_ID (must be numeric)") + + if missing_vars: + raise ConfigurationError( + f"Missing or invalid required environment variables: {', '.join(missing_vars)}. " + "Please ensure these are set in your ESC environment." + ) + + +def _get_env_var_with_fallback( + var_name: str, fallback: Optional[str] = None +) -> Optional[str]: + """Get environment variable with optional fallback. + + Args: + var_name: Name of the environment variable + fallback: Optional fallback value if variable is not set + + Returns: + Optional[str]: Environment variable value or fallback + """ + value = os.environ.get(var_name) + if not value and fallback: + print( + f"Warning: {var_name} not found in ESC environment, using fallback: {fallback}" + ) + return fallback + return value + + +def get_configuration() -> Dict[str, Any]: + """Get configuration values from ESC environment variables. + + All configuration comes from ESC environment variables which are automatically + set when the ESC environment is imported. + + Returns: + Dict[str, Any]: Configuration dictionary compatible with existing code + + Raises: + ConfigurationError: If required configuration is missing or invalid + """ + # Get workspace ID from environment or fall back to default + workspace_id = _get_env_var_with_fallback( + "TOWER_WORKSPACE_ID", DEFAULT_ENV_VARS.get("TOWER_WORKSPACE_ID") + ) + + # Create typed configuration object + config = InfrastructureConfig( + tower_access_token=os.environ.get("TOWER_ACCESS_TOKEN"), + tower_workspace_id=workspace_id or "", + github_token=os.environ.get("GITHUB_TOKEN"), + platform_github_org_token=os.environ.get("PLATFORM_GITHUB_ORG_TOKEN"), + ) + + # Validate configuration + config.validate() + + # Return dictionary format for backward compatibility + return { + "tower_access_token": config.tower_access_token, + "tower_workspace_id": config.tower_workspace_id, + "github_token": config.github_token, + "platform_github_org_token": config.platform_github_org_token, + # AWS credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) + # are automatically handled by ESC and picked up by the AWS provider + } diff --git a/pulumi/AWSMegatests/src/infrastructure/__init__.py b/pulumi/AWSMegatests/src/infrastructure/__init__.py new file mode 100644 index 00000000..f342fd1e --- /dev/null +++ b/pulumi/AWSMegatests/src/infrastructure/__init__.py @@ -0,0 +1,16 @@ +"""Infrastructure components for AWS Megatests.""" + +from .s3 import create_s3_infrastructure +from .credentials import create_towerforge_credentials, get_towerforge_resources +from .compute_environments import ( + deploy_seqera_environments_terraform, + get_compute_environment_ids_terraform, +) + +__all__ = [ + "create_s3_infrastructure", + "create_towerforge_credentials", + "get_towerforge_resources", + "deploy_seqera_environments_terraform", + "get_compute_environment_ids_terraform", +] diff --git a/pulumi/AWSMegatests/src/infrastructure/compute_environments.py b/pulumi/AWSMegatests/src/infrastructure/compute_environments.py new file mode 100644 index 00000000..ff254467 --- /dev/null +++ b/pulumi/AWSMegatests/src/infrastructure/compute_environments.py @@ -0,0 +1,387 @@ +"""Seqera Platform compute environment deployment using Seqera Terraform provider.""" + +import json +import os +from typing import Dict, Any, Optional + +import pulumi +import pulumi_seqera as seqera + +from ..utils.constants import ( + COMPUTE_ENV_NAMES, + COMPUTE_ENV_DESCRIPTIONS, + CONFIG_FILES, + NEXTFLOW_CONFIG_FILES, + DEFAULT_COMPUTE_ENV_CONFIG, + DEFAULT_FORGE_CONFIG, + TIMEOUTS, + ERROR_MESSAGES, +) + + +class ComputeEnvironmentError(Exception): + """Exception raised when compute environment operations fail.""" + + pass + + +class ConfigurationError(Exception): + """Exception raised when configuration loading fails.""" + + pass + + +def load_nextflow_config(env_type: str) -> str: + """Load and merge Nextflow configuration from base and environment-specific files. + + Args: + env_type: Environment type (cpu, gpu, arm) + + Returns: + str: Merged Nextflow configuration content + + Raises: + ConfigurationError: If file loading fails + """ + config_file = NEXTFLOW_CONFIG_FILES.get(env_type) + if not config_file: + raise ConfigurationError( + f"No Nextflow config file defined for environment type: {env_type}" + ) + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Nextflow config file not found: {config_file}") + + # Load base configuration + base_config_file = os.path.join( + os.path.dirname(config_file), "nextflow-base.config" + ) + base_config = "" + if os.path.exists(base_config_file): + try: + with open(base_config_file, "r") as f: + base_config = f.read().strip() + except Exception as e: + raise ConfigurationError( + f"Failed to read base Nextflow config file {base_config_file}: {e}" + ) + + # Load environment-specific configuration + try: + with open(config_file, "r") as f: + env_config = f.read().strip() + except Exception as e: + raise ConfigurationError( + f"Failed to read Nextflow config file {config_file}: {e}" + ) + + # Remove includeConfig line from environment config since we're injecting base config + env_config_lines = env_config.split("\n") + env_config_filtered = [ + line + for line in env_config_lines + if not line.strip().startswith("includeConfig") + ] + env_config_clean = "\n".join(env_config_filtered) + + # Merge base config with environment-specific config + if base_config: + merged_config = f"{base_config}\n\n{env_config_clean}" + else: + merged_config = env_config_clean + + return merged_config.strip() + + +def load_config_file(filename: str) -> Dict[str, Any]: + """Load configuration file with comprehensive error handling. + + Args: + filename: Path to the JSON configuration file + + Returns: + Dict[str, Any]: Loaded configuration data + + Raises: + ConfigurationError: If file loading or parsing fails + """ + if not os.path.exists(filename): + raise FileNotFoundError( + ERROR_MESSAGES["config_file_not_found"].format(filename) + ) + + with open(filename, "r") as f: + config_data = json.load(f) + + return config_data + + +def create_forge_config( + config_args: Dict[str, Any], +) -> seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs: + """Create forge configuration for AWS Batch compute environment. + + Args: + config_args: Configuration arguments from JSON file + + Returns: + seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs: Forge configuration + """ + forge_data = config_args.get("forge", {}) + + return seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs( + type=forge_data.get("type", DEFAULT_FORGE_CONFIG["type"]), + min_cpus=forge_data.get("minCpus", DEFAULT_FORGE_CONFIG["minCpus"]), + max_cpus=forge_data.get("maxCpus", DEFAULT_FORGE_CONFIG["maxCpus"]), + gpu_enabled=forge_data.get("gpuEnabled", DEFAULT_FORGE_CONFIG["gpuEnabled"]), + instance_types=forge_data.get( + "instanceTypes", DEFAULT_FORGE_CONFIG["instanceTypes"] + ), + subnets=forge_data.get("subnets", DEFAULT_FORGE_CONFIG["subnets"]), + security_groups=forge_data.get( + "securityGroups", DEFAULT_FORGE_CONFIG["securityGroups"] + ), + dispose_on_deletion=forge_data.get( + "disposeOnDeletion", DEFAULT_FORGE_CONFIG["disposeOnDeletion"] + ), + allow_buckets=forge_data.get( + "allowBuckets", DEFAULT_FORGE_CONFIG["allowBuckets"] + ), + efs_create=forge_data.get("efsCreate", DEFAULT_FORGE_CONFIG["efsCreate"]), + ebs_boot_size=forge_data.get( + "ebsBootSize", DEFAULT_FORGE_CONFIG["ebsBootSize"] + ), + fargate_head_enabled=forge_data.get( + "fargateHeadEnabled", DEFAULT_FORGE_CONFIG["fargateHeadEnabled"] + ), + arm64_enabled=forge_data.get( + "arm64Enabled", DEFAULT_FORGE_CONFIG["arm64Enabled"] + ), + ) + + +def create_compute_environment( + provider: seqera.Provider, + name: str, + credentials_id: str, + workspace_id: float, + config_args: Dict[str, Any], + env_type: str, + description: Optional[str] = None, + depends_on: Optional[list] = None, + iam_policy_version: Optional[str] = None, +) -> seqera.ComputeEnv: + """Create a Seqera compute environment using Terraform provider with error handling. + + Args: + provider: Configured Seqera provider instance + name: Name for the compute environment + credentials_id: Seqera credentials ID + workspace_id: Seqera workspace ID + config_args: Configuration arguments from JSON file + env_type: Environment type (cpu, gpu, arm) for loading external nextflow config + description: Optional description for the compute environment + depends_on: Optional list of resources this compute environment depends on + iam_policy_version: Optional IAM policy version hash to trigger recreation on policy changes + + Returns: + seqera.ComputeEnv: Created compute environment resource + + Raises: + ComputeEnvironmentError: If compute environment creation fails + ValueError: If required parameters are missing + ConfigurationError: If nextflow config loading fails + """ + pulumi.log.info(f"Creating compute environment: {name}") + + # Validate input parameters + if not name or not credentials_id: + raise ValueError(ERROR_MESSAGES["missing_compute_env_params"].format(name)) + + if not config_args: + raise ValueError(ERROR_MESSAGES["missing_config_args"].format(name)) + + # Create the forge configuration + forge_config = create_forge_config(config_args) + + # Load Nextflow configuration from external file + nextflow_config = load_nextflow_config(env_type) + + # Create AWS Batch configuration + aws_batch_config = seqera.ComputeEnvComputeEnvConfigAwsBatchArgs( + region=config_args.get("region", DEFAULT_COMPUTE_ENV_CONFIG["region"]), + work_dir=config_args.get("workDir", DEFAULT_COMPUTE_ENV_CONFIG["workDir"]), + forge=forge_config, + wave_enabled=config_args.get( + "waveEnabled", DEFAULT_COMPUTE_ENV_CONFIG["waveEnabled"] + ), + fusion2_enabled=config_args.get( + "fusion2Enabled", DEFAULT_COMPUTE_ENV_CONFIG["fusion2Enabled"] + ), + nvnme_storage_enabled=config_args.get( + "nvnmeStorageEnabled", DEFAULT_COMPUTE_ENV_CONFIG["nvnmeStorageEnabled"] + ), + fusion_snapshots=config_args.get( + "fusionSnapshots", DEFAULT_COMPUTE_ENV_CONFIG["fusionSnapshots"] + ), + nextflow_config=nextflow_config, # Use external config file + ) + + # Create the compute environment configuration + compute_env_config = seqera.ComputeEnvComputeEnvConfigArgs( + aws_batch=aws_batch_config + ) + + # Create the compute environment args + compute_env_args = seqera.ComputeEnvComputeEnvArgs( + name=name, + platform="aws-batch", + credentials_id=credentials_id, + config=compute_env_config, + description=description, + ) + + # Add IAM policy version to compute environment description to trigger recreation on policy changes + if iam_policy_version: + # Append policy version hash to description to force recreation when IAM policies change + policy_suffix = f" (IAM Policy Version: {iam_policy_version[:8]})" + if description: + compute_env_args = seqera.ComputeEnvComputeEnvArgs( + name=name, + platform="aws-batch", + credentials_id=credentials_id, + config=compute_env_config, + description=f"{description}{policy_suffix}", + ) + else: + compute_env_args = seqera.ComputeEnvComputeEnvArgs( + name=name, + platform="aws-batch", + credentials_id=credentials_id, + config=compute_env_config, + description=f"Compute environment{policy_suffix}", + ) + + # Create the compute environment resource + resource_options = pulumi.ResourceOptions( + provider=provider, + # Force delete before replace to avoid name conflicts + delete_before_replace=True, + # Add custom timeout for compute environment creation + custom_timeouts=pulumi.CustomTimeouts( + create=TIMEOUTS["compute_env_create"], + update=TIMEOUTS["compute_env_update"], + delete=TIMEOUTS["compute_env_delete"], + ), + ) + + # Add dependencies if specified + if depends_on: + resource_options.depends_on = depends_on + + compute_env = seqera.ComputeEnv( + name, + compute_env=compute_env_args, + workspace_id=workspace_id, + opts=resource_options, + ) + + return compute_env + + +def deploy_seqera_environments_terraform( + config: Dict[str, Any], + towerforge_credentials_id: str, + seqera_provider: Optional[seqera.Provider] = None, + seqera_credential_resource: Optional[seqera.Credential] = None, + iam_policy_hash: Optional[str] = None, +) -> Dict[str, Any]: + """Deploy Seqera Platform compute environments using Terraform provider. + + Args: + config: Configuration dictionary + towerforge_credentials_id: Dynamic TowerForge credentials ID + seqera_provider: Optional existing Seqera provider instance + seqera_credential_resource: Optional Seqera credential resource for dependency + iam_policy_hash: Optional IAM policy hash to force recreation on policy changes + + Returns: + Dict[str, Any]: Dictionary containing created compute environments and provider + + Raises: + ConfigurationError: If configuration loading fails + ComputeEnvironmentError: If compute environment creation fails + ValueError: If workspace ID is invalid + """ + pulumi.log.info( + "Starting Seqera compute environment deployment using Terraform provider" + ) + + # Use provided seqera provider or create a new one + if seqera_provider is not None: + provider = seqera_provider + pulumi.log.info("Using existing Seqera provider") + else: + # Import here to avoid circular imports + from ..providers.seqera import create_seqera_provider + + provider = create_seqera_provider(config) + + # Load all configuration files + cpu_config = load_config_file(CONFIG_FILES["cpu"]) + gpu_config = load_config_file(CONFIG_FILES["gpu"]) + arm_config = load_config_file(CONFIG_FILES["arm"]) + + # Validate workspace ID + workspace_id = float(config["tower_workspace_id"]) + + # Create all three compute environments + environments = {} + + # Set up dependencies - compute environments depend on Seqera credential resource + depends_on_resources = [] + if seqera_credential_resource: + depends_on_resources.append(seqera_credential_resource) + + for env_type, config_data in [ + ("cpu", cpu_config), + ("gpu", gpu_config), + ("arm", arm_config), + ]: + env_name = COMPUTE_ENV_NAMES[env_type] + description = COMPUTE_ENV_DESCRIPTIONS[env_type] + + environments[f"{env_type}_env"] = create_compute_environment( + provider=provider, + name=env_name, + credentials_id=towerforge_credentials_id, + workspace_id=workspace_id, + config_args=config_data, + env_type=env_type, + description=description, + depends_on=depends_on_resources if depends_on_resources else None, + iam_policy_version=iam_policy_hash, + ) + + return { + **environments, + "provider": provider, + } + + +def get_compute_environment_ids_terraform( + terraform_resources: Dict[str, Any], +) -> Dict[str, Any]: + """Extract compute environment IDs from Terraform provider resources. + + Args: + terraform_resources: Dictionary containing terraform resources + + Returns: + Dict[str, Any]: Dictionary mapping environment types to their IDs + """ + return { + "cpu": terraform_resources["cpu_env"].compute_env_id, + "gpu": terraform_resources["gpu_env"].compute_env_id, + "arm": terraform_resources["arm_env"].compute_env_id, + } diff --git a/pulumi/AWSMegatests/src/infrastructure/credentials.py b/pulumi/AWSMegatests/src/infrastructure/credentials.py new file mode 100644 index 00000000..d133dec3 --- /dev/null +++ b/pulumi/AWSMegatests/src/infrastructure/credentials.py @@ -0,0 +1,508 @@ +"""TowerForge IAM credentials management module. + +This module creates and manages IAM resources for TowerForge AWS operations, +including policies for Forge operations, Launch operations, and S3 bucket access. +It also uploads the credentials to Seqera Platform for use by compute environments. +""" + +import json +import hashlib +from typing import Optional, Tuple, Dict, Any + +import pulumi +import pulumi_aws as aws +import pulumi_seqera as seqera + +from ..utils.constants import ( + TOWERFORGE_USER_NAME, + TOWERFORGE_POLICY_NAMES, + TOWERFORGE_CREDENTIAL_NAME, + TOWERFORGE_CREDENTIAL_DESCRIPTION, + TIMEOUTS, +) + + +class CredentialError(Exception): + """Exception raised when credential operations fail.""" + + pass + + +def _create_forge_policy_document() -> Dict[str, Any]: + """Create TowerForge Forge Policy document with comprehensive permissions.""" + return { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TowerForge0", + "Effect": "Allow", + "Action": [ + "ssm:GetParameters", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:CreateRole", + "iam:DeleteRole", + "iam:AttachRolePolicy", + "iam:DetachRolePolicy", + "iam:PutRolePolicy", + "iam:DeleteRolePolicy", + "iam:PassRole", + "iam:TagRole", + "iam:TagInstanceProfile", + "iam:ListRolePolicies", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "batch:CreateComputeEnvironment", + "batch:UpdateComputeEnvironment", + "batch:DeleteComputeEnvironment", + "batch:CreateJobQueue", + "batch:UpdateJobQueue", + "batch:DeleteJobQueue", + "batch:DescribeComputeEnvironments", + "batch:DescribeJobQueues", + "fsx:CreateFileSystem", + "fsx:DeleteFileSystem", + "fsx:DescribeFileSystems", + "fsx:TagResource", + "ec2:DescribeSecurityGroups", + "ec2:DescribeAccountAttributes", + "ec2:DescribeSubnets", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:CreateLaunchTemplate", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeKeyPairs", + "ec2:DescribeVpcs", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:GetEbsEncryptionByDefault", + "efs:CreateFileSystem", + "efs:DeleteFileSystem", + "efs:DescribeFileSystems", + "efs:CreateMountTarget", + "efs:DeleteMountTarget", + "efs:DescribeMountTargets", + "efs:ModifyFileSystem", + "efs:PutLifecycleConfiguration", + "efs:TagResource", + "elasticfilesystem:CreateFileSystem", + "elasticfilesystem:DeleteFileSystem", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:CreateMountTarget", + "elasticfilesystem:DeleteMountTarget", + "elasticfilesystem:DescribeMountTargets", + "elasticfilesystem:UpdateFileSystem", + "elasticfilesystem:PutLifecycleConfiguration", + "elasticfilesystem:TagResource", + ], + "Resource": "*", + }, + { + "Sid": "TowerLaunch0", + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*", + "batch:DescribeJobQueues", + "batch:CancelJob", + "batch:SubmitJob", + "batch:ListJobs", + "batch:TagResource", + "batch:DescribeComputeEnvironments", + "batch:TerminateJob", + "batch:DescribeJobs", + "batch:RegisterJobDefinition", + "batch:DescribeJobDefinitions", + "ecs:DescribeTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceAttribute", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeImages", + "logs:Describe*", + "logs:Get*", + "logs:List*", + "logs:StartQuery", + "logs:StopQuery", + "logs:TestMetricFilter", + "logs:FilterLogEvents", + "ses:SendRawEmail", + "secretsmanager:ListSecrets", + ], + "Resource": "*", + }, + ], + } + + +def _create_launch_policy_document() -> Dict[str, Any]: + """Create TowerForge Launch Policy document with limited permissions.""" + return { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TowerLaunch0", + "Effect": "Allow", + "Action": [ + "batch:DescribeJobQueues", + "batch:CancelJob", + "batch:SubmitJob", + "batch:ListJobs", + "batch:TagResource", + "batch:DescribeComputeEnvironments", + "batch:TerminateJob", + "batch:DescribeJobs", + "batch:RegisterJobDefinition", + "batch:DescribeJobDefinitions", + "ecs:DescribeTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceAttribute", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeImages", + "logs:Describe*", + "logs:Get*", + "logs:List*", + "logs:StartQuery", + "logs:StopQuery", + "logs:TestMetricFilter", + "logs:FilterLogEvents", + "ses:SendRawEmail", + "secretsmanager:ListSecrets", + ], + "Resource": "*", + } + ], + } + + +def _create_s3_policy_document(bucket_arn: str) -> Dict[str, Any]: + """Create S3 bucket access policy document with multipart upload support. + + Includes permissions for: + - Basic bucket operations (list, get location) + - Object operations (get, put, tag, delete) + - Multipart upload operations (for large files >5GB) + + Args: + bucket_arn: ARN of the S3 bucket to grant access to + + Returns: + Dict[str, Any]: S3 policy document with enhanced permissions + """ + return { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads" + ], + "Resource": [bucket_arn], + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:PutObjectTagging", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [f"{bucket_arn}/*"], + "Effect": "Allow", + }, + ], + } + + +def create_seqera_credentials( + seqera_provider: seqera.Provider, + workspace_id: float, + access_key_id: pulumi.Output[str], + access_key_secret: pulumi.Output[str], +) -> seqera.Credential: + """Upload TowerForge AWS credentials to Seqera Platform. + + Args: + seqera_provider: Configured Seqera provider instance + workspace_id: Seqera Platform workspace ID + access_key_id: AWS access key ID from TowerForge IAM user + access_key_secret: AWS secret access key from TowerForge IAM user + + Returns: + seqera.Credential: Seqera credential resource with credentials_id + + Raises: + CredentialError: If credential upload fails + """ + pulumi.log.info("Uploading TowerForge credentials to Seqera Platform") + + # Create AWS credentials configuration for Seqera Platform + aws_keys = seqera.CredentialKeysArgs( + aws=seqera.CredentialKeysAwsArgs( + access_key=access_key_id, + secret_key=access_key_secret, + # Note: assume_role_arn is optional and not needed for direct IAM user credentials + ) + ) + + # Upload credentials to Seqera Platform + seqera_credential = seqera.Credential( + "towerforge-aws-credential", + name=TOWERFORGE_CREDENTIAL_NAME, + description=TOWERFORGE_CREDENTIAL_DESCRIPTION, + provider_type="aws", + workspace_id=workspace_id, + keys=aws_keys, + opts=pulumi.ResourceOptions( + provider=seqera_provider, + # Ensure credentials are uploaded after IAM access key is created + custom_timeouts=pulumi.CustomTimeouts( + create=TIMEOUTS["seqera_credential_create"], + update=TIMEOUTS["seqera_credential_update"], + delete=TIMEOUTS["seqera_credential_delete"], + ), + ), + ) + + return seqera_credential + + +def _create_iam_policies( + aws_provider: aws.Provider, s3_bucket +) -> Tuple[aws.iam.Policy, aws.iam.Policy, aws.iam.Policy]: + """Create IAM policies for TowerForge operations. + + Args: + aws_provider: Configured AWS provider instance + s3_bucket: S3 bucket resource for policy attachment + + Returns: + Tuple of (forge_policy, launch_policy, s3_policy) + """ + # TowerForge Forge Policy - Comprehensive permissions for resource creation + forge_policy = aws.iam.Policy( + "towerforge-forge-policy", + name=TOWERFORGE_POLICY_NAMES["forge"], + description="IAM policy for TowerForge to create and manage AWS Batch resources", + policy=json.dumps(_create_forge_policy_document()), + opts=pulumi.ResourceOptions(provider=aws_provider), + ) + + # TowerForge Launch Policy - Limited permissions for pipeline execution + launch_policy = aws.iam.Policy( + "towerforge-launch-policy", + name=TOWERFORGE_POLICY_NAMES["launch"], + description="IAM policy for TowerForge to launch and monitor pipeline executions", + policy=json.dumps(_create_launch_policy_document()), + opts=pulumi.ResourceOptions(provider=aws_provider), + ) + + # TowerForge S3 Bucket Access Policy - Access to specified S3 bucket + s3_policy = aws.iam.Policy( + "towerforge-s3-policy", + name=TOWERFORGE_POLICY_NAMES["s3"], + description=s3_bucket.bucket.apply( + lambda bucket_name: f"IAM policy for TowerForge to access {bucket_name} S3 bucket" + ), + policy=s3_bucket.arn.apply( + lambda arn: json.dumps(_create_s3_policy_document(arn)) + ), + opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[s3_bucket]), + ) + + return forge_policy, launch_policy, s3_policy + + +def _generate_policy_hash( + forge_policy: aws.iam.Policy, + launch_policy: aws.iam.Policy, + s3_policy: aws.iam.Policy, +) -> str: + """Generate a hash of IAM policies to detect changes. + + Args: + forge_policy: TowerForge Forge policy + launch_policy: TowerForge Launch policy + s3_policy: TowerForge S3 policy + + Returns: + str: SHA256 hash of the combined policy documents + """ + # Create a deterministic hash of all policy documents + forge_doc = _create_forge_policy_document() + launch_doc = _create_launch_policy_document() + + # Combine all policy documents for hashing + combined_policies = json.dumps( + { + "forge": forge_doc, + "launch": launch_doc, + # Note: S3 policy is bucket-specific, so we'll use a placeholder for consistent hashing + # Version updated to include multipart upload permissions + "s3": {"bucket_dependent": True, "version": "v2-multipart"}, + }, + sort_keys=True, + ) + + return hashlib.sha256(combined_policies.encode()).hexdigest() + + +def create_towerforge_credentials( + aws_provider: aws.Provider, + s3_bucket, + seqera_provider: seqera.Provider, + workspace_id: float, +) -> Tuple[ + pulumi.Output[str], pulumi.Output[str], pulumi.Output[str], seqera.Credential, str +]: + """Create TowerForge IAM resources and upload to Seqera Platform. + + Creates IAM policies, user, and access keys for TowerForge operations, + then uploads the credentials to Seqera Platform for use by compute environments. + Based on https://github.com/seqeralabs/nf-tower-aws + + Args: + aws_provider: Configured AWS provider instance + s3_bucket: S3 bucket resource for policy attachment + seqera_provider: Configured Seqera provider instance + workspace_id: Seqera Platform workspace ID + + Returns: + Tuple: (access_key_id, access_key_secret, seqera_credentials_id, seqera_credential_resource, iam_policy_hash) + """ + # Create IAM policies + forge_policy, launch_policy, s3_policy = _create_iam_policies( + aws_provider, s3_bucket + ) + + # Generate policy version hash for compute environment recreation on policy changes + iam_policy_hash = _generate_policy_hash(forge_policy, launch_policy, s3_policy) + + # Create TowerForge IAM User + towerforge_user = aws.iam.User( + "towerforge-user", + name=TOWERFORGE_USER_NAME, + opts=pulumi.ResourceOptions(provider=aws_provider), + ) + + # Attach policies to the TowerForge user + forge_attachment = aws.iam.UserPolicyAttachment( + "towerforge-forge-policy-attachment", + user=towerforge_user.name, + policy_arn=forge_policy.arn, + opts=pulumi.ResourceOptions( + provider=aws_provider, depends_on=[towerforge_user, forge_policy] + ), + ) + + launch_attachment = aws.iam.UserPolicyAttachment( + "towerforge-launch-policy-attachment", + user=towerforge_user.name, + policy_arn=launch_policy.arn, + opts=pulumi.ResourceOptions( + provider=aws_provider, + depends_on=[towerforge_user, launch_policy], + ), + ) + + s3_attachment = aws.iam.UserPolicyAttachment( + "towerforge-s3-policy-attachment", + user=towerforge_user.name, + policy_arn=s3_policy.arn, + opts=pulumi.ResourceOptions( + provider=aws_provider, depends_on=[towerforge_user, s3_policy] + ), + ) + + # Create access keys for the TowerForge user + towerforge_access_key = aws.iam.AccessKey( + "towerforge-access-key", + user=towerforge_user.name, + opts=pulumi.ResourceOptions( + provider=aws_provider, + depends_on=[forge_attachment, launch_attachment, s3_attachment], + additional_secret_outputs=["secret"], + ), + ) + + # Upload the credentials to Seqera Platform + seqera_credential = create_seqera_credentials( + seqera_provider=seqera_provider, + workspace_id=workspace_id, + access_key_id=towerforge_access_key.id, + access_key_secret=towerforge_access_key.secret, + ) + + # Return the access key credentials, Seqera credentials ID, credential resource, and policy hash + return ( + towerforge_access_key.id, + towerforge_access_key.secret, + seqera_credential.credentials_id, + seqera_credential, + iam_policy_hash, + ) + + +def get_towerforge_resources( + aws_provider: aws.Provider, + s3_bucket, + seqera_provider: Optional[seqera.Provider] = None, + workspace_id: Optional[float] = None, +) -> Dict[str, Any]: + """Create TowerForge resources and return resource information for exports. + + This function creates the TowerForge IAM resources and returns a dictionary + containing resource information for Pulumi exports. + + Args: + aws_provider: Configured AWS provider instance + s3_bucket: S3 bucket resource for policy attachment + seqera_provider: Optional Seqera provider for credential upload + workspace_id: Optional workspace ID for Seqera Platform + + Returns: + Dict[str, Any]: Resource information for Pulumi exports + + Raises: + ValueError: If required parameters are missing + """ + # Create the credentials (this will create all the resources) + if seqera_provider and workspace_id: + ( + access_key_id, + access_key_secret, + seqera_credentials_id, + seqera_credential, + iam_policy_hash, + ) = create_towerforge_credentials( + aws_provider, s3_bucket, seqera_provider, workspace_id + ) + else: + # Fallback for backward compatibility - this will raise an error since signature changed + raise ValueError( + "get_towerforge_resources now requires seqera_provider and workspace_id parameters. " + "Please update your code to use the new signature or call create_towerforge_credentials directly." + ) + + return { + "user": { + "name": TOWERFORGE_USER_NAME, + "arn": f"arn:aws:iam::{{aws_account_id}}:user/{TOWERFORGE_USER_NAME}", # Will be populated by Pulumi + }, + "access_key_id": access_key_id, + "access_key_secret": access_key_secret, + "seqera_credentials_id": seqera_credentials_id, + "policies": { + "forge_policy_name": TOWERFORGE_POLICY_NAMES["forge"], + "launch_policy_name": TOWERFORGE_POLICY_NAMES["launch"], + "s3_policy_name": TOWERFORGE_POLICY_NAMES["s3"], + }, + } diff --git a/pulumi/AWSMegatests/src/infrastructure/s3.py b/pulumi/AWSMegatests/src/infrastructure/s3.py new file mode 100644 index 00000000..70afda88 --- /dev/null +++ b/pulumi/AWSMegatests/src/infrastructure/s3.py @@ -0,0 +1,189 @@ +"""S3 infrastructure management for AWS Megatests.""" + +from typing import Dict, Any + +import pulumi +from pulumi_aws import s3 + +from ..utils.constants import S3_BUCKET_NAME + + +def create_s3_infrastructure(aws_provider) -> Dict[str, Any]: + """Create S3 bucket and lifecycle configuration. + + Args: + aws_provider: Configured AWS provider instance + + Returns: + Dict[str, Any]: Dictionary containing bucket and lifecycle configuration + """ + # Import existing AWS resources used by nf-core megatests + # S3 bucket for Nextflow work directory (already exists) + nf_core_awsmegatests_bucket = s3.Bucket( + "nf-core-awsmegatests", + bucket=S3_BUCKET_NAME, + opts=pulumi.ResourceOptions( + import_=S3_BUCKET_NAME, # Import existing bucket + protect=True, # Protect from accidental deletion + provider=aws_provider, # Use configured AWS provider + ignore_changes=[ + "lifecycle_rules", + "versioning", + ], # Don't modify existing configurations - managed manually due to permission constraints + ), + ) + + # S3 bucket lifecycle configuration + # Create lifecycle rules for automated cost optimization and cleanup + bucket_lifecycle_configuration = create_s3_lifecycle_configuration( + aws_provider, nf_core_awsmegatests_bucket + ) + + # S3 bucket CORS configuration for Seqera Data Explorer compatibility + bucket_cors_configuration = create_s3_cors_configuration( + aws_provider, nf_core_awsmegatests_bucket + ) + + return { + "bucket": nf_core_awsmegatests_bucket, + "lifecycle_configuration": bucket_lifecycle_configuration, + "cors_configuration": bucket_cors_configuration, + } + + +def create_s3_lifecycle_configuration(aws_provider, bucket): + """Create S3 lifecycle configuration with proper rules for Nextflow workflows. + + Args: + aws_provider: Configured AWS provider instance + bucket: S3 bucket resource + + Returns: + S3 bucket lifecycle configuration resource + """ + # S3 bucket lifecycle configuration for cost optimization and cleanup + # Rules designed specifically for Nextflow workflow patterns + lifecycle_configuration = s3.BucketLifecycleConfigurationV2( + "nf-core-awsmegatests-lifecycle", + bucket=bucket.id, + rules=[ + # Rule 1: Preserve metadata files with cost optimization + s3.BucketLifecycleConfigurationV2RuleArgs( + id="preserve-metadata-files", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + tag=s3.BucketLifecycleConfigurationV2RuleFilterTagArgs( + key="nextflow.io/metadata", value="true" + ) + ), + transitions=[ + s3.BucketLifecycleConfigurationV2RuleTransitionArgs( + days=30, storage_class="STANDARD_IA" + ), + s3.BucketLifecycleConfigurationV2RuleTransitionArgs( + days=90, storage_class="GLACIER" + ), + ], + ), + # Rule 2: Clean up temporary files after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-temporary-files", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + tag=s3.BucketLifecycleConfigurationV2RuleFilterTagArgs( + key="nextflow.io/temporary", value="true" + ) + ), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 3: Clean up work directory after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-work-directory", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs(prefix="work/"), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 4: Clean up scratch directory after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-scratch-directory", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + prefix="scratch/" + ), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 5: Clean up cache directories after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-cache-directories", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs(prefix="cache/"), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 6: Clean up .cache directories after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-dot-cache-directories", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + prefix=".cache/" + ), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 7: Clean up incomplete multipart uploads + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-incomplete-multipart-uploads", + status="Enabled", + abort_incomplete_multipart_upload=s3.BucketLifecycleConfigurationV2RuleAbortIncompleteMultipartUploadArgs( + days_after_initiation=7 + ), + ), + ], + opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[bucket]), + ) + + return lifecycle_configuration + + +def create_s3_cors_configuration(aws_provider, bucket): + """Create S3 CORS configuration for Seqera Data Explorer compatibility. + + Args: + aws_provider: Configured AWS provider instance + bucket: S3 bucket resource + + Returns: + S3 bucket CORS configuration resource + """ + # S3 CORS configuration for Seqera Data Explorer compatibility + # Based on official Seqera documentation: + # https://docs.seqera.io/platform-cloud/data/data-explorer#amazon-s3-cors-configuration + cors_configuration = s3.BucketCorsConfigurationV2( + "nf-core-awsmegatests-cors", + bucket=bucket.id, + cors_rules=[ + s3.BucketCorsConfigurationV2CorsRuleArgs( + id="SeqeraDataExplorerAccess", + allowed_headers=["*"], + allowed_methods=["GET", "HEAD", "POST", "PUT", "DELETE"], + allowed_origins=[ + "https://*.cloud.seqera.io", + "https://*.tower.nf", + "https://cloud.seqera.io", + "https://tower.nf", + ], + expose_headers=["ETag"], + max_age_seconds=3000, + ), + # Additional rule for direct browser access + s3.BucketCorsConfigurationV2CorsRuleArgs( + id="BrowserDirectAccess", + allowed_headers=["Authorization", "Content-Type", "Range"], + allowed_methods=["GET", "HEAD"], + allowed_origins=["*"], + expose_headers=["Content-Range", "Content-Length", "ETag"], + max_age_seconds=3000, + ), + ], + opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[bucket]), + ) + + return cors_configuration diff --git a/pulumi/AWSMegatests/src/integrations/__init__.py b/pulumi/AWSMegatests/src/integrations/__init__.py new file mode 100644 index 00000000..ba069edd --- /dev/null +++ b/pulumi/AWSMegatests/src/integrations/__init__.py @@ -0,0 +1,10 @@ +"""Third-party integrations for AWS Megatests.""" + +from .github import create_github_resources +from .github_credentials import create_github_credential, get_github_credential_config + +__all__ = [ + "create_github_resources", + "create_github_credential", + "get_github_credential_config", +] diff --git a/pulumi/AWSMegatests/src/integrations/github.py b/pulumi/AWSMegatests/src/integrations/github.py new file mode 100644 index 00000000..7a43500d --- /dev/null +++ b/pulumi/AWSMegatests/src/integrations/github.py @@ -0,0 +1,169 @@ +"""GitHub integration for AWS Megatests - secrets and variables management.""" + +from typing import Dict, Any, List, Optional, Union + +import pulumi +import pulumi_github as github + +from ..utils.constants import GITHUB_VARIABLE_NAMES, S3_BUCKET_NAME + + +class GitHubIntegrationError(Exception): + """Exception raised when GitHub integration operations fail.""" + + pass + + +def _create_organization_variable( + provider: github.Provider, + resource_name: str, + variable_name: str, + value: Union[str, pulumi.Output[str]], +) -> github.ActionsOrganizationVariable: + """Create a GitHub organization variable with consistent configuration. + + Args: + provider: GitHub provider instance + resource_name: Pulumi resource name + variable_name: GitHub variable name + value: Variable value + + Returns: + github.ActionsOrganizationVariable: Created variable resource + """ + return github.ActionsOrganizationVariable( + resource_name, + visibility="all", + variable_name=variable_name, + value=value, + opts=pulumi.ResourceOptions( + provider=provider, + delete_before_replace=True, # Workaround for GitHub provider issue #250 + ignore_changes=[ + "visibility" + ], # Ignore changes to visibility if variable exists + ), + ) + + +def _create_gh_commands( + workspace_id_val: str, cpu_env_id_val: str, tower_token_val: Optional[str] = None +) -> List[str]: + """Generate manual gh CLI commands for secrets management. + + Args: + workspace_id_val: Workspace ID value + cpu_env_id_val: CPU environment ID value + tower_token_val: Optional tower access token placeholder + + Returns: + List[str]: List of gh CLI commands + """ + commands = [] + + # Legacy workspace ID secret + commands.append( + f'gh secret set TOWER_WORKSPACE_ID --org nf-core --body "{workspace_id_val}" --visibility all' + ) + + # Legacy compute env secret (CPU) + commands.append( + f'gh secret set TOWER_COMPUTE_ENV --org nf-core --body "{cpu_env_id_val}" --visibility all' + ) + + # Tower access token (if provided) + if tower_token_val: + commands.append( + "OP_ACCOUNT=nf-core gh secret set TOWER_ACCESS_TOKEN --org nf-core " + "--body \"$(op read 'op://Dev/Seqera Platform/TOWER_ACCESS_TOKEN')\" --visibility all" + ) + + return commands + + +def create_github_resources( + github_provider: github.Provider, + compute_env_ids: Dict[str, Any], + tower_workspace_id: Union[str, pulumi.Output[str]], + tower_access_token: Optional[str] = None, +) -> Dict[str, Any]: + """Create GitHub organization variables and provide manual secret commands. + + Args: + github_provider: GitHub provider instance + compute_env_ids: Dictionary containing compute environment IDs + tower_workspace_id: Seqera Platform workspace ID + tower_access_token: Tower access token for manual secret commands (optional) + + Returns: + Dict[str, Any]: Dictionary containing created variables and manual commands + + Raises: + GitHubIntegrationError: If GitHub resource creation fails + """ + # Create org-level GitHub variables for compute environment IDs (non-sensitive) + # Using delete_before_replace to work around pulumi/pulumi-github#250 + variables = {} + + # Compute environment variables + for env_type in ["cpu", "gpu", "arm"]: + var_name = GITHUB_VARIABLE_NAMES[env_type] + resource_name = f"tower-compute-env-{env_type}" + + variables[env_type] = _create_organization_variable( + github_provider, + resource_name, + var_name, + compute_env_ids[env_type], + ) + + # Workspace ID variable + variables["workspace_id"] = _create_organization_variable( + github_provider, + "tower-workspace-id", + GITHUB_VARIABLE_NAMES["workspace_id"], + tower_workspace_id, + ) + + # Legacy S3 bucket variable + variables["legacy_s3_bucket"] = _create_organization_variable( + github_provider, + "legacy-aws-s3-bucket", + GITHUB_VARIABLE_NAMES["s3_bucket"], + S3_BUCKET_NAME, + ) + + # GitHub Secrets Management - Manual Commands Only + # NOTE: Due to pulumi/pulumi-github#250, secrets must be managed manually + # https://github.com/nf-core/ops/issues/162 - Legacy compatibility needed + + # Generate manual gh CLI commands for secrets management + if all(isinstance(compute_env_ids[k], str) for k in compute_env_ids) and isinstance( + tower_workspace_id, str + ): + # All static values + gh_cli_commands: Union[List[str], pulumi.Output[List[str]]] = ( + _create_gh_commands( + tower_workspace_id, + compute_env_ids["cpu"], + "" if tower_access_token else None, + ) + ) + else: + # Dynamic values - create commands that will be resolved at runtime + gh_cli_commands = pulumi.Output.all( + workspace_id=tower_workspace_id, cpu_env_id=compute_env_ids["cpu"] + ).apply( + lambda args: _create_gh_commands( + args["workspace_id"], + args["cpu_env_id"], + "" if tower_access_token else None, + ) + ) + + return { + "variables": variables, + "secrets": {}, # No Pulumi-managed secrets due to provider issue + "gh_cli_commands": gh_cli_commands, + "note": "Secrets must be managed manually due to pulumi/pulumi-github#250", + } diff --git a/pulumi/AWSMegatests/src/integrations/github_credentials.py b/pulumi/AWSMegatests/src/integrations/github_credentials.py new file mode 100644 index 00000000..707bc8c7 --- /dev/null +++ b/pulumi/AWSMegatests/src/integrations/github_credentials.py @@ -0,0 +1,96 @@ +"""GitHub credentials integration for Seqera Platform.""" + +import pulumi +import pulumi_seqera as seqera +from typing import Dict, Tuple + + +class GitHubCredentialError(Exception): + """Exception raised when GitHub credential creation fails.""" + + pass + + +def create_github_credential( + seqera_provider: seqera.Provider, + workspace_id: int, + github_token: str, + github_username: str = "nf-core-bot", + credential_name: str = "nf-core-github-finegrained", +) -> Tuple[seqera.Credential, str]: + """Create a GitHub fine-grained credential in Seqera Platform. + + This credential allows Seqera Platform to pull pipeline repositories from GitHub + without hitting GitHub rate limits. The fine-grained token provides secure, + scoped access to nf-core repositories with minimal required permissions. + + Args: + seqera_provider: Configured Seqera provider instance + workspace_id: Seqera workspace ID + github_token: Fine-grained GitHub personal access token for repository access + github_username: GitHub username (default: nf-core-bot) + credential_name: Name for the credential in Seqera + + Returns: + Tuple of (credential_resource, credential_id) + + Raises: + GitHubCredentialError: If credential creation fails + ValueError: If required parameters are missing + """ + # Validate required parameters + if not github_token: + raise ValueError("GitHub token is required") + if not workspace_id: + raise ValueError("Workspace ID is required") + + pulumi.log.info( + f"Creating GitHub credential '{credential_name}' in workspace {workspace_id}" + ) + + try: + # Create GitHub credential using Seqera Terraform provider + github_credential = seqera.Credential( + f"github-credential-{credential_name}", + name=credential_name, + description="Fine-grained GitHub token to avoid rate limits when Platform pulls pipeline repositories", + provider_type="github", + base_url="https://github.com/nf-core/", # Scope to nf-core organization + keys=seqera.CredentialKeysArgs( + github=seqera.CredentialKeysGithubArgs( + username=github_username, + password=github_token, # GitHub tokens go in the password field + ) + ), + workspace_id=workspace_id, + opts=pulumi.ResourceOptions( + provider=seqera_provider, + protect=True, # Protect credential from accidental deletion + ), + ) + + # Return both the resource and the credential ID for reference + return github_credential, github_credential.id + + except Exception as e: + pulumi.log.error(f"Failed to create GitHub credential: {str(e)}") + raise GitHubCredentialError( + f"GitHub credential creation failed: {str(e)}" + ) from e + + +def get_github_credential_config() -> Dict[str, str]: + """Get configuration for GitHub credential creation. + + Returns: + Dict containing configuration values from ESC environment + """ + import os + + return { + "github_finegrained_token": os.environ.get("PLATFORM_GITHUB_ORG_TOKEN", ""), + "github_username": os.environ.get("GITHUB_USERNAME", "nf-core-bot"), + "credential_name": os.environ.get( + "GITHUB_CREDENTIAL_NAME", "nf-core-github-finegrained" + ), + } diff --git a/pulumi/AWSMegatests/src/integrations/workspace_participants_command.py b/pulumi/AWSMegatests/src/integrations/workspace_participants_command.py new file mode 100644 index 00000000..7e99190c --- /dev/null +++ b/pulumi/AWSMegatests/src/integrations/workspace_participants_command.py @@ -0,0 +1,268 @@ +"""Seqera Platform workspace participant management using Pulumi Command provider.""" + +import json +import pulumi +import pulumi_command as command +from typing import Dict, List, Optional +from ..utils.logging import log_info + + +def create_team_data_setup_command( + workspace_id: int, + seqera_token: str, + github_token: str, + opts: Optional[pulumi.ResourceOptions] = None, +) -> command.local.Command: + """ + Create a Pulumi Command that generates team data with proper credentials. + + This runs the team data setup scripts automatically during Pulumi deployment. + """ + setup_cmd = command.local.Command( + "team-data-setup", + create=""" +# Generate team member data with proper credentials +echo "=== Setting up team member data with privacy protection ===" + +# Run setup script with environment credentials +uv run python scripts/setup_team_data.py + +echo "โœ“ Team data setup completed" +echo "Files generated locally (not committed to git):" +echo " - scripts/maintainers_data.json" +echo " - scripts/core_team_data.json" +echo " - scripts/unified_team_data.json" + """, + environment={ + "GITHUB_TOKEN": github_token, + "TOWER_ACCESS_TOKEN": seqera_token, + }, + opts=opts, + ) + + return setup_cmd + + +def create_individual_member_commands( + workspace_id: int, + token: str, + github_token: str, + org_id: int = 252464779077610, # nf-core + opts: Optional[pulumi.ResourceOptions] = None, +) -> tuple[command.local.Command, Dict[str, command.local.Command]]: + """ + Create individual Pulumi Command resources for each GitHub team member. + + This provides granular tracking of each maintainer's workspace participant status. + """ + # First, ensure team data is set up with proper credentials + setup_cmd = create_team_data_setup_command(workspace_id, token, github_token, opts) + + # Generate team data at runtime to avoid committing private emails + log_info("Team data will be generated automatically during deployment...") + + # Load team data (will be available after setup command runs) + try: + with open("scripts/unified_team_data.json", "r") as f: + data = json.load(f) + team_members = data.get("seqera_participants", []) + log_info(f"Loaded {len(team_members)} team members from runtime data") + except FileNotFoundError: + # For preview purposes, show expected team count + log_info("Team data will be generated during deployment (35 expected members)") + team_members = [] + except Exception as e: + log_info(f"Team data will be generated at runtime: {e}") + team_members = [] + + member_commands = {} + + log_info(f"Creating individual tracking for {len(team_members)} team members") + + for member in team_members: + email = member["name"] + github_username = member["github_username"] + role = member["role"] # OWNER for core team, MAINTAIN for maintainers + + # Create safe resource name + safe_name = github_username.replace("-", "_").replace(".", "_") + + # Note: Role precedence handled in bash script (core team checked first) + + # Create individual command for this member + member_cmd = command.local.Command( + f"team_sync_{safe_name}", + create=f''' +#!/bin/bash +# Sync GitHub team member '{github_username}' to Seqera workspace with {role} role +echo "=== Syncing {github_username} ({email}) as {role} ===" + +# Verify user is still in appropriate GitHub teams +echo "Checking GitHub team membership..." +found_in_team=false + +# Check core team first (higher precedence) +if gh api orgs/nf-core/teams/core/members --jq '.[].login' | grep -q "^{github_username}$"; then + echo "โœ“ {github_username} confirmed in nf-core/core team (OWNER role)" + current_role="OWNER" + found_in_team=true +elif gh api orgs/nf-core/teams/maintainers/members --jq '.[].login' | grep -q "^{github_username}$"; then + echo "โœ“ {github_username} confirmed in nf-core/maintainers team (MAINTAIN role)" + current_role="MAINTAIN" + found_in_team=true +fi + +if [ "$found_in_team" = false ]; then + echo "โš ๏ธ {github_username} not found in any relevant team, skipping" + exit 0 +fi + +# Ensure we're using the correct role (core team precedence) +target_role="{role}" +if [ "$current_role" != "$target_role" ]; then + echo "๐Ÿ”„ Role precedence: Using $current_role (detected) instead of $target_role" + target_role="$current_role" +fi + +# Check current email (in case it changed) +echo "Fetching current email..." +current_email=$(gh api /users/{github_username} --jq '.email // empty') + +# Handle members without public emails +if [[ "{email}" == github:* ]]; then + # Member has no public email, try to get current one + if [ -z "$current_email" ]; then + echo "โš ๏ธ {github_username} has no public email - cannot add to Seqera Platform" + echo "STATUS:NO_EMAIL:{github_username}:$target_role" + exit 0 + else + echo "โœ“ {github_username} now has public email: $current_email" + fi +else + # Member had cached email, check if it changed + cached_email="{email}" + if [ -z "$current_email" ]; then + echo "โš ๏ธ {github_username} email no longer public, using cached: $cached_email" + current_email="$cached_email" + elif [ "$current_email" != "$cached_email" ]; then + echo "๐Ÿ”„ {github_username} email changed: $cached_email โ†’ $current_email" + else + echo "โœ“ Current email confirmed: $current_email" + fi +fi + +# Add to Seqera workspace +echo "Adding to Seqera workspace {workspace_id}..." +response=$(curl -s -w "%{{http_code}}" -X PUT \\ + "https://api.cloud.seqera.io/orgs/{org_id}/workspaces/{workspace_id}/participants/add" \\ + -H "Authorization: Bearer {token}" \\ + -H "Content-Type: application/json" \\ + -d '{{"userNameOrEmail": "'$current_email'"}}') + +http_code="${{response: -3}}" +response_body="${{response%???}}" + +case $http_code in + 200|201|204) + echo "โœ“ Successfully added {github_username} with $target_role role" + echo "STATUS:ADDED:$current_email:$target_role" + ;; + 409) + echo "~ {github_username} already exists in workspace" + echo "STATUS:EXISTS:$current_email:$target_role" + ;; + 404) + echo "โœ— User not found in Seqera Platform: $current_email" + echo "STATUS:USER_NOT_FOUND:$current_email:N/A" + ;; + *) + echo "โœ— Failed to add {github_username}: HTTP $http_code" + echo "Response: $response_body" + echo "STATUS:FAILED:$current_email:ERROR" + exit 1 + ;; +esac + +echo "Completed sync for {github_username}" + ''', + environment={ + "GITHUB_TOKEN": github_token, + }, + opts=pulumi.ResourceOptions( + depends_on=[setup_cmd], + parent=opts.parent if opts else None, + ), + ) + + member_commands[github_username] = member_cmd + + return setup_cmd, member_commands + + +def create_workspace_participants_via_command( + workspace_id: int, + token: str, + participants_data: List[Dict[str, str]], + opts: Optional[pulumi.ResourceOptions] = None, +) -> command.local.Command: + """ + Create workspace participants using Pulumi Command provider to run the Python script. + + Args: + workspace_id: Seqera workspace ID + token: Seqera API access token + participants_data: List of participant dictionaries + opts: Pulumi resource options + + Returns: + Command resource that manages workspace participants + """ + # Create the command that will run our Python script + participant_count = len(participants_data) + log_info(f"Creating command to add {participant_count} workspace participants") + + # The command runs within the Pulumi execution context + add_participants_cmd = command.local.Command( + "add-workspace-participants", + create="uv run python scripts/add_maintainers_to_workspace.py --yes", + environment={ + "TOWER_ACCESS_TOKEN": token, + "TOWER_WORKSPACE_ID": str(workspace_id), + }, + opts=pulumi.ResourceOptions(**opts.__dict__ if opts else {}), + ) + + return add_participants_cmd + + +def create_workspace_participants_verification( + workspace_id: int, + token: str, + depends_on: List[pulumi.Resource], + opts: Optional[pulumi.ResourceOptions] = None, +) -> command.local.Command: + """ + Create a verification command that checks workspace participants were added correctly. + + Args: + workspace_id: Seqera workspace ID + token: Seqera API access token + depends_on: Resources this command depends on + opts: Pulumi resource options + + Returns: + Command resource that verifies workspace participants + """ + verification_cmd = command.local.Command( + "verify-workspace-participants", + create="uv run python scripts/inspect_participants.py", + environment={ + "TOWER_ACCESS_TOKEN": token, + "TOWER_WORKSPACE_ID": str(workspace_id), + }, + opts=pulumi.ResourceOptions( + depends_on=depends_on, **(opts.__dict__ if opts else {}) + ), + ) + + return verification_cmd diff --git a/pulumi/AWSMegatests/src/integrations/workspace_participants_simple.py b/pulumi/AWSMegatests/src/integrations/workspace_participants_simple.py new file mode 100644 index 00000000..5cd50d1a --- /dev/null +++ b/pulumi/AWSMegatests/src/integrations/workspace_participants_simple.py @@ -0,0 +1,124 @@ +"""Simple workspace participant management using Pulumi's apply() pattern.""" + +import json +import pulumi +import requests +from typing import Dict, List, Any +from ..utils.logging import log_info + + +def add_workspace_participant_simple( + email: str, + role: str, + workspace_id: pulumi.Output[str], + token: pulumi.Output[str], + org_id: int = 252464779077610, +) -> pulumi.Output[Dict[str, Any]]: + """ + Add a workspace participant using Pulumi's apply() pattern. + + This is simpler than dynamic resources but still integrates with Pulumi. + """ + + def _add_participant(args): + """Internal function that does the actual API call.""" + workspace_id_val, token_val = args + + headers = { + "Authorization": f"Bearer {token_val}", + "Content-Type": "application/json", + } + + url = f"https://api.cloud.seqera.io/orgs/{org_id}/workspaces/{workspace_id_val}/participants/add" + payload = {"userNameOrEmail": email} + + try: + response = requests.put(url, headers=headers, json=payload, timeout=30) + + if response.status_code in [200, 201, 204]: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "added", + "participant_id": f"{org_id}:{workspace_id_val}:{email}", + } + elif response.status_code == 409: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "already_exists", + "participant_id": f"{org_id}:{workspace_id_val}:{email}", + } + else: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "failed", + "error": f"HTTP {response.status_code}", + } + + except Exception as e: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "error", + "error": str(e), + } + + # Use Pulumi's apply to handle the async nature of Outputs + return pulumi.Output.all(workspace_id, token).apply(_add_participant) + + +def create_workspace_participants_simple( + workspace_id: pulumi.Output[str], + token: pulumi.Output[str], + maintainer_emails: List[str], + role: str = "MAINTAIN", +) -> pulumi.Output[List[Dict[str, Any]]]: + """ + Create multiple workspace participants using the simple approach. + + Args: + workspace_id: Seqera workspace ID as Pulumi Output + token: Seqera API token as Pulumi Output + maintainer_emails: List of email addresses to add + role: Role to assign (default: MAINTAIN) + + Returns: + Pulumi Output containing list of participant creation results + """ + + participant_outputs = [] + + for email in maintainer_emails: + participant_result = add_workspace_participant_simple( + email, role, workspace_id, token + ) + participant_outputs.append(participant_result) + + # Combine all outputs into a single list + return pulumi.Output.all(*participant_outputs) + + +def load_maintainer_emails_static() -> List[str]: + """Load maintainer emails from the JSON file (static version for Pulumi).""" + try: + with open("scripts/maintainers_data.json", "r") as f: + data = json.load(f) + + participants = data.get("seqera_participants", []) + emails = [p["name"] for p in participants] + + log_info(f"Loaded {len(emails)} maintainer emails for workspace participants") + return emails + + except FileNotFoundError: + log_info("Maintainers data file not found, skipping workspace participants") + return [] + except Exception as e: + log_info(f"Error loading maintainers data: {e}") + return [] diff --git a/pulumi/AWSMegatests/src/providers/__init__.py b/pulumi/AWSMegatests/src/providers/__init__.py new file mode 100644 index 00000000..39eb2523 --- /dev/null +++ b/pulumi/AWSMegatests/src/providers/__init__.py @@ -0,0 +1,7 @@ +"""Provider configurations for AWS Megatests infrastructure.""" + +from .aws import create_aws_provider +from .github import create_github_provider +from .seqera import create_seqera_provider + +__all__ = ["create_aws_provider", "create_github_provider", "create_seqera_provider"] diff --git a/pulumi/AWSMegatests/src/providers/aws.py b/pulumi/AWSMegatests/src/providers/aws.py new file mode 100644 index 00000000..9c0febdb --- /dev/null +++ b/pulumi/AWSMegatests/src/providers/aws.py @@ -0,0 +1,19 @@ +"""AWS provider configuration for AWS Megatests infrastructure.""" + +import pulumi_aws as aws +from ..utils.constants import AWS_REGION + + +def create_aws_provider() -> aws.Provider: + """Create AWS provider using ESC OIDC authentication. + + The ESC environment should automatically provide AWS credentials + when the environment is imported in Pulumi.prod.yaml. + + Returns: + aws.Provider: Configured AWS provider instance + """ + return aws.Provider( + "aws-provider", + region=AWS_REGION, + ) diff --git a/pulumi/AWSMegatests/src/providers/github.py b/pulumi/AWSMegatests/src/providers/github.py new file mode 100644 index 00000000..03ac9db9 --- /dev/null +++ b/pulumi/AWSMegatests/src/providers/github.py @@ -0,0 +1,16 @@ +"""GitHub provider configuration for AWS Megatests infrastructure.""" + +import pulumi_github as github +from ..utils.constants import GITHUB_ORG + + +def create_github_provider(github_token: str) -> github.Provider: + """Create GitHub provider with token authentication. + + Args: + github_token: GitHub personal access token with org admin permissions + + Returns: + github.Provider: Configured GitHub provider instance + """ + return github.Provider("github-provider", token=github_token, owner=GITHUB_ORG) diff --git a/pulumi/AWSMegatests/src/providers/seqera.py b/pulumi/AWSMegatests/src/providers/seqera.py new file mode 100644 index 00000000..465f728c --- /dev/null +++ b/pulumi/AWSMegatests/src/providers/seqera.py @@ -0,0 +1,38 @@ +"""Seqera provider configuration for AWS Megatests infrastructure.""" + +import pulumi +import pulumi_seqera as seqera +from typing import Dict, Any +from ..utils.constants import SEQERA_API_URL, ERROR_MESSAGES + + +class SeqeraProviderError(Exception): + """Exception raised when Seqera provider initialization fails.""" + + pass + + +def create_seqera_provider(config: Dict[str, Any]) -> seqera.Provider: + """Create and configure the Seqera provider with error handling. + + Args: + config: Configuration dictionary containing tower_access_token + + Returns: + seqera.Provider: Configured Seqera provider instance + + Raises: + SeqeraProviderError: If provider creation fails + ValueError: If required configuration is missing + """ + # Validate required configuration + if not config.get("tower_access_token"): + raise ValueError(ERROR_MESSAGES["missing_tower_token"]) + + pulumi.log.info("Creating Seqera provider with Cloud API endpoint") + + return seqera.Provider( + "seqera-provider", + bearer_auth=config["tower_access_token"], + server_url=SEQERA_API_URL, + ) diff --git a/pulumi/AWSMegatests/src/utils/__init__.py b/pulumi/AWSMegatests/src/utils/__init__.py new file mode 100644 index 00000000..5687405f --- /dev/null +++ b/pulumi/AWSMegatests/src/utils/__init__.py @@ -0,0 +1,31 @@ +"""Utility functions and constants for AWS Megatests.""" + +from .constants import ( + AWS_REGION, + S3_BUCKET_NAME, + SEQERA_API_URL, + COMPUTE_ENV_NAMES, + TOWERFORGE_POLICY_NAMES, +) +from .logging import ( + log_info, + log_error, + log_warning, + log_step, + log_resource_creation, + log_resource_success, +) + +__all__ = [ + "AWS_REGION", + "S3_BUCKET_NAME", + "SEQERA_API_URL", + "COMPUTE_ENV_NAMES", + "TOWERFORGE_POLICY_NAMES", + "log_info", + "log_error", + "log_warning", + "log_step", + "log_resource_creation", + "log_resource_success", +] diff --git a/pulumi/AWSMegatests/src/utils/constants.py b/pulumi/AWSMegatests/src/utils/constants.py new file mode 100644 index 00000000..baf1eefa --- /dev/null +++ b/pulumi/AWSMegatests/src/utils/constants.py @@ -0,0 +1,145 @@ +"""Constants and configuration values for AWS Megatests infrastructure.""" + +# AWS Configuration +AWS_REGION = "eu-west-1" +S3_BUCKET_NAME = "nf-core-awsmegatests" +S3_WORK_DIR = f"s3://{S3_BUCKET_NAME}" + +# Seqera Configuration +SEQERA_API_URL = "https://api.cloud.seqera.io" + +# Compute Environment Names +COMPUTE_ENV_NAMES = { + "cpu": "aws_ireland_fusionv2_nvme_cpu_snapshots", + "gpu": "aws_ireland_fusionv2_nvme_gpu_snapshots", + "arm": "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots", +} + +# Compute Environment Descriptions +COMPUTE_ENV_DESCRIPTIONS = { + "cpu": "CPU compute environment with Fusion v2 and NVMe storage", + "gpu": "GPU compute environment with Fusion v2 and NVMe storage", + "arm": "ARM CPU compute environment with Fusion v2 and NVMe storage", +} + +# Configuration File Paths +CONFIG_FILES = { + "cpu": "seqerakit/current-env-cpu.json", + "gpu": "seqerakit/current-env-gpu.json", + "arm": "seqerakit/current-env-cpu-arm.json", +} + +# Nextflow configuration files for compute environments +NEXTFLOW_CONFIG_FILES = { + "cpu": "seqerakit/configs/nextflow-cpu.config", + "gpu": "seqerakit/configs/nextflow-gpu.config", + "arm": "seqerakit/configs/nextflow-arm.config", +} + +# TowerForge Configuration +TOWERFORGE_USER_NAME = "TowerForge-AWSMegatests" +TOWERFORGE_POLICY_NAMES = { + "forge": "TowerForge-Forge-Policy", + "launch": "TowerForge-Launch-Policy", + "s3": "TowerForge-S3-Policy", +} + +TOWERFORGE_CREDENTIAL_NAME = "TowerForge-AWSMegatests-Dynamic" +TOWERFORGE_CREDENTIAL_DESCRIPTION = ( + "Dynamically created TowerForge credentials for AWS Megatests compute environments" +) + +# GitHub Configuration +GITHUB_ORG = "nf-core" +GITHUB_VARIABLE_NAMES = { + "cpu": "TOWER_COMPUTE_ENV_CPU", + "gpu": "TOWER_COMPUTE_ENV_GPU", + "arm": "TOWER_COMPUTE_ENV_ARM", + "workspace_id": "TOWER_WORKSPACE_ID", + "s3_bucket": "AWS_S3_BUCKET", +} + +# Timeout Configuration (in minutes) +TIMEOUTS = { + "seqera_credential_create": "5m", + "seqera_credential_update": "5m", + "seqera_credential_delete": "2m", + "compute_env_create": "10m", + "compute_env_update": "10m", + "compute_env_delete": "5m", +} + +# Default Compute Environment Settings +DEFAULT_COMPUTE_ENV_CONFIG = { + "region": AWS_REGION, + "workDir": S3_WORK_DIR, + "waveEnabled": True, + "fusion2Enabled": True, + "nvnmeStorageEnabled": True, + "fusionSnapshots": True, + "nextflowConfig": "", +} + +DEFAULT_FORGE_CONFIG = { + "type": "SPOT", + "minCpus": 0, + "maxCpus": 1000, + "gpuEnabled": False, + "instanceTypes": [], + "subnets": [], + "securityGroups": [], + "disposeOnDeletion": True, + "allowBuckets": [], + "efsCreate": False, + "ebsBootSize": 50, + "fargateHeadEnabled": True, + "arm64Enabled": False, +} + +# Error Messages +ERROR_MESSAGES = { + "missing_tower_token": ( + "TOWER_ACCESS_TOKEN is required for Seqera provider. " + "Please ensure it's set in your ESC environment with proper permissions: " + "WORKSPACE_ADMIN or COMPUTE_ENV_ADMIN scope." + ), + "seqera_provider_init_failed": ( + "Seqera provider initialization failed. " + "This usually indicates token permissions issues. " + "Ensure your TOWER_ACCESS_TOKEN has WORKSPACE_ADMIN or COMPUTE_ENV_ADMIN permissions." + ), + "config_file_not_found": "Configuration file not found: {}", + "invalid_json": "Invalid JSON in configuration file {}: {}", + "config_load_failed": "Failed to load configuration file {}: {}", + "invalid_workspace_id": "Invalid or missing workspace ID: {}", + "missing_compute_env_params": "Missing required parameters for compute environment {}", + "missing_config_args": "Configuration arguments are required for compute environment {}", + "compute_env_create_failed": ( + "Failed to create compute environment '{}'. " + "Common causes: " + "1. Seqera API token lacks required permissions (403 Forbidden) " + "2. Invalid credentials_id reference " + "3. Workspace access restrictions " + "4. Network connectivity issues" + ), + "credential_upload_failed": ( + "Failed to upload credentials to Seqera Platform. " + "Common causes: " + "1. Seqera provider not properly configured " + "2. Invalid workspace ID " + "3. Network connectivity issues to api.cloud.seqera.io " + "4. Invalid AWS credentials format" + ), +} + +# Required Environment Variables +REQUIRED_ENV_VARS = [ + "TOWER_ACCESS_TOKEN", + "TOWER_WORKSPACE_ID", + "GITHUB_TOKEN", +] + +# Optional Environment Variables with Defaults +DEFAULT_ENV_VARS = { + "TOWER_WORKSPACE_ID": "59994744926013", # Fallback workspace ID +} diff --git a/pulumi/AWSMegatests/src/utils/logging.py b/pulumi/AWSMegatests/src/utils/logging.py new file mode 100644 index 00000000..2cb342bd --- /dev/null +++ b/pulumi/AWSMegatests/src/utils/logging.py @@ -0,0 +1,68 @@ +"""Logging utilities for AWS Megatests infrastructure.""" + +import pulumi +from typing import Optional + + +def log_info(message: str, context: Optional[str] = None) -> None: + """Log an informational message with optional context. + + Args: + message: The message to log + context: Optional context prefix + """ + formatted_message = f"[{context}] {message}" if context else message + pulumi.log.info(formatted_message) + + +def log_error(message: str, context: Optional[str] = None) -> None: + """Log an error message with optional context. + + Args: + message: The error message to log + context: Optional context prefix + """ + formatted_message = f"[{context}] {message}" if context else message + pulumi.log.error(formatted_message) + + +def log_warning(message: str, context: Optional[str] = None) -> None: + """Log a warning message with optional context. + + Args: + message: The warning message to log + context: Optional context prefix + """ + formatted_message = f"[{context}] {message}" if context else message + pulumi.log.warn(formatted_message) + + +def log_step(step_number: int, step_name: str, description: str) -> None: + """Log a deployment step with consistent formatting. + + Args: + step_number: The step number + step_name: Short name for the step + description: Detailed description of what the step does + """ + log_info(f"Step {step_number}: {step_name} - {description}") + + +def log_resource_creation(resource_type: str, resource_name: str) -> None: + """Log resource creation with consistent formatting. + + Args: + resource_type: Type of resource being created + resource_name: Name of the resource + """ + log_info(f"Creating {resource_type}: {resource_name}", "Resource") + + +def log_resource_success(resource_type: str, resource_name: str) -> None: + """Log successful resource creation. + + Args: + resource_type: Type of resource that was created + resource_name: Name of the resource + """ + log_info(f"Successfully created {resource_type}: {resource_name}", "Resource") diff --git a/pulumi/seqera_platform/README.md b/pulumi/seqera_platform/README.md new file mode 100644 index 00000000..78255ea7 --- /dev/null +++ b/pulumi/seqera_platform/README.md @@ -0,0 +1,161 @@ +# Seqera Platform Multi-Workspace Infrastructure + +This directory contains Pulumi infrastructure-as-code for managing multiple Seqera Platform workspaces for nf-core. + +## Structure + +``` +seqera_platform/ +โ”œโ”€โ”€ shared/ # Shared Python modules +โ”‚ โ”œโ”€โ”€ providers/ # Provider factory functions (AWS, GitHub, Seqera) +โ”‚ โ”œโ”€โ”€ infrastructure/ # Reusable infrastructure modules (S3, IAM, compute envs) +โ”‚ โ”œโ”€โ”€ integrations/ # GitHub and Seqera integrations +โ”‚ โ”œโ”€โ”€ config/ # Configuration management +โ”‚ โ””โ”€โ”€ utils/ # Utility functions +โ”œโ”€โ”€ awsmegatests/ # AWS Megatests workspace +โ”‚ โ”œโ”€โ”€ __main__.py # Main Pulumi program +โ”‚ โ”œโ”€โ”€ workspace_config.py # Workspace-specific configuration +โ”‚ โ”œโ”€โ”€ Pulumi.yaml # Pulumi project definition +โ”‚ โ””โ”€โ”€ pyproject.toml # Python dependencies +โ””โ”€โ”€ resource_optimization/ # Resource Optimization workspace + โ”œโ”€โ”€ __main__.py # Main Pulumi program + โ”œโ”€โ”€ workspace_config.py # Workspace-specific configuration + โ”œโ”€โ”€ Pulumi.yaml # Pulumi project definition + โ””โ”€โ”€ pyproject.toml # Python dependencies +``` + +## Workspaces + +### AWS Megatests (`awsmegatests/`) + +Full testing infrastructure for nf-core pipelines with: +- **Compute Environments**: CPU, GPU, and ARM +- **S3 Bucket**: `nf-core-awsmegatests` +- **GitHub Integration**: Full CI/CD integration +- **Workspace Participants**: nf-core team members + +**Usage:** +```bash +cd awsmegatests +uv sync +uv run pulumi preview +uv run pulumi up +``` + +### Resource Optimization (`resource_optimization/`) + +Dedicated workspace for resource optimization testing with: +- **Compute Environments**: CPU only (no ARM or GPU) +- **S3 Bucket**: `nf-core-resource-optimization` +- **Purpose**: Testing and analyzing pipeline resource requirements + +**Usage:** +```bash +cd resource_optimization +uv sync +uv run pulumi preview +uv run pulumi up +``` + +## Shared Modules + +The `shared/` directory contains reusable code shared across all workspaces: + +- **`providers/`**: Factory functions for creating Pulumi providers +- **`infrastructure/`**: Modules for S3, IAM, compute environments, and credentials +- **`integrations/`**: GitHub and Seqera Platform integrations +- **`config/`**: Configuration management and ESC integration +- **`utils/`**: Logging, constants, and helper functions + +## Configuration + +Each workspace has a `workspace_config.py` file that defines: +- Workspace name and organization +- Enabled compute environments (CPU, GPU, ARM) +- AWS region and S3 bucket names +- GitHub integration settings +- Workspace participant settings + +Example: +```python +def get_workspace_config(): + return { + "workspace_name": "AWSmegatests", + "compute_environments": { + "cpu": {"enabled": True, ...}, + "gpu": {"enabled": True, ...}, + "arm": {"enabled": True, ...}, + }, + ... + } +``` + +## Adding a New Workspace + +1. Create a new directory under `seqera_platform/`: + ```bash + mkdir seqera_platform/new_workspace + ``` + +2. Copy template files from an existing workspace: + ```bash + cp awsmegatests/{Pulumi.yaml,pyproject.toml,requirements.txt,.gitignore} new_workspace/ + ``` + +3. Create `workspace_config.py` with your specific configuration + +4. Create `__main__.py` that imports from `shared/` and uses your workspace config + +5. Update the Pulumi project name in `Pulumi.yaml` + +6. Initialize and deploy: + ```bash + cd new_workspace + uv sync + pulumi stack init prod + pulumi up + ``` + +## Dependencies + +- Python >= 3.12 +- uv (Python package manager) +- Pulumi >= 3.173.0 +- Pulumi AWS Provider >= 6.81.0 +- Pulumi GitHub Provider >= 6.4.0 +- Pulumi Command Provider >= 1.0.1 + +## Environment Variables + +Required for deployment: +- `TOWER_ACCESS_TOKEN`: Seqera Platform API token +- `GITHUB_TOKEN`: GitHub personal access token +- `AWS_*`: AWS credentials (usually via ESC) + +## Migration from Legacy Structure + +The legacy `pulumi/AWSMegatests/` directory is still operational. Once this new structure is validated: + +1. Test the new `awsmegatests/` workspace in a separate Pulumi stack +2. Compare outputs and resources +3. Migrate production traffic +4. Deprecate the legacy directory + +## Notes + +- Each workspace is a separate Pulumi project with independent state +- Workspaces share code via the `shared/` module +- Each workspace can have different compute environment configurations +- S3 buckets and IAM resources are workspace-specific (no sharing) + +## Troubleshooting + +**Import errors from shared module:** +```python +# Each __main__.py adds the shared path to sys.path: +shared_path = Path(__file__).parent.parent / "shared" +sys.path.insert(0, str(shared_path)) +``` + +**Different configurations per workspace:** +Modify `workspace_config.py` in each workspace to enable/disable features. diff --git a/pulumi/seqera_platform/awsmegatests/.gitignore b/pulumi/seqera_platform/awsmegatests/.gitignore new file mode 100644 index 00000000..5e8e899e --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/.gitignore @@ -0,0 +1,10 @@ +*.pyc +__pycache__/ +venv/ +.venv/ +*.egg-info/ +dist/ +build/ +.pulumi/ +Pulumi.*.yaml +!Pulumi.yaml diff --git a/pulumi/seqera_platform/awsmegatests/Pulumi.yaml b/pulumi/seqera_platform/awsmegatests/Pulumi.yaml new file mode 100644 index 00000000..511743bb --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/Pulumi.yaml @@ -0,0 +1,11 @@ +name: seqera-awsmegatests +runtime: + name: python + options: + virtualenv: venv +description: Seqera Platform workspace for nf-core AWS megatests +config: + pulumi:tags: + value: + pulumi:template: seqera-platform-workspace + workspace: awsmegatests diff --git a/pulumi/seqera_platform/awsmegatests/README.md b/pulumi/seqera_platform/awsmegatests/README.md new file mode 100644 index 00000000..da743ba5 --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/README.md @@ -0,0 +1,155 @@ +# AWS Megatests Workspace + +Seqera Platform workspace for nf-core AWS megatests infrastructure. + +## Overview + +This workspace provides the complete testing infrastructure for nf-core pipelines on AWS, including: + +- **Multiple Compute Environments**: CPU, GPU, and ARM instances +- **S3 Storage**: Dedicated bucket for test data and results +- **GitHub CI/CD Integration**: Automatic variable and secret management +- **Team Access**: Automated workspace participant management +- **TowerForge Credentials**: IAM user and policies for compute environment access + +## Configuration + +The workspace configuration is defined in `workspace_config.py`: + +- **Workspace Name**: AWSmegatests +- **Organization**: nf-core +- **AWS Region**: eu-west-1 +- **S3 Bucket**: nf-core-awsmegatests + +### Compute Environments + +1. **CPU Environment** + - Instance types: c6id, m6id, r6id + - Max CPUs: 500 + - Fusion v2 with NVMe storage + +2. **GPU Environment** + - Instance types: g4dn, g5 + - Max CPUs: 500 + - GPU-enabled for accelerated workloads + +3. **ARM Environment** + - Instance types: c6gd, m6gd, r6gd + - Max CPUs: 500 + - ARM64 architecture testing + +## Deployment + +### Prerequisites + +1. Python 3.12+ +2. uv package manager +3. Pulumi CLI +4. AWS credentials +5. Seqera Platform access token +6. GitHub personal access token + +### Initial Setup + +```bash +# Install dependencies +uv sync + +# Login to Pulumi +pulumi login + +# Create a new stack (if needed) +pulumi stack init prod + +# Configure ESC environment (if using) +# Edit Pulumi..yaml to reference ESC environment +``` + +### Deploy Infrastructure + +```bash +# Preview changes +uv run pulumi preview + +# Deploy +uv run pulumi up + +# View outputs +uv run pulumi stack output +``` + +## Outputs + +The stack exports: + +- `workspace`: Workspace details (name, organization, ID) +- `megatests_bucket`: S3 bucket information +- `compute_env_ids`: IDs of all compute environments +- `github_resources`: GitHub variables and secrets +- `github_credential`: Seqera Platform credential for GitHub access +- `terraform_resources`: Compute environment IDs by type +- `workspace_participants`: Team member sync status + +## Structure + +``` +awsmegatests/ +โ”œโ”€โ”€ __main__.py # Main Pulumi program +โ”œโ”€โ”€ workspace_config.py # Workspace-specific configuration +โ”œโ”€โ”€ Pulumi.yaml # Pulumi project definition +โ”œโ”€โ”€ pyproject.toml # Python dependencies +โ”œโ”€โ”€ requirements.txt # Alternative dependency specification +โ””โ”€โ”€ .gitignore # Git ignore patterns +``` + +## Shared Modules + +This workspace uses shared modules from `../shared/`: +- `providers/`: AWS, GitHub, Seqera providers +- `infrastructure/`: S3, IAM, compute environments +- `integrations/`: GitHub resources, credentials +- `config/`: Configuration management +- `utils/`: Helper functions + +## Maintenance + +### Updating Compute Environments + +Edit `workspace_config.py` to modify compute environment settings: + +```python +"compute_environments": { + "cpu": { + "enabled": True, # Set to False to disable + "max_cpus": 500, # Adjust as needed + ... + } +} +``` + +Then run `pulumi up` to apply changes. + +### Adding Team Members + +Workspace participants are managed automatically via GitHub teams: +- `nf-core` team โ†’ OWNER role +- `nf-core-maintainers` team โ†’ MAINTAIN role + +Team membership updates are synced on each deployment. + +## Troubleshooting + +**Issue**: Import errors from shared modules +**Solution**: The `__main__.py` automatically adds `../shared` to the Python path. Ensure the shared directory exists. + +**Issue**: GitHub resources not creating +**Solution**: Check that `GITHUB_TOKEN` has appropriate permissions for organization variables and secrets. + +**Issue**: Compute environments not deploying +**Solution**: Verify `TOWER_ACCESS_TOKEN` and workspace ID in ESC configuration. + +## Related Documentation + +- [Main Seqera Platform README](../README.md) +- [Shared Module Documentation](../shared/README.md) +- [Legacy AWSMegatests](../../AWSMegatests/CLAUDE.md) diff --git a/pulumi/seqera_platform/awsmegatests/__init__.py b/pulumi/seqera_platform/awsmegatests/__init__.py new file mode 100644 index 00000000..6247142a --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/__init__.py @@ -0,0 +1 @@ +"""Seqera Platform AWS Megatests workspace""" diff --git a/pulumi/seqera_platform/awsmegatests/__main__.py b/pulumi/seqera_platform/awsmegatests/__main__.py new file mode 100644 index 00000000..00a27832 --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/__main__.py @@ -0,0 +1,169 @@ +"""Pulumi program for Seqera Platform - AWS Megatests workspace""" + +import sys +from pathlib import Path + +# Add shared module to Python path +shared_path = Path(__file__).parent.parent / "shared" +sys.path.insert(0, str(shared_path)) + +import pulumi + +# Import shared modules +from providers import ( + create_aws_provider, + create_github_provider, + create_seqera_provider, +) +from config import get_configuration +from infrastructure import create_s3_infrastructure, create_towerforge_credentials +from infrastructure import ( + deploy_seqera_environments_terraform, + get_compute_environment_ids_terraform, +) +from integrations import create_github_resources, create_github_credential +from integrations.workspace_participants_command import ( + create_individual_member_commands, +) + +# Import workspace-specific configuration +from workspace_config import get_workspace_config + + +def main(): + """Main Pulumi program function for AWS Megatests workspace""" + + # Get workspace-specific configuration + workspace_config = get_workspace_config() + + # Get ESC configuration + config = get_configuration() + + # Create providers + aws_provider = create_aws_provider() + github_provider = create_github_provider(config["github_token"]) + seqera_provider = create_seqera_provider(config) + + # Create GitHub credential in Seqera Platform + github_credential, github_credential_id = create_github_credential( + seqera_provider=seqera_provider, + workspace_id=int(config["tower_workspace_id"]), + github_token=config.get("platform_github_org_token", ""), + ) + + # Set up S3 infrastructure + s3_resources = create_s3_infrastructure(aws_provider) + nf_core_awsmegatests_bucket = s3_resources["bucket"] + + # Create TowerForge IAM credentials and upload to Seqera Platform + ( + towerforge_access_key_id, + towerforge_access_key_secret, + seqera_credentials_id, + seqera_credential_resource, + iam_policy_hash, + ) = create_towerforge_credentials( + aws_provider, + nf_core_awsmegatests_bucket, + seqera_provider, + float(config["tower_workspace_id"]), + ) + + # Deploy Seqera Platform compute environments + terraform_resources = deploy_seqera_environments_terraform( + config, + seqera_credentials_id, + seqera_provider, + seqera_credential_resource, + iam_policy_hash, + ) + + # Get compute environment IDs + compute_env_ids = get_compute_environment_ids_terraform(terraform_resources) + deployment_method = "terraform-provider" + + # Create GitHub resources + github_resources = create_github_resources( + github_provider, + compute_env_ids, + config["tower_workspace_id"], + tower_access_token=config["tower_access_token"], + ) + + # Add workspace participants + setup_cmd, member_commands = create_individual_member_commands( + workspace_id=int(config["tower_workspace_id"]), + token=config["tower_access_token"], + github_token=config["github_token"], + opts=pulumi.ResourceOptions(depends_on=[seqera_credential_resource]), + ) + + # Exports + pulumi.export( + "workspace", + { + "name": workspace_config["workspace_name"], + "organization": workspace_config["organization_name"], + "workspace_id": config["tower_workspace_id"], + }, + ) + + pulumi.export( + "megatests_bucket", + { + "name": nf_core_awsmegatests_bucket.bucket, + "arn": nf_core_awsmegatests_bucket.arn, + "region": workspace_config["aws_region"], + }, + ) + + pulumi.export( + "github_resources", + { + "variables": { + k: v.id for k, v in github_resources.get("variables", {}).items() + } + if github_resources.get("variables") + else {}, + "secrets": {k: v.id for k, v in github_resources.get("secrets", {}).items()} + if github_resources.get("secrets") + else {}, + "manual_secret_commands": github_resources.get("gh_cli_commands", []), + }, + ) + + pulumi.export("compute_env_ids", compute_env_ids) + pulumi.export("deployment_method", deployment_method) + + pulumi.export( + "github_credential", + { + "credential_id": github_credential_id, + "credential_name": "nf-core-github-finegrained", + }, + ) + + pulumi.export( + "terraform_resources", + { + "cpu_env_id": terraform_resources["cpu_env"].compute_env_id, + "gpu_env_id": terraform_resources["gpu_env"].compute_env_id, + "arm_env_id": terraform_resources["arm_env"].compute_env_id, + }, + ) + + pulumi.export( + "workspace_participants", + { + "setup_command_id": setup_cmd.id, + "individual_member_commands": { + username: {"command_id": cmd.id, "github_username": username} + for username, cmd in member_commands.items() + }, + "total_tracked_members": len(member_commands), + }, + ) + + +if __name__ == "__main__": + main() diff --git a/pulumi/seqera_platform/awsmegatests/pyproject.toml b/pulumi/seqera_platform/awsmegatests/pyproject.toml new file mode 100644 index 00000000..8ae397db --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "seqera-awsmegatests" +version = "0.1.0" +description = "Pulumi project for Seqera Platform - AWS Megatests workspace" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "pulumi>=3.173.0,<4.0.0", + "pulumi-aws>=6.81.0,<7.0.0", + "pulumi-github>=6.4.0,<7.0.0", + "pulumi-command>=1.0.1,<2.0.0", +] diff --git a/pulumi/seqera_platform/awsmegatests/requirements.txt b/pulumi/seqera_platform/awsmegatests/requirements.txt new file mode 100644 index 00000000..2adc78fa --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/requirements.txt @@ -0,0 +1,4 @@ +pulumi>=3.173.0,<4.0.0 +pulumi-aws>=6.81.0,<7.0.0 +pulumi-github>=6.4.0,<7.0.0 +pulumi-command>=1.0.1,<2.0.0 diff --git a/pulumi/seqera_platform/awsmegatests/workspace_config.py b/pulumi/seqera_platform/awsmegatests/workspace_config.py new file mode 100644 index 00000000..1983dc5f --- /dev/null +++ b/pulumi/seqera_platform/awsmegatests/workspace_config.py @@ -0,0 +1,55 @@ +"""Configuration specific to the AWS Megatests workspace""" + +from typing import Dict, Any + + +def get_workspace_config() -> Dict[str, Any]: + """ + Get workspace-specific configuration for AWS Megatests. + + Returns: + Dict containing workspace settings including compute environment configuration + """ + return { + "workspace_name": "AWSmegatests", + "organization_name": "nf-core", + "description": "Workspace for nf-core AWS megatests infrastructure", + + # Compute environment configuration + "compute_environments": { + "cpu": { + "enabled": True, + "name": "aws_ireland_fusionv2_nvme_cpu", + "instance_types": ["c6id", "m6id", "r6id"], + "max_cpus": 500, + }, + "gpu": { + "enabled": True, + "name": "aws_ireland_fusionv2_nvme_gpu", + "instance_types": ["g4dn", "g5"], + "max_cpus": 500, + }, + "arm": { + "enabled": True, + "name": "aws_ireland_fusionv2_nvme_arm", + "instance_types": ["c6gd", "m6gd", "r6gd"], + "max_cpus": 500, + }, + }, + + # AWS configuration + "aws_region": "eu-west-1", + "s3_bucket_name": "nf-core-awsmegatests", + + # GitHub integration + "github_integration": { + "enabled": True, + "organization": "nf-core", + }, + + # Workspace participants + "workspace_participants": { + "enabled": True, + "teams": ["nf-core", "nf-core-maintainers"], + }, + } diff --git a/pulumi/seqera_platform/resource_optimization/.gitignore b/pulumi/seqera_platform/resource_optimization/.gitignore new file mode 100644 index 00000000..5e8e899e --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/.gitignore @@ -0,0 +1,10 @@ +*.pyc +__pycache__/ +venv/ +.venv/ +*.egg-info/ +dist/ +build/ +.pulumi/ +Pulumi.*.yaml +!Pulumi.yaml diff --git a/pulumi/seqera_platform/resource_optimization/Pulumi.yaml b/pulumi/seqera_platform/resource_optimization/Pulumi.yaml new file mode 100644 index 00000000..16a80d7a --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/Pulumi.yaml @@ -0,0 +1,11 @@ +name: seqera-resource-optimization +runtime: + name: python + options: + virtualenv: venv +description: Seqera Platform workspace for resource optimization testing +config: + pulumi:tags: + value: + pulumi:template: seqera-platform-workspace + workspace: resource-optimization diff --git a/pulumi/seqera_platform/resource_optimization/README.md b/pulumi/seqera_platform/resource_optimization/README.md new file mode 100644 index 00000000..3642225e --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/README.md @@ -0,0 +1,188 @@ +# Resource Optimization Workspace + +Seqera Platform workspace for resource optimization testing and analysis. + +## Overview + +This workspace provides a focused testing environment for analyzing and optimizing pipeline resource requirements. It includes: + +- **CPU-Only Compute Environment**: Single compute environment for resource profiling +- **S3 Storage**: Dedicated bucket for test results +- **Minimal Configuration**: Streamlined setup for resource analysis +- **TowerForge Credentials**: IAM user and policies for compute environment access + +Unlike the full AWS Megatests workspace, this workspace intentionally omits GPU and ARM compute environments to focus specifically on CPU resource optimization work. + +## Configuration + +The workspace configuration is defined in `workspace_config.py`: + +- **Workspace Name**: ResourceOptimization +- **Organization**: nf-core +- **AWS Region**: eu-west-1 +- **S3 Bucket**: nf-core-resource-optimization + +### Compute Environments + +1. **CPU Environment** (Only) + - Instance types: c6id, m6id, r6id + - Max CPUs: 500 + - Fusion v2 with NVMe storage + - SPOT provisioning model + +**Note**: GPU and ARM environments are explicitly disabled for this workspace. + +## Deployment + +### Prerequisites + +1. Python 3.12+ +2. uv package manager +3. Pulumi CLI +4. AWS credentials +5. Seqera Platform access token + +### Initial Setup + +```bash +# Install dependencies +uv sync + +# Login to Pulumi +pulumi login + +# Create a new stack +pulumi stack init prod + +# Configure ESC environment +# Edit Pulumi..yaml to reference ESC environment +``` + +### Deploy Infrastructure + +```bash +# Preview changes +uv run pulumi preview + +# Deploy +uv run pulumi up + +# View outputs +uv run pulumi stack output +``` + +## Outputs + +The stack exports: + +- `workspace`: Workspace details (name, organization, ID, description) +- `s3_bucket`: S3 bucket information +- `compute_env_ids`: IDs of compute environment (CPU only) +- `terraform_resources`: CPU compute environment ID +- `workspace_participants`: Team member sync status (if enabled) + +**Note**: GitHub integration outputs are omitted as GitHub integration is disabled by default. + +## Structure + +``` +resource_optimization/ +โ”œโ”€โ”€ __main__.py # Main Pulumi program +โ”œโ”€โ”€ workspace_config.py # Workspace-specific configuration +โ”œโ”€โ”€ Pulumi.yaml # Pulumi project definition +โ”œโ”€โ”€ pyproject.toml # Python dependencies +โ”œโ”€โ”€ requirements.txt # Alternative dependency specification +โ””โ”€โ”€ .gitignore # Git ignore patterns +``` + +## Shared Modules + +This workspace uses shared modules from `../shared/`: +- `providers/`: AWS, Seqera providers +- `infrastructure/`: S3, IAM, compute environments +- `config/`: Configuration management +- `utils/`: Helper functions + +## Use Cases + +This workspace is designed for: + +1. **Resource Profiling**: Running pipelines with detailed resource monitoring +2. **Optimization Testing**: Testing resource requirement adjustments +3. **Cost Analysis**: Analyzing compute costs for different configurations +4. **Performance Benchmarking**: Comparing execution times across instance types + +## Customization + +### Enabling GitHub Integration + +Edit `workspace_config.py`: + +```python +"github_integration": { + "enabled": True, # Change from False + "organization": "nf-core", +} +``` + +### Adjusting Compute Limits + +Edit `workspace_config.py`: + +```python +"compute_environments": { + "cpu": { + "enabled": True, + "max_cpus": 1000, # Increase from 500 + ... + } +} +``` + +### Adding GPU or ARM Support + +If resource optimization work expands to include GPU or ARM: + +```python +"compute_environments": { + "cpu": {...}, + "gpu": { + "enabled": True, # Change from False + "name": "aws_ireland_fusionv2_nvme_gpu", + "instance_types": ["g4dn", "g5"], + "max_cpus": 500, + }, +} +``` + +## Maintenance + +### Resource Monitoring + +Monitor resource usage through: +1. Seqera Platform dashboard +2. CloudWatch metrics +3. S3 bucket analytics + +### Cost Optimization + +- Review instance type usage +- Adjust max_cpus based on actual usage +- Consider SPOT vs on-demand mix + +## Troubleshooting + +**Issue**: Import errors from shared modules +**Solution**: The `__main__.py` automatically adds `../shared` to the Python path. Ensure the shared directory exists. + +**Issue**: Compute environment not deploying +**Solution**: Verify `TOWER_ACCESS_TOKEN` and workspace ID in ESC configuration. + +**Issue**: S3 bucket creation fails +**Solution**: Check if bucket name is globally unique and region is correct. + +## Related Documentation + +- [Main Seqera Platform README](../README.md) +- [Shared Module Documentation](../shared/README.md) +- [AWS Megatests Workspace](../awsmegatests/README.md) diff --git a/pulumi/seqera_platform/resource_optimization/__init__.py b/pulumi/seqera_platform/resource_optimization/__init__.py new file mode 100644 index 00000000..9e41c308 --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/__init__.py @@ -0,0 +1 @@ +"""Seqera Platform Resource Optimization workspace""" diff --git a/pulumi/seqera_platform/resource_optimization/__main__.py b/pulumi/seqera_platform/resource_optimization/__main__.py new file mode 100644 index 00000000..5b4ab16a --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/__main__.py @@ -0,0 +1,182 @@ +"""Pulumi program for Seqera Platform - Resource Optimization workspace""" + +import sys +from pathlib import Path + +# Add shared module to Python path +shared_path = Path(__file__).parent.parent / "shared" +sys.path.insert(0, str(shared_path)) + +import pulumi + +# Import shared modules +from providers import ( + create_aws_provider, + create_github_provider, + create_seqera_provider, +) +from config import get_configuration +from infrastructure import create_s3_infrastructure, create_towerforge_credentials +from infrastructure import ( + deploy_seqera_environments_terraform, + get_compute_environment_ids_terraform, +) +from integrations import create_github_resources, create_github_credential +from integrations.workspace_participants_command import ( + create_individual_member_commands, +) + +# Import workspace-specific configuration +from workspace_config import get_workspace_config + + +def main(): + """Main Pulumi program function for Resource Optimization workspace""" + + # Get workspace-specific configuration + workspace_config = get_workspace_config() + + # Get ESC configuration + config = get_configuration() + + # Create providers + aws_provider = create_aws_provider() + seqera_provider = create_seqera_provider(config) + + # Only create GitHub provider if GitHub integration is enabled + github_provider = None + github_credential = None + github_credential_id = None + if workspace_config["github_integration"]["enabled"]: + github_provider = create_github_provider(config["github_token"]) + # Create GitHub credential in Seqera Platform + github_credential, github_credential_id = create_github_credential( + seqera_provider=seqera_provider, + workspace_id=int(config["tower_workspace_id"]), + github_token=config.get("platform_github_org_token", ""), + ) + + # Set up S3 infrastructure with workspace-specific bucket name + s3_resources = create_s3_infrastructure( + aws_provider, + bucket_name=workspace_config["s3_bucket_name"] + ) + resource_opt_bucket = s3_resources["bucket"] + + # Create TowerForge IAM credentials and upload to Seqera Platform + ( + towerforge_access_key_id, + towerforge_access_key_secret, + seqera_credentials_id, + seqera_credential_resource, + iam_policy_hash, + ) = create_towerforge_credentials( + aws_provider, + resource_opt_bucket, + seqera_provider, + float(config["tower_workspace_id"]), + ) + + # Deploy Seqera Platform compute environments (CPU only for this workspace) + # Note: The compute environment deployment will need to be adapted to handle + # the workspace config that disables GPU and ARM + terraform_resources = deploy_seqera_environments_terraform( + config, + seqera_credentials_id, + seqera_provider, + seqera_credential_resource, + iam_policy_hash, + ) + + # Get compute environment IDs + compute_env_ids = get_compute_environment_ids_terraform(terraform_resources) + deployment_method = "terraform-provider" + + # Create GitHub resources only if enabled + github_resources = {} + if workspace_config["github_integration"]["enabled"] and github_provider: + github_resources = create_github_resources( + github_provider, + compute_env_ids, + config["tower_workspace_id"], + tower_access_token=config["tower_access_token"], + ) + + # Add workspace participants if enabled + if workspace_config["workspace_participants"]["enabled"]: + setup_cmd, member_commands = create_individual_member_commands( + workspace_id=int(config["tower_workspace_id"]), + token=config["tower_access_token"], + github_token=config["github_token"], + opts=pulumi.ResourceOptions(depends_on=[seqera_credential_resource]), + ) + + pulumi.export( + "workspace_participants", + { + "setup_command_id": setup_cmd.id, + "individual_member_commands": { + username: {"command_id": cmd.id, "github_username": username} + for username, cmd in member_commands.items() + }, + "total_tracked_members": len(member_commands), + }, + ) + + # Exports + pulumi.export( + "workspace", + { + "name": workspace_config["workspace_name"], + "organization": workspace_config["organization_name"], + "workspace_id": config["tower_workspace_id"], + "description": workspace_config["description"], + }, + ) + + pulumi.export( + "s3_bucket", + { + "name": resource_opt_bucket.bucket, + "arn": resource_opt_bucket.arn, + "region": workspace_config["aws_region"], + }, + ) + + if github_resources: + pulumi.export( + "github_resources", + { + "variables": { + k: v.id for k, v in github_resources.get("variables", {}).items() + } + if github_resources.get("variables") + else {}, + "secrets": {k: v.id for k, v in github_resources.get("secrets", {}).items()} + if github_resources.get("secrets") + else {}, + }, + ) + + pulumi.export("compute_env_ids", compute_env_ids) + pulumi.export("deployment_method", deployment_method) + + if github_credential_id: + pulumi.export( + "github_credential", + { + "credential_id": github_credential_id, + "credential_name": "nf-core-github-finegrained", + }, + ) + + # Only export enabled compute environments + terraform_exports = {} + if workspace_config["compute_environments"]["cpu"]["enabled"]: + terraform_exports["cpu_env_id"] = terraform_resources["cpu_env"].compute_env_id + + pulumi.export("terraform_resources", terraform_exports) + + +if __name__ == "__main__": + main() diff --git a/pulumi/seqera_platform/resource_optimization/pyproject.toml b/pulumi/seqera_platform/resource_optimization/pyproject.toml new file mode 100644 index 00000000..4f1be96c --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "seqera-resource-optimization" +version = "0.1.0" +description = "Pulumi project for Seqera Platform - Resource Optimization workspace" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "pulumi>=3.173.0,<4.0.0", + "pulumi-aws>=6.81.0,<7.0.0", + "pulumi-github>=6.4.0,<7.0.0", + "pulumi-command>=1.0.1,<2.0.0", +] diff --git a/pulumi/seqera_platform/resource_optimization/requirements.txt b/pulumi/seqera_platform/resource_optimization/requirements.txt new file mode 100644 index 00000000..2adc78fa --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/requirements.txt @@ -0,0 +1,4 @@ +pulumi>=3.173.0,<4.0.0 +pulumi-aws>=6.81.0,<7.0.0 +pulumi-github>=6.4.0,<7.0.0 +pulumi-command>=1.0.1,<2.0.0 diff --git a/pulumi/seqera_platform/resource_optimization/workspace_config.py b/pulumi/seqera_platform/resource_optimization/workspace_config.py new file mode 100644 index 00000000..43c207a7 --- /dev/null +++ b/pulumi/seqera_platform/resource_optimization/workspace_config.py @@ -0,0 +1,51 @@ +"""Configuration specific to the Resource Optimization workspace""" + +from typing import Dict, Any + + +def get_workspace_config() -> Dict[str, Any]: + """ + Get workspace-specific configuration for Resource Optimization testing. + + This workspace only includes CPU compute environments (no ARM or GPU) + for Florian's resource optimization work. + + Returns: + Dict containing workspace settings including compute environment configuration + """ + return { + "workspace_name": "ResourceOptimization", + "organization_name": "nf-core", + "description": "Workspace for resource optimization testing and analysis", + + # Compute environment configuration - CPU only + "compute_environments": { + "cpu": { + "enabled": True, + "name": "aws_ireland_fusionv2_nvme_cpu", + "instance_types": ["c6id", "m6id", "r6id"], + "max_cpus": 500, + }, + "gpu": { + "enabled": False, # Not needed for resource optimization + }, + "arm": { + "enabled": False, # Not needed for resource optimization + }, + }, + + # AWS configuration + "aws_region": "eu-west-1", + "s3_bucket_name": "nf-core-resource-optimization", + + # GitHub integration - may not be needed for this workspace + "github_integration": { + "enabled": False, # Can enable later if needed + }, + + # Workspace participants - configure based on team needs + "workspace_participants": { + "enabled": True, + "teams": [], # Configure team access as needed + }, + } diff --git a/pulumi/seqera_platform/shared/__init__.py b/pulumi/seqera_platform/shared/__init__.py new file mode 100644 index 00000000..b5e4da14 --- /dev/null +++ b/pulumi/seqera_platform/shared/__init__.py @@ -0,0 +1,8 @@ +"""AWS Megatests Infrastructure Package + +This package provides modular infrastructure components for the nf-core AWS Megatests +Pulumi project, including provider configurations, infrastructure resources, +and third-party integrations. +""" + +__version__ = "1.0.0" diff --git a/pulumi/seqera_platform/shared/config/__init__.py b/pulumi/seqera_platform/shared/config/__init__.py new file mode 100644 index 00000000..0d879069 --- /dev/null +++ b/pulumi/seqera_platform/shared/config/__init__.py @@ -0,0 +1,5 @@ +"""Configuration management for AWS Megatests infrastructure.""" + +from .settings import get_configuration, ConfigurationError + +__all__ = ["get_configuration", "ConfigurationError"] diff --git a/pulumi/seqera_platform/shared/config/settings.py b/pulumi/seqera_platform/shared/config/settings.py new file mode 100644 index 00000000..335c5534 --- /dev/null +++ b/pulumi/seqera_platform/shared/config/settings.py @@ -0,0 +1,117 @@ +"""Configuration management for AWS Megatests infrastructure using Pulumi ESC.""" + +import os +from typing import Dict, Any, Optional +from dataclasses import dataclass + +from ..utils.constants import DEFAULT_ENV_VARS + + +class ConfigurationError(Exception): + """Exception raised when configuration validation fails.""" + + pass + + +@dataclass +class InfrastructureConfig: + """Typed configuration for AWS Megatests infrastructure. + + Attributes: + tower_access_token: Seqera Platform access token + tower_workspace_id: Seqera Platform workspace ID + github_token: GitHub personal access token (classic) + platform_github_org_token: GitHub fine-grained token to avoid rate limits when pulling pipelines + """ + + tower_access_token: Optional[str] + tower_workspace_id: str + github_token: Optional[str] + platform_github_org_token: Optional[str] + + def validate(self) -> None: + """Validate configuration values. + + Raises: + ConfigurationError: If required configuration is missing or invalid + """ + missing_vars = [] + + if not self.tower_access_token: + missing_vars.append("TOWER_ACCESS_TOKEN") + + if not self.github_token: + missing_vars.append("GITHUB_TOKEN") + + # Validate workspace ID is numeric + if ( + not self.tower_workspace_id + or not self.tower_workspace_id.replace(".", "").isdigit() + ): + missing_vars.append("TOWER_WORKSPACE_ID (must be numeric)") + + if missing_vars: + raise ConfigurationError( + f"Missing or invalid required environment variables: {', '.join(missing_vars)}. " + "Please ensure these are set in your ESC environment." + ) + + +def _get_env_var_with_fallback( + var_name: str, fallback: Optional[str] = None +) -> Optional[str]: + """Get environment variable with optional fallback. + + Args: + var_name: Name of the environment variable + fallback: Optional fallback value if variable is not set + + Returns: + Optional[str]: Environment variable value or fallback + """ + value = os.environ.get(var_name) + if not value and fallback: + print( + f"Warning: {var_name} not found in ESC environment, using fallback: {fallback}" + ) + return fallback + return value + + +def get_configuration() -> Dict[str, Any]: + """Get configuration values from ESC environment variables. + + All configuration comes from ESC environment variables which are automatically + set when the ESC environment is imported. + + Returns: + Dict[str, Any]: Configuration dictionary compatible with existing code + + Raises: + ConfigurationError: If required configuration is missing or invalid + """ + # Get workspace ID from environment or fall back to default + workspace_id = _get_env_var_with_fallback( + "TOWER_WORKSPACE_ID", DEFAULT_ENV_VARS.get("TOWER_WORKSPACE_ID") + ) + + # Create typed configuration object + config = InfrastructureConfig( + tower_access_token=os.environ.get("TOWER_ACCESS_TOKEN"), + tower_workspace_id=workspace_id or "", + github_token=os.environ.get("GITHUB_TOKEN"), + platform_github_org_token=os.environ.get("PLATFORM_GITHUB_ORG_TOKEN"), + ) + + # Validate configuration + config.validate() + + # Return dictionary format for backward compatibility + return { + "tower_access_token": config.tower_access_token, + "tower_workspace_id": config.tower_workspace_id, + "github_token": config.github_token, + "platform_github_org_token": config.platform_github_org_token, + # AWS credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN) + # are automatically handled by ESC and picked up by the AWS provider + } diff --git a/pulumi/seqera_platform/shared/infrastructure/__init__.py b/pulumi/seqera_platform/shared/infrastructure/__init__.py new file mode 100644 index 00000000..f342fd1e --- /dev/null +++ b/pulumi/seqera_platform/shared/infrastructure/__init__.py @@ -0,0 +1,16 @@ +"""Infrastructure components for AWS Megatests.""" + +from .s3 import create_s3_infrastructure +from .credentials import create_towerforge_credentials, get_towerforge_resources +from .compute_environments import ( + deploy_seqera_environments_terraform, + get_compute_environment_ids_terraform, +) + +__all__ = [ + "create_s3_infrastructure", + "create_towerforge_credentials", + "get_towerforge_resources", + "deploy_seqera_environments_terraform", + "get_compute_environment_ids_terraform", +] diff --git a/pulumi/seqera_platform/shared/infrastructure/compute_environments.py b/pulumi/seqera_platform/shared/infrastructure/compute_environments.py new file mode 100644 index 00000000..ff254467 --- /dev/null +++ b/pulumi/seqera_platform/shared/infrastructure/compute_environments.py @@ -0,0 +1,387 @@ +"""Seqera Platform compute environment deployment using Seqera Terraform provider.""" + +import json +import os +from typing import Dict, Any, Optional + +import pulumi +import pulumi_seqera as seqera + +from ..utils.constants import ( + COMPUTE_ENV_NAMES, + COMPUTE_ENV_DESCRIPTIONS, + CONFIG_FILES, + NEXTFLOW_CONFIG_FILES, + DEFAULT_COMPUTE_ENV_CONFIG, + DEFAULT_FORGE_CONFIG, + TIMEOUTS, + ERROR_MESSAGES, +) + + +class ComputeEnvironmentError(Exception): + """Exception raised when compute environment operations fail.""" + + pass + + +class ConfigurationError(Exception): + """Exception raised when configuration loading fails.""" + + pass + + +def load_nextflow_config(env_type: str) -> str: + """Load and merge Nextflow configuration from base and environment-specific files. + + Args: + env_type: Environment type (cpu, gpu, arm) + + Returns: + str: Merged Nextflow configuration content + + Raises: + ConfigurationError: If file loading fails + """ + config_file = NEXTFLOW_CONFIG_FILES.get(env_type) + if not config_file: + raise ConfigurationError( + f"No Nextflow config file defined for environment type: {env_type}" + ) + + if not os.path.exists(config_file): + raise FileNotFoundError(f"Nextflow config file not found: {config_file}") + + # Load base configuration + base_config_file = os.path.join( + os.path.dirname(config_file), "nextflow-base.config" + ) + base_config = "" + if os.path.exists(base_config_file): + try: + with open(base_config_file, "r") as f: + base_config = f.read().strip() + except Exception as e: + raise ConfigurationError( + f"Failed to read base Nextflow config file {base_config_file}: {e}" + ) + + # Load environment-specific configuration + try: + with open(config_file, "r") as f: + env_config = f.read().strip() + except Exception as e: + raise ConfigurationError( + f"Failed to read Nextflow config file {config_file}: {e}" + ) + + # Remove includeConfig line from environment config since we're injecting base config + env_config_lines = env_config.split("\n") + env_config_filtered = [ + line + for line in env_config_lines + if not line.strip().startswith("includeConfig") + ] + env_config_clean = "\n".join(env_config_filtered) + + # Merge base config with environment-specific config + if base_config: + merged_config = f"{base_config}\n\n{env_config_clean}" + else: + merged_config = env_config_clean + + return merged_config.strip() + + +def load_config_file(filename: str) -> Dict[str, Any]: + """Load configuration file with comprehensive error handling. + + Args: + filename: Path to the JSON configuration file + + Returns: + Dict[str, Any]: Loaded configuration data + + Raises: + ConfigurationError: If file loading or parsing fails + """ + if not os.path.exists(filename): + raise FileNotFoundError( + ERROR_MESSAGES["config_file_not_found"].format(filename) + ) + + with open(filename, "r") as f: + config_data = json.load(f) + + return config_data + + +def create_forge_config( + config_args: Dict[str, Any], +) -> seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs: + """Create forge configuration for AWS Batch compute environment. + + Args: + config_args: Configuration arguments from JSON file + + Returns: + seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs: Forge configuration + """ + forge_data = config_args.get("forge", {}) + + return seqera.ComputeEnvComputeEnvConfigAwsBatchForgeArgs( + type=forge_data.get("type", DEFAULT_FORGE_CONFIG["type"]), + min_cpus=forge_data.get("minCpus", DEFAULT_FORGE_CONFIG["minCpus"]), + max_cpus=forge_data.get("maxCpus", DEFAULT_FORGE_CONFIG["maxCpus"]), + gpu_enabled=forge_data.get("gpuEnabled", DEFAULT_FORGE_CONFIG["gpuEnabled"]), + instance_types=forge_data.get( + "instanceTypes", DEFAULT_FORGE_CONFIG["instanceTypes"] + ), + subnets=forge_data.get("subnets", DEFAULT_FORGE_CONFIG["subnets"]), + security_groups=forge_data.get( + "securityGroups", DEFAULT_FORGE_CONFIG["securityGroups"] + ), + dispose_on_deletion=forge_data.get( + "disposeOnDeletion", DEFAULT_FORGE_CONFIG["disposeOnDeletion"] + ), + allow_buckets=forge_data.get( + "allowBuckets", DEFAULT_FORGE_CONFIG["allowBuckets"] + ), + efs_create=forge_data.get("efsCreate", DEFAULT_FORGE_CONFIG["efsCreate"]), + ebs_boot_size=forge_data.get( + "ebsBootSize", DEFAULT_FORGE_CONFIG["ebsBootSize"] + ), + fargate_head_enabled=forge_data.get( + "fargateHeadEnabled", DEFAULT_FORGE_CONFIG["fargateHeadEnabled"] + ), + arm64_enabled=forge_data.get( + "arm64Enabled", DEFAULT_FORGE_CONFIG["arm64Enabled"] + ), + ) + + +def create_compute_environment( + provider: seqera.Provider, + name: str, + credentials_id: str, + workspace_id: float, + config_args: Dict[str, Any], + env_type: str, + description: Optional[str] = None, + depends_on: Optional[list] = None, + iam_policy_version: Optional[str] = None, +) -> seqera.ComputeEnv: + """Create a Seqera compute environment using Terraform provider with error handling. + + Args: + provider: Configured Seqera provider instance + name: Name for the compute environment + credentials_id: Seqera credentials ID + workspace_id: Seqera workspace ID + config_args: Configuration arguments from JSON file + env_type: Environment type (cpu, gpu, arm) for loading external nextflow config + description: Optional description for the compute environment + depends_on: Optional list of resources this compute environment depends on + iam_policy_version: Optional IAM policy version hash to trigger recreation on policy changes + + Returns: + seqera.ComputeEnv: Created compute environment resource + + Raises: + ComputeEnvironmentError: If compute environment creation fails + ValueError: If required parameters are missing + ConfigurationError: If nextflow config loading fails + """ + pulumi.log.info(f"Creating compute environment: {name}") + + # Validate input parameters + if not name or not credentials_id: + raise ValueError(ERROR_MESSAGES["missing_compute_env_params"].format(name)) + + if not config_args: + raise ValueError(ERROR_MESSAGES["missing_config_args"].format(name)) + + # Create the forge configuration + forge_config = create_forge_config(config_args) + + # Load Nextflow configuration from external file + nextflow_config = load_nextflow_config(env_type) + + # Create AWS Batch configuration + aws_batch_config = seqera.ComputeEnvComputeEnvConfigAwsBatchArgs( + region=config_args.get("region", DEFAULT_COMPUTE_ENV_CONFIG["region"]), + work_dir=config_args.get("workDir", DEFAULT_COMPUTE_ENV_CONFIG["workDir"]), + forge=forge_config, + wave_enabled=config_args.get( + "waveEnabled", DEFAULT_COMPUTE_ENV_CONFIG["waveEnabled"] + ), + fusion2_enabled=config_args.get( + "fusion2Enabled", DEFAULT_COMPUTE_ENV_CONFIG["fusion2Enabled"] + ), + nvnme_storage_enabled=config_args.get( + "nvnmeStorageEnabled", DEFAULT_COMPUTE_ENV_CONFIG["nvnmeStorageEnabled"] + ), + fusion_snapshots=config_args.get( + "fusionSnapshots", DEFAULT_COMPUTE_ENV_CONFIG["fusionSnapshots"] + ), + nextflow_config=nextflow_config, # Use external config file + ) + + # Create the compute environment configuration + compute_env_config = seqera.ComputeEnvComputeEnvConfigArgs( + aws_batch=aws_batch_config + ) + + # Create the compute environment args + compute_env_args = seqera.ComputeEnvComputeEnvArgs( + name=name, + platform="aws-batch", + credentials_id=credentials_id, + config=compute_env_config, + description=description, + ) + + # Add IAM policy version to compute environment description to trigger recreation on policy changes + if iam_policy_version: + # Append policy version hash to description to force recreation when IAM policies change + policy_suffix = f" (IAM Policy Version: {iam_policy_version[:8]})" + if description: + compute_env_args = seqera.ComputeEnvComputeEnvArgs( + name=name, + platform="aws-batch", + credentials_id=credentials_id, + config=compute_env_config, + description=f"{description}{policy_suffix}", + ) + else: + compute_env_args = seqera.ComputeEnvComputeEnvArgs( + name=name, + platform="aws-batch", + credentials_id=credentials_id, + config=compute_env_config, + description=f"Compute environment{policy_suffix}", + ) + + # Create the compute environment resource + resource_options = pulumi.ResourceOptions( + provider=provider, + # Force delete before replace to avoid name conflicts + delete_before_replace=True, + # Add custom timeout for compute environment creation + custom_timeouts=pulumi.CustomTimeouts( + create=TIMEOUTS["compute_env_create"], + update=TIMEOUTS["compute_env_update"], + delete=TIMEOUTS["compute_env_delete"], + ), + ) + + # Add dependencies if specified + if depends_on: + resource_options.depends_on = depends_on + + compute_env = seqera.ComputeEnv( + name, + compute_env=compute_env_args, + workspace_id=workspace_id, + opts=resource_options, + ) + + return compute_env + + +def deploy_seqera_environments_terraform( + config: Dict[str, Any], + towerforge_credentials_id: str, + seqera_provider: Optional[seqera.Provider] = None, + seqera_credential_resource: Optional[seqera.Credential] = None, + iam_policy_hash: Optional[str] = None, +) -> Dict[str, Any]: + """Deploy Seqera Platform compute environments using Terraform provider. + + Args: + config: Configuration dictionary + towerforge_credentials_id: Dynamic TowerForge credentials ID + seqera_provider: Optional existing Seqera provider instance + seqera_credential_resource: Optional Seqera credential resource for dependency + iam_policy_hash: Optional IAM policy hash to force recreation on policy changes + + Returns: + Dict[str, Any]: Dictionary containing created compute environments and provider + + Raises: + ConfigurationError: If configuration loading fails + ComputeEnvironmentError: If compute environment creation fails + ValueError: If workspace ID is invalid + """ + pulumi.log.info( + "Starting Seqera compute environment deployment using Terraform provider" + ) + + # Use provided seqera provider or create a new one + if seqera_provider is not None: + provider = seqera_provider + pulumi.log.info("Using existing Seqera provider") + else: + # Import here to avoid circular imports + from ..providers.seqera import create_seqera_provider + + provider = create_seqera_provider(config) + + # Load all configuration files + cpu_config = load_config_file(CONFIG_FILES["cpu"]) + gpu_config = load_config_file(CONFIG_FILES["gpu"]) + arm_config = load_config_file(CONFIG_FILES["arm"]) + + # Validate workspace ID + workspace_id = float(config["tower_workspace_id"]) + + # Create all three compute environments + environments = {} + + # Set up dependencies - compute environments depend on Seqera credential resource + depends_on_resources = [] + if seqera_credential_resource: + depends_on_resources.append(seqera_credential_resource) + + for env_type, config_data in [ + ("cpu", cpu_config), + ("gpu", gpu_config), + ("arm", arm_config), + ]: + env_name = COMPUTE_ENV_NAMES[env_type] + description = COMPUTE_ENV_DESCRIPTIONS[env_type] + + environments[f"{env_type}_env"] = create_compute_environment( + provider=provider, + name=env_name, + credentials_id=towerforge_credentials_id, + workspace_id=workspace_id, + config_args=config_data, + env_type=env_type, + description=description, + depends_on=depends_on_resources if depends_on_resources else None, + iam_policy_version=iam_policy_hash, + ) + + return { + **environments, + "provider": provider, + } + + +def get_compute_environment_ids_terraform( + terraform_resources: Dict[str, Any], +) -> Dict[str, Any]: + """Extract compute environment IDs from Terraform provider resources. + + Args: + terraform_resources: Dictionary containing terraform resources + + Returns: + Dict[str, Any]: Dictionary mapping environment types to their IDs + """ + return { + "cpu": terraform_resources["cpu_env"].compute_env_id, + "gpu": terraform_resources["gpu_env"].compute_env_id, + "arm": terraform_resources["arm_env"].compute_env_id, + } diff --git a/pulumi/seqera_platform/shared/infrastructure/credentials.py b/pulumi/seqera_platform/shared/infrastructure/credentials.py new file mode 100644 index 00000000..d133dec3 --- /dev/null +++ b/pulumi/seqera_platform/shared/infrastructure/credentials.py @@ -0,0 +1,508 @@ +"""TowerForge IAM credentials management module. + +This module creates and manages IAM resources for TowerForge AWS operations, +including policies for Forge operations, Launch operations, and S3 bucket access. +It also uploads the credentials to Seqera Platform for use by compute environments. +""" + +import json +import hashlib +from typing import Optional, Tuple, Dict, Any + +import pulumi +import pulumi_aws as aws +import pulumi_seqera as seqera + +from ..utils.constants import ( + TOWERFORGE_USER_NAME, + TOWERFORGE_POLICY_NAMES, + TOWERFORGE_CREDENTIAL_NAME, + TOWERFORGE_CREDENTIAL_DESCRIPTION, + TIMEOUTS, +) + + +class CredentialError(Exception): + """Exception raised when credential operations fail.""" + + pass + + +def _create_forge_policy_document() -> Dict[str, Any]: + """Create TowerForge Forge Policy document with comprehensive permissions.""" + return { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TowerForge0", + "Effect": "Allow", + "Action": [ + "ssm:GetParameters", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:CreateRole", + "iam:DeleteRole", + "iam:AttachRolePolicy", + "iam:DetachRolePolicy", + "iam:PutRolePolicy", + "iam:DeleteRolePolicy", + "iam:PassRole", + "iam:TagRole", + "iam:TagInstanceProfile", + "iam:ListRolePolicies", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "batch:CreateComputeEnvironment", + "batch:UpdateComputeEnvironment", + "batch:DeleteComputeEnvironment", + "batch:CreateJobQueue", + "batch:UpdateJobQueue", + "batch:DeleteJobQueue", + "batch:DescribeComputeEnvironments", + "batch:DescribeJobQueues", + "fsx:CreateFileSystem", + "fsx:DeleteFileSystem", + "fsx:DescribeFileSystems", + "fsx:TagResource", + "ec2:DescribeSecurityGroups", + "ec2:DescribeAccountAttributes", + "ec2:DescribeSubnets", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:CreateLaunchTemplate", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeKeyPairs", + "ec2:DescribeVpcs", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:GetEbsEncryptionByDefault", + "efs:CreateFileSystem", + "efs:DeleteFileSystem", + "efs:DescribeFileSystems", + "efs:CreateMountTarget", + "efs:DeleteMountTarget", + "efs:DescribeMountTargets", + "efs:ModifyFileSystem", + "efs:PutLifecycleConfiguration", + "efs:TagResource", + "elasticfilesystem:CreateFileSystem", + "elasticfilesystem:DeleteFileSystem", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:CreateMountTarget", + "elasticfilesystem:DeleteMountTarget", + "elasticfilesystem:DescribeMountTargets", + "elasticfilesystem:UpdateFileSystem", + "elasticfilesystem:PutLifecycleConfiguration", + "elasticfilesystem:TagResource", + ], + "Resource": "*", + }, + { + "Sid": "TowerLaunch0", + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*", + "batch:DescribeJobQueues", + "batch:CancelJob", + "batch:SubmitJob", + "batch:ListJobs", + "batch:TagResource", + "batch:DescribeComputeEnvironments", + "batch:TerminateJob", + "batch:DescribeJobs", + "batch:RegisterJobDefinition", + "batch:DescribeJobDefinitions", + "ecs:DescribeTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceAttribute", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeImages", + "logs:Describe*", + "logs:Get*", + "logs:List*", + "logs:StartQuery", + "logs:StopQuery", + "logs:TestMetricFilter", + "logs:FilterLogEvents", + "ses:SendRawEmail", + "secretsmanager:ListSecrets", + ], + "Resource": "*", + }, + ], + } + + +def _create_launch_policy_document() -> Dict[str, Any]: + """Create TowerForge Launch Policy document with limited permissions.""" + return { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "TowerLaunch0", + "Effect": "Allow", + "Action": [ + "batch:DescribeJobQueues", + "batch:CancelJob", + "batch:SubmitJob", + "batch:ListJobs", + "batch:TagResource", + "batch:DescribeComputeEnvironments", + "batch:TerminateJob", + "batch:DescribeJobs", + "batch:RegisterJobDefinition", + "batch:DescribeJobDefinitions", + "ecs:DescribeTasks", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceAttribute", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeImages", + "logs:Describe*", + "logs:Get*", + "logs:List*", + "logs:StartQuery", + "logs:StopQuery", + "logs:TestMetricFilter", + "logs:FilterLogEvents", + "ses:SendRawEmail", + "secretsmanager:ListSecrets", + ], + "Resource": "*", + } + ], + } + + +def _create_s3_policy_document(bucket_arn: str) -> Dict[str, Any]: + """Create S3 bucket access policy document with multipart upload support. + + Includes permissions for: + - Basic bucket operations (list, get location) + - Object operations (get, put, tag, delete) + - Multipart upload operations (for large files >5GB) + + Args: + bucket_arn: ARN of the S3 bucket to grant access to + + Returns: + Dict[str, Any]: S3 policy document with enhanced permissions + """ + return { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads" + ], + "Resource": [bucket_arn], + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:PutObjectTagging", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [f"{bucket_arn}/*"], + "Effect": "Allow", + }, + ], + } + + +def create_seqera_credentials( + seqera_provider: seqera.Provider, + workspace_id: float, + access_key_id: pulumi.Output[str], + access_key_secret: pulumi.Output[str], +) -> seqera.Credential: + """Upload TowerForge AWS credentials to Seqera Platform. + + Args: + seqera_provider: Configured Seqera provider instance + workspace_id: Seqera Platform workspace ID + access_key_id: AWS access key ID from TowerForge IAM user + access_key_secret: AWS secret access key from TowerForge IAM user + + Returns: + seqera.Credential: Seqera credential resource with credentials_id + + Raises: + CredentialError: If credential upload fails + """ + pulumi.log.info("Uploading TowerForge credentials to Seqera Platform") + + # Create AWS credentials configuration for Seqera Platform + aws_keys = seqera.CredentialKeysArgs( + aws=seqera.CredentialKeysAwsArgs( + access_key=access_key_id, + secret_key=access_key_secret, + # Note: assume_role_arn is optional and not needed for direct IAM user credentials + ) + ) + + # Upload credentials to Seqera Platform + seqera_credential = seqera.Credential( + "towerforge-aws-credential", + name=TOWERFORGE_CREDENTIAL_NAME, + description=TOWERFORGE_CREDENTIAL_DESCRIPTION, + provider_type="aws", + workspace_id=workspace_id, + keys=aws_keys, + opts=pulumi.ResourceOptions( + provider=seqera_provider, + # Ensure credentials are uploaded after IAM access key is created + custom_timeouts=pulumi.CustomTimeouts( + create=TIMEOUTS["seqera_credential_create"], + update=TIMEOUTS["seqera_credential_update"], + delete=TIMEOUTS["seqera_credential_delete"], + ), + ), + ) + + return seqera_credential + + +def _create_iam_policies( + aws_provider: aws.Provider, s3_bucket +) -> Tuple[aws.iam.Policy, aws.iam.Policy, aws.iam.Policy]: + """Create IAM policies for TowerForge operations. + + Args: + aws_provider: Configured AWS provider instance + s3_bucket: S3 bucket resource for policy attachment + + Returns: + Tuple of (forge_policy, launch_policy, s3_policy) + """ + # TowerForge Forge Policy - Comprehensive permissions for resource creation + forge_policy = aws.iam.Policy( + "towerforge-forge-policy", + name=TOWERFORGE_POLICY_NAMES["forge"], + description="IAM policy for TowerForge to create and manage AWS Batch resources", + policy=json.dumps(_create_forge_policy_document()), + opts=pulumi.ResourceOptions(provider=aws_provider), + ) + + # TowerForge Launch Policy - Limited permissions for pipeline execution + launch_policy = aws.iam.Policy( + "towerforge-launch-policy", + name=TOWERFORGE_POLICY_NAMES["launch"], + description="IAM policy for TowerForge to launch and monitor pipeline executions", + policy=json.dumps(_create_launch_policy_document()), + opts=pulumi.ResourceOptions(provider=aws_provider), + ) + + # TowerForge S3 Bucket Access Policy - Access to specified S3 bucket + s3_policy = aws.iam.Policy( + "towerforge-s3-policy", + name=TOWERFORGE_POLICY_NAMES["s3"], + description=s3_bucket.bucket.apply( + lambda bucket_name: f"IAM policy for TowerForge to access {bucket_name} S3 bucket" + ), + policy=s3_bucket.arn.apply( + lambda arn: json.dumps(_create_s3_policy_document(arn)) + ), + opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[s3_bucket]), + ) + + return forge_policy, launch_policy, s3_policy + + +def _generate_policy_hash( + forge_policy: aws.iam.Policy, + launch_policy: aws.iam.Policy, + s3_policy: aws.iam.Policy, +) -> str: + """Generate a hash of IAM policies to detect changes. + + Args: + forge_policy: TowerForge Forge policy + launch_policy: TowerForge Launch policy + s3_policy: TowerForge S3 policy + + Returns: + str: SHA256 hash of the combined policy documents + """ + # Create a deterministic hash of all policy documents + forge_doc = _create_forge_policy_document() + launch_doc = _create_launch_policy_document() + + # Combine all policy documents for hashing + combined_policies = json.dumps( + { + "forge": forge_doc, + "launch": launch_doc, + # Note: S3 policy is bucket-specific, so we'll use a placeholder for consistent hashing + # Version updated to include multipart upload permissions + "s3": {"bucket_dependent": True, "version": "v2-multipart"}, + }, + sort_keys=True, + ) + + return hashlib.sha256(combined_policies.encode()).hexdigest() + + +def create_towerforge_credentials( + aws_provider: aws.Provider, + s3_bucket, + seqera_provider: seqera.Provider, + workspace_id: float, +) -> Tuple[ + pulumi.Output[str], pulumi.Output[str], pulumi.Output[str], seqera.Credential, str +]: + """Create TowerForge IAM resources and upload to Seqera Platform. + + Creates IAM policies, user, and access keys for TowerForge operations, + then uploads the credentials to Seqera Platform for use by compute environments. + Based on https://github.com/seqeralabs/nf-tower-aws + + Args: + aws_provider: Configured AWS provider instance + s3_bucket: S3 bucket resource for policy attachment + seqera_provider: Configured Seqera provider instance + workspace_id: Seqera Platform workspace ID + + Returns: + Tuple: (access_key_id, access_key_secret, seqera_credentials_id, seqera_credential_resource, iam_policy_hash) + """ + # Create IAM policies + forge_policy, launch_policy, s3_policy = _create_iam_policies( + aws_provider, s3_bucket + ) + + # Generate policy version hash for compute environment recreation on policy changes + iam_policy_hash = _generate_policy_hash(forge_policy, launch_policy, s3_policy) + + # Create TowerForge IAM User + towerforge_user = aws.iam.User( + "towerforge-user", + name=TOWERFORGE_USER_NAME, + opts=pulumi.ResourceOptions(provider=aws_provider), + ) + + # Attach policies to the TowerForge user + forge_attachment = aws.iam.UserPolicyAttachment( + "towerforge-forge-policy-attachment", + user=towerforge_user.name, + policy_arn=forge_policy.arn, + opts=pulumi.ResourceOptions( + provider=aws_provider, depends_on=[towerforge_user, forge_policy] + ), + ) + + launch_attachment = aws.iam.UserPolicyAttachment( + "towerforge-launch-policy-attachment", + user=towerforge_user.name, + policy_arn=launch_policy.arn, + opts=pulumi.ResourceOptions( + provider=aws_provider, + depends_on=[towerforge_user, launch_policy], + ), + ) + + s3_attachment = aws.iam.UserPolicyAttachment( + "towerforge-s3-policy-attachment", + user=towerforge_user.name, + policy_arn=s3_policy.arn, + opts=pulumi.ResourceOptions( + provider=aws_provider, depends_on=[towerforge_user, s3_policy] + ), + ) + + # Create access keys for the TowerForge user + towerforge_access_key = aws.iam.AccessKey( + "towerforge-access-key", + user=towerforge_user.name, + opts=pulumi.ResourceOptions( + provider=aws_provider, + depends_on=[forge_attachment, launch_attachment, s3_attachment], + additional_secret_outputs=["secret"], + ), + ) + + # Upload the credentials to Seqera Platform + seqera_credential = create_seqera_credentials( + seqera_provider=seqera_provider, + workspace_id=workspace_id, + access_key_id=towerforge_access_key.id, + access_key_secret=towerforge_access_key.secret, + ) + + # Return the access key credentials, Seqera credentials ID, credential resource, and policy hash + return ( + towerforge_access_key.id, + towerforge_access_key.secret, + seqera_credential.credentials_id, + seqera_credential, + iam_policy_hash, + ) + + +def get_towerforge_resources( + aws_provider: aws.Provider, + s3_bucket, + seqera_provider: Optional[seqera.Provider] = None, + workspace_id: Optional[float] = None, +) -> Dict[str, Any]: + """Create TowerForge resources and return resource information for exports. + + This function creates the TowerForge IAM resources and returns a dictionary + containing resource information for Pulumi exports. + + Args: + aws_provider: Configured AWS provider instance + s3_bucket: S3 bucket resource for policy attachment + seqera_provider: Optional Seqera provider for credential upload + workspace_id: Optional workspace ID for Seqera Platform + + Returns: + Dict[str, Any]: Resource information for Pulumi exports + + Raises: + ValueError: If required parameters are missing + """ + # Create the credentials (this will create all the resources) + if seqera_provider and workspace_id: + ( + access_key_id, + access_key_secret, + seqera_credentials_id, + seqera_credential, + iam_policy_hash, + ) = create_towerforge_credentials( + aws_provider, s3_bucket, seqera_provider, workspace_id + ) + else: + # Fallback for backward compatibility - this will raise an error since signature changed + raise ValueError( + "get_towerforge_resources now requires seqera_provider and workspace_id parameters. " + "Please update your code to use the new signature or call create_towerforge_credentials directly." + ) + + return { + "user": { + "name": TOWERFORGE_USER_NAME, + "arn": f"arn:aws:iam::{{aws_account_id}}:user/{TOWERFORGE_USER_NAME}", # Will be populated by Pulumi + }, + "access_key_id": access_key_id, + "access_key_secret": access_key_secret, + "seqera_credentials_id": seqera_credentials_id, + "policies": { + "forge_policy_name": TOWERFORGE_POLICY_NAMES["forge"], + "launch_policy_name": TOWERFORGE_POLICY_NAMES["launch"], + "s3_policy_name": TOWERFORGE_POLICY_NAMES["s3"], + }, + } diff --git a/pulumi/seqera_platform/shared/infrastructure/s3.py b/pulumi/seqera_platform/shared/infrastructure/s3.py new file mode 100644 index 00000000..70afda88 --- /dev/null +++ b/pulumi/seqera_platform/shared/infrastructure/s3.py @@ -0,0 +1,189 @@ +"""S3 infrastructure management for AWS Megatests.""" + +from typing import Dict, Any + +import pulumi +from pulumi_aws import s3 + +from ..utils.constants import S3_BUCKET_NAME + + +def create_s3_infrastructure(aws_provider) -> Dict[str, Any]: + """Create S3 bucket and lifecycle configuration. + + Args: + aws_provider: Configured AWS provider instance + + Returns: + Dict[str, Any]: Dictionary containing bucket and lifecycle configuration + """ + # Import existing AWS resources used by nf-core megatests + # S3 bucket for Nextflow work directory (already exists) + nf_core_awsmegatests_bucket = s3.Bucket( + "nf-core-awsmegatests", + bucket=S3_BUCKET_NAME, + opts=pulumi.ResourceOptions( + import_=S3_BUCKET_NAME, # Import existing bucket + protect=True, # Protect from accidental deletion + provider=aws_provider, # Use configured AWS provider + ignore_changes=[ + "lifecycle_rules", + "versioning", + ], # Don't modify existing configurations - managed manually due to permission constraints + ), + ) + + # S3 bucket lifecycle configuration + # Create lifecycle rules for automated cost optimization and cleanup + bucket_lifecycle_configuration = create_s3_lifecycle_configuration( + aws_provider, nf_core_awsmegatests_bucket + ) + + # S3 bucket CORS configuration for Seqera Data Explorer compatibility + bucket_cors_configuration = create_s3_cors_configuration( + aws_provider, nf_core_awsmegatests_bucket + ) + + return { + "bucket": nf_core_awsmegatests_bucket, + "lifecycle_configuration": bucket_lifecycle_configuration, + "cors_configuration": bucket_cors_configuration, + } + + +def create_s3_lifecycle_configuration(aws_provider, bucket): + """Create S3 lifecycle configuration with proper rules for Nextflow workflows. + + Args: + aws_provider: Configured AWS provider instance + bucket: S3 bucket resource + + Returns: + S3 bucket lifecycle configuration resource + """ + # S3 bucket lifecycle configuration for cost optimization and cleanup + # Rules designed specifically for Nextflow workflow patterns + lifecycle_configuration = s3.BucketLifecycleConfigurationV2( + "nf-core-awsmegatests-lifecycle", + bucket=bucket.id, + rules=[ + # Rule 1: Preserve metadata files with cost optimization + s3.BucketLifecycleConfigurationV2RuleArgs( + id="preserve-metadata-files", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + tag=s3.BucketLifecycleConfigurationV2RuleFilterTagArgs( + key="nextflow.io/metadata", value="true" + ) + ), + transitions=[ + s3.BucketLifecycleConfigurationV2RuleTransitionArgs( + days=30, storage_class="STANDARD_IA" + ), + s3.BucketLifecycleConfigurationV2RuleTransitionArgs( + days=90, storage_class="GLACIER" + ), + ], + ), + # Rule 2: Clean up temporary files after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-temporary-files", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + tag=s3.BucketLifecycleConfigurationV2RuleFilterTagArgs( + key="nextflow.io/temporary", value="true" + ) + ), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 3: Clean up work directory after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-work-directory", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs(prefix="work/"), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 4: Clean up scratch directory after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-scratch-directory", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + prefix="scratch/" + ), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 5: Clean up cache directories after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-cache-directories", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs(prefix="cache/"), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 6: Clean up .cache directories after 30 days + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-dot-cache-directories", + status="Enabled", + filter=s3.BucketLifecycleConfigurationV2RuleFilterArgs( + prefix=".cache/" + ), + expiration=s3.BucketLifecycleConfigurationV2RuleExpirationArgs(days=30), + ), + # Rule 7: Clean up incomplete multipart uploads + s3.BucketLifecycleConfigurationV2RuleArgs( + id="cleanup-incomplete-multipart-uploads", + status="Enabled", + abort_incomplete_multipart_upload=s3.BucketLifecycleConfigurationV2RuleAbortIncompleteMultipartUploadArgs( + days_after_initiation=7 + ), + ), + ], + opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[bucket]), + ) + + return lifecycle_configuration + + +def create_s3_cors_configuration(aws_provider, bucket): + """Create S3 CORS configuration for Seqera Data Explorer compatibility. + + Args: + aws_provider: Configured AWS provider instance + bucket: S3 bucket resource + + Returns: + S3 bucket CORS configuration resource + """ + # S3 CORS configuration for Seqera Data Explorer compatibility + # Based on official Seqera documentation: + # https://docs.seqera.io/platform-cloud/data/data-explorer#amazon-s3-cors-configuration + cors_configuration = s3.BucketCorsConfigurationV2( + "nf-core-awsmegatests-cors", + bucket=bucket.id, + cors_rules=[ + s3.BucketCorsConfigurationV2CorsRuleArgs( + id="SeqeraDataExplorerAccess", + allowed_headers=["*"], + allowed_methods=["GET", "HEAD", "POST", "PUT", "DELETE"], + allowed_origins=[ + "https://*.cloud.seqera.io", + "https://*.tower.nf", + "https://cloud.seqera.io", + "https://tower.nf", + ], + expose_headers=["ETag"], + max_age_seconds=3000, + ), + # Additional rule for direct browser access + s3.BucketCorsConfigurationV2CorsRuleArgs( + id="BrowserDirectAccess", + allowed_headers=["Authorization", "Content-Type", "Range"], + allowed_methods=["GET", "HEAD"], + allowed_origins=["*"], + expose_headers=["Content-Range", "Content-Length", "ETag"], + max_age_seconds=3000, + ), + ], + opts=pulumi.ResourceOptions(provider=aws_provider, depends_on=[bucket]), + ) + + return cors_configuration diff --git a/pulumi/seqera_platform/shared/integrations/__init__.py b/pulumi/seqera_platform/shared/integrations/__init__.py new file mode 100644 index 00000000..ba069edd --- /dev/null +++ b/pulumi/seqera_platform/shared/integrations/__init__.py @@ -0,0 +1,10 @@ +"""Third-party integrations for AWS Megatests.""" + +from .github import create_github_resources +from .github_credentials import create_github_credential, get_github_credential_config + +__all__ = [ + "create_github_resources", + "create_github_credential", + "get_github_credential_config", +] diff --git a/pulumi/seqera_platform/shared/integrations/github.py b/pulumi/seqera_platform/shared/integrations/github.py new file mode 100644 index 00000000..7a43500d --- /dev/null +++ b/pulumi/seqera_platform/shared/integrations/github.py @@ -0,0 +1,169 @@ +"""GitHub integration for AWS Megatests - secrets and variables management.""" + +from typing import Dict, Any, List, Optional, Union + +import pulumi +import pulumi_github as github + +from ..utils.constants import GITHUB_VARIABLE_NAMES, S3_BUCKET_NAME + + +class GitHubIntegrationError(Exception): + """Exception raised when GitHub integration operations fail.""" + + pass + + +def _create_organization_variable( + provider: github.Provider, + resource_name: str, + variable_name: str, + value: Union[str, pulumi.Output[str]], +) -> github.ActionsOrganizationVariable: + """Create a GitHub organization variable with consistent configuration. + + Args: + provider: GitHub provider instance + resource_name: Pulumi resource name + variable_name: GitHub variable name + value: Variable value + + Returns: + github.ActionsOrganizationVariable: Created variable resource + """ + return github.ActionsOrganizationVariable( + resource_name, + visibility="all", + variable_name=variable_name, + value=value, + opts=pulumi.ResourceOptions( + provider=provider, + delete_before_replace=True, # Workaround for GitHub provider issue #250 + ignore_changes=[ + "visibility" + ], # Ignore changes to visibility if variable exists + ), + ) + + +def _create_gh_commands( + workspace_id_val: str, cpu_env_id_val: str, tower_token_val: Optional[str] = None +) -> List[str]: + """Generate manual gh CLI commands for secrets management. + + Args: + workspace_id_val: Workspace ID value + cpu_env_id_val: CPU environment ID value + tower_token_val: Optional tower access token placeholder + + Returns: + List[str]: List of gh CLI commands + """ + commands = [] + + # Legacy workspace ID secret + commands.append( + f'gh secret set TOWER_WORKSPACE_ID --org nf-core --body "{workspace_id_val}" --visibility all' + ) + + # Legacy compute env secret (CPU) + commands.append( + f'gh secret set TOWER_COMPUTE_ENV --org nf-core --body "{cpu_env_id_val}" --visibility all' + ) + + # Tower access token (if provided) + if tower_token_val: + commands.append( + "OP_ACCOUNT=nf-core gh secret set TOWER_ACCESS_TOKEN --org nf-core " + "--body \"$(op read 'op://Dev/Seqera Platform/TOWER_ACCESS_TOKEN')\" --visibility all" + ) + + return commands + + +def create_github_resources( + github_provider: github.Provider, + compute_env_ids: Dict[str, Any], + tower_workspace_id: Union[str, pulumi.Output[str]], + tower_access_token: Optional[str] = None, +) -> Dict[str, Any]: + """Create GitHub organization variables and provide manual secret commands. + + Args: + github_provider: GitHub provider instance + compute_env_ids: Dictionary containing compute environment IDs + tower_workspace_id: Seqera Platform workspace ID + tower_access_token: Tower access token for manual secret commands (optional) + + Returns: + Dict[str, Any]: Dictionary containing created variables and manual commands + + Raises: + GitHubIntegrationError: If GitHub resource creation fails + """ + # Create org-level GitHub variables for compute environment IDs (non-sensitive) + # Using delete_before_replace to work around pulumi/pulumi-github#250 + variables = {} + + # Compute environment variables + for env_type in ["cpu", "gpu", "arm"]: + var_name = GITHUB_VARIABLE_NAMES[env_type] + resource_name = f"tower-compute-env-{env_type}" + + variables[env_type] = _create_organization_variable( + github_provider, + resource_name, + var_name, + compute_env_ids[env_type], + ) + + # Workspace ID variable + variables["workspace_id"] = _create_organization_variable( + github_provider, + "tower-workspace-id", + GITHUB_VARIABLE_NAMES["workspace_id"], + tower_workspace_id, + ) + + # Legacy S3 bucket variable + variables["legacy_s3_bucket"] = _create_organization_variable( + github_provider, + "legacy-aws-s3-bucket", + GITHUB_VARIABLE_NAMES["s3_bucket"], + S3_BUCKET_NAME, + ) + + # GitHub Secrets Management - Manual Commands Only + # NOTE: Due to pulumi/pulumi-github#250, secrets must be managed manually + # https://github.com/nf-core/ops/issues/162 - Legacy compatibility needed + + # Generate manual gh CLI commands for secrets management + if all(isinstance(compute_env_ids[k], str) for k in compute_env_ids) and isinstance( + tower_workspace_id, str + ): + # All static values + gh_cli_commands: Union[List[str], pulumi.Output[List[str]]] = ( + _create_gh_commands( + tower_workspace_id, + compute_env_ids["cpu"], + "" if tower_access_token else None, + ) + ) + else: + # Dynamic values - create commands that will be resolved at runtime + gh_cli_commands = pulumi.Output.all( + workspace_id=tower_workspace_id, cpu_env_id=compute_env_ids["cpu"] + ).apply( + lambda args: _create_gh_commands( + args["workspace_id"], + args["cpu_env_id"], + "" if tower_access_token else None, + ) + ) + + return { + "variables": variables, + "secrets": {}, # No Pulumi-managed secrets due to provider issue + "gh_cli_commands": gh_cli_commands, + "note": "Secrets must be managed manually due to pulumi/pulumi-github#250", + } diff --git a/pulumi/seqera_platform/shared/integrations/github_credentials.py b/pulumi/seqera_platform/shared/integrations/github_credentials.py new file mode 100644 index 00000000..707bc8c7 --- /dev/null +++ b/pulumi/seqera_platform/shared/integrations/github_credentials.py @@ -0,0 +1,96 @@ +"""GitHub credentials integration for Seqera Platform.""" + +import pulumi +import pulumi_seqera as seqera +from typing import Dict, Tuple + + +class GitHubCredentialError(Exception): + """Exception raised when GitHub credential creation fails.""" + + pass + + +def create_github_credential( + seqera_provider: seqera.Provider, + workspace_id: int, + github_token: str, + github_username: str = "nf-core-bot", + credential_name: str = "nf-core-github-finegrained", +) -> Tuple[seqera.Credential, str]: + """Create a GitHub fine-grained credential in Seqera Platform. + + This credential allows Seqera Platform to pull pipeline repositories from GitHub + without hitting GitHub rate limits. The fine-grained token provides secure, + scoped access to nf-core repositories with minimal required permissions. + + Args: + seqera_provider: Configured Seqera provider instance + workspace_id: Seqera workspace ID + github_token: Fine-grained GitHub personal access token for repository access + github_username: GitHub username (default: nf-core-bot) + credential_name: Name for the credential in Seqera + + Returns: + Tuple of (credential_resource, credential_id) + + Raises: + GitHubCredentialError: If credential creation fails + ValueError: If required parameters are missing + """ + # Validate required parameters + if not github_token: + raise ValueError("GitHub token is required") + if not workspace_id: + raise ValueError("Workspace ID is required") + + pulumi.log.info( + f"Creating GitHub credential '{credential_name}' in workspace {workspace_id}" + ) + + try: + # Create GitHub credential using Seqera Terraform provider + github_credential = seqera.Credential( + f"github-credential-{credential_name}", + name=credential_name, + description="Fine-grained GitHub token to avoid rate limits when Platform pulls pipeline repositories", + provider_type="github", + base_url="https://github.com/nf-core/", # Scope to nf-core organization + keys=seqera.CredentialKeysArgs( + github=seqera.CredentialKeysGithubArgs( + username=github_username, + password=github_token, # GitHub tokens go in the password field + ) + ), + workspace_id=workspace_id, + opts=pulumi.ResourceOptions( + provider=seqera_provider, + protect=True, # Protect credential from accidental deletion + ), + ) + + # Return both the resource and the credential ID for reference + return github_credential, github_credential.id + + except Exception as e: + pulumi.log.error(f"Failed to create GitHub credential: {str(e)}") + raise GitHubCredentialError( + f"GitHub credential creation failed: {str(e)}" + ) from e + + +def get_github_credential_config() -> Dict[str, str]: + """Get configuration for GitHub credential creation. + + Returns: + Dict containing configuration values from ESC environment + """ + import os + + return { + "github_finegrained_token": os.environ.get("PLATFORM_GITHUB_ORG_TOKEN", ""), + "github_username": os.environ.get("GITHUB_USERNAME", "nf-core-bot"), + "credential_name": os.environ.get( + "GITHUB_CREDENTIAL_NAME", "nf-core-github-finegrained" + ), + } diff --git a/pulumi/seqera_platform/shared/integrations/workspace_participants_command.py b/pulumi/seqera_platform/shared/integrations/workspace_participants_command.py new file mode 100644 index 00000000..7e99190c --- /dev/null +++ b/pulumi/seqera_platform/shared/integrations/workspace_participants_command.py @@ -0,0 +1,268 @@ +"""Seqera Platform workspace participant management using Pulumi Command provider.""" + +import json +import pulumi +import pulumi_command as command +from typing import Dict, List, Optional +from ..utils.logging import log_info + + +def create_team_data_setup_command( + workspace_id: int, + seqera_token: str, + github_token: str, + opts: Optional[pulumi.ResourceOptions] = None, +) -> command.local.Command: + """ + Create a Pulumi Command that generates team data with proper credentials. + + This runs the team data setup scripts automatically during Pulumi deployment. + """ + setup_cmd = command.local.Command( + "team-data-setup", + create=""" +# Generate team member data with proper credentials +echo "=== Setting up team member data with privacy protection ===" + +# Run setup script with environment credentials +uv run python scripts/setup_team_data.py + +echo "โœ“ Team data setup completed" +echo "Files generated locally (not committed to git):" +echo " - scripts/maintainers_data.json" +echo " - scripts/core_team_data.json" +echo " - scripts/unified_team_data.json" + """, + environment={ + "GITHUB_TOKEN": github_token, + "TOWER_ACCESS_TOKEN": seqera_token, + }, + opts=opts, + ) + + return setup_cmd + + +def create_individual_member_commands( + workspace_id: int, + token: str, + github_token: str, + org_id: int = 252464779077610, # nf-core + opts: Optional[pulumi.ResourceOptions] = None, +) -> tuple[command.local.Command, Dict[str, command.local.Command]]: + """ + Create individual Pulumi Command resources for each GitHub team member. + + This provides granular tracking of each maintainer's workspace participant status. + """ + # First, ensure team data is set up with proper credentials + setup_cmd = create_team_data_setup_command(workspace_id, token, github_token, opts) + + # Generate team data at runtime to avoid committing private emails + log_info("Team data will be generated automatically during deployment...") + + # Load team data (will be available after setup command runs) + try: + with open("scripts/unified_team_data.json", "r") as f: + data = json.load(f) + team_members = data.get("seqera_participants", []) + log_info(f"Loaded {len(team_members)} team members from runtime data") + except FileNotFoundError: + # For preview purposes, show expected team count + log_info("Team data will be generated during deployment (35 expected members)") + team_members = [] + except Exception as e: + log_info(f"Team data will be generated at runtime: {e}") + team_members = [] + + member_commands = {} + + log_info(f"Creating individual tracking for {len(team_members)} team members") + + for member in team_members: + email = member["name"] + github_username = member["github_username"] + role = member["role"] # OWNER for core team, MAINTAIN for maintainers + + # Create safe resource name + safe_name = github_username.replace("-", "_").replace(".", "_") + + # Note: Role precedence handled in bash script (core team checked first) + + # Create individual command for this member + member_cmd = command.local.Command( + f"team_sync_{safe_name}", + create=f''' +#!/bin/bash +# Sync GitHub team member '{github_username}' to Seqera workspace with {role} role +echo "=== Syncing {github_username} ({email}) as {role} ===" + +# Verify user is still in appropriate GitHub teams +echo "Checking GitHub team membership..." +found_in_team=false + +# Check core team first (higher precedence) +if gh api orgs/nf-core/teams/core/members --jq '.[].login' | grep -q "^{github_username}$"; then + echo "โœ“ {github_username} confirmed in nf-core/core team (OWNER role)" + current_role="OWNER" + found_in_team=true +elif gh api orgs/nf-core/teams/maintainers/members --jq '.[].login' | grep -q "^{github_username}$"; then + echo "โœ“ {github_username} confirmed in nf-core/maintainers team (MAINTAIN role)" + current_role="MAINTAIN" + found_in_team=true +fi + +if [ "$found_in_team" = false ]; then + echo "โš ๏ธ {github_username} not found in any relevant team, skipping" + exit 0 +fi + +# Ensure we're using the correct role (core team precedence) +target_role="{role}" +if [ "$current_role" != "$target_role" ]; then + echo "๐Ÿ”„ Role precedence: Using $current_role (detected) instead of $target_role" + target_role="$current_role" +fi + +# Check current email (in case it changed) +echo "Fetching current email..." +current_email=$(gh api /users/{github_username} --jq '.email // empty') + +# Handle members without public emails +if [[ "{email}" == github:* ]]; then + # Member has no public email, try to get current one + if [ -z "$current_email" ]; then + echo "โš ๏ธ {github_username} has no public email - cannot add to Seqera Platform" + echo "STATUS:NO_EMAIL:{github_username}:$target_role" + exit 0 + else + echo "โœ“ {github_username} now has public email: $current_email" + fi +else + # Member had cached email, check if it changed + cached_email="{email}" + if [ -z "$current_email" ]; then + echo "โš ๏ธ {github_username} email no longer public, using cached: $cached_email" + current_email="$cached_email" + elif [ "$current_email" != "$cached_email" ]; then + echo "๐Ÿ”„ {github_username} email changed: $cached_email โ†’ $current_email" + else + echo "โœ“ Current email confirmed: $current_email" + fi +fi + +# Add to Seqera workspace +echo "Adding to Seqera workspace {workspace_id}..." +response=$(curl -s -w "%{{http_code}}" -X PUT \\ + "https://api.cloud.seqera.io/orgs/{org_id}/workspaces/{workspace_id}/participants/add" \\ + -H "Authorization: Bearer {token}" \\ + -H "Content-Type: application/json" \\ + -d '{{"userNameOrEmail": "'$current_email'"}}') + +http_code="${{response: -3}}" +response_body="${{response%???}}" + +case $http_code in + 200|201|204) + echo "โœ“ Successfully added {github_username} with $target_role role" + echo "STATUS:ADDED:$current_email:$target_role" + ;; + 409) + echo "~ {github_username} already exists in workspace" + echo "STATUS:EXISTS:$current_email:$target_role" + ;; + 404) + echo "โœ— User not found in Seqera Platform: $current_email" + echo "STATUS:USER_NOT_FOUND:$current_email:N/A" + ;; + *) + echo "โœ— Failed to add {github_username}: HTTP $http_code" + echo "Response: $response_body" + echo "STATUS:FAILED:$current_email:ERROR" + exit 1 + ;; +esac + +echo "Completed sync for {github_username}" + ''', + environment={ + "GITHUB_TOKEN": github_token, + }, + opts=pulumi.ResourceOptions( + depends_on=[setup_cmd], + parent=opts.parent if opts else None, + ), + ) + + member_commands[github_username] = member_cmd + + return setup_cmd, member_commands + + +def create_workspace_participants_via_command( + workspace_id: int, + token: str, + participants_data: List[Dict[str, str]], + opts: Optional[pulumi.ResourceOptions] = None, +) -> command.local.Command: + """ + Create workspace participants using Pulumi Command provider to run the Python script. + + Args: + workspace_id: Seqera workspace ID + token: Seqera API access token + participants_data: List of participant dictionaries + opts: Pulumi resource options + + Returns: + Command resource that manages workspace participants + """ + # Create the command that will run our Python script + participant_count = len(participants_data) + log_info(f"Creating command to add {participant_count} workspace participants") + + # The command runs within the Pulumi execution context + add_participants_cmd = command.local.Command( + "add-workspace-participants", + create="uv run python scripts/add_maintainers_to_workspace.py --yes", + environment={ + "TOWER_ACCESS_TOKEN": token, + "TOWER_WORKSPACE_ID": str(workspace_id), + }, + opts=pulumi.ResourceOptions(**opts.__dict__ if opts else {}), + ) + + return add_participants_cmd + + +def create_workspace_participants_verification( + workspace_id: int, + token: str, + depends_on: List[pulumi.Resource], + opts: Optional[pulumi.ResourceOptions] = None, +) -> command.local.Command: + """ + Create a verification command that checks workspace participants were added correctly. + + Args: + workspace_id: Seqera workspace ID + token: Seqera API access token + depends_on: Resources this command depends on + opts: Pulumi resource options + + Returns: + Command resource that verifies workspace participants + """ + verification_cmd = command.local.Command( + "verify-workspace-participants", + create="uv run python scripts/inspect_participants.py", + environment={ + "TOWER_ACCESS_TOKEN": token, + "TOWER_WORKSPACE_ID": str(workspace_id), + }, + opts=pulumi.ResourceOptions( + depends_on=depends_on, **(opts.__dict__ if opts else {}) + ), + ) + + return verification_cmd diff --git a/pulumi/seqera_platform/shared/integrations/workspace_participants_simple.py b/pulumi/seqera_platform/shared/integrations/workspace_participants_simple.py new file mode 100644 index 00000000..5cd50d1a --- /dev/null +++ b/pulumi/seqera_platform/shared/integrations/workspace_participants_simple.py @@ -0,0 +1,124 @@ +"""Simple workspace participant management using Pulumi's apply() pattern.""" + +import json +import pulumi +import requests +from typing import Dict, List, Any +from ..utils.logging import log_info + + +def add_workspace_participant_simple( + email: str, + role: str, + workspace_id: pulumi.Output[str], + token: pulumi.Output[str], + org_id: int = 252464779077610, +) -> pulumi.Output[Dict[str, Any]]: + """ + Add a workspace participant using Pulumi's apply() pattern. + + This is simpler than dynamic resources but still integrates with Pulumi. + """ + + def _add_participant(args): + """Internal function that does the actual API call.""" + workspace_id_val, token_val = args + + headers = { + "Authorization": f"Bearer {token_val}", + "Content-Type": "application/json", + } + + url = f"https://api.cloud.seqera.io/orgs/{org_id}/workspaces/{workspace_id_val}/participants/add" + payload = {"userNameOrEmail": email} + + try: + response = requests.put(url, headers=headers, json=payload, timeout=30) + + if response.status_code in [200, 201, 204]: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "added", + "participant_id": f"{org_id}:{workspace_id_val}:{email}", + } + elif response.status_code == 409: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "already_exists", + "participant_id": f"{org_id}:{workspace_id_val}:{email}", + } + else: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "failed", + "error": f"HTTP {response.status_code}", + } + + except Exception as e: + return { + "email": email, + "role": role, + "workspace_id": workspace_id_val, + "status": "error", + "error": str(e), + } + + # Use Pulumi's apply to handle the async nature of Outputs + return pulumi.Output.all(workspace_id, token).apply(_add_participant) + + +def create_workspace_participants_simple( + workspace_id: pulumi.Output[str], + token: pulumi.Output[str], + maintainer_emails: List[str], + role: str = "MAINTAIN", +) -> pulumi.Output[List[Dict[str, Any]]]: + """ + Create multiple workspace participants using the simple approach. + + Args: + workspace_id: Seqera workspace ID as Pulumi Output + token: Seqera API token as Pulumi Output + maintainer_emails: List of email addresses to add + role: Role to assign (default: MAINTAIN) + + Returns: + Pulumi Output containing list of participant creation results + """ + + participant_outputs = [] + + for email in maintainer_emails: + participant_result = add_workspace_participant_simple( + email, role, workspace_id, token + ) + participant_outputs.append(participant_result) + + # Combine all outputs into a single list + return pulumi.Output.all(*participant_outputs) + + +def load_maintainer_emails_static() -> List[str]: + """Load maintainer emails from the JSON file (static version for Pulumi).""" + try: + with open("scripts/maintainers_data.json", "r") as f: + data = json.load(f) + + participants = data.get("seqera_participants", []) + emails = [p["name"] for p in participants] + + log_info(f"Loaded {len(emails)} maintainer emails for workspace participants") + return emails + + except FileNotFoundError: + log_info("Maintainers data file not found, skipping workspace participants") + return [] + except Exception as e: + log_info(f"Error loading maintainers data: {e}") + return [] diff --git a/pulumi/seqera_platform/shared/providers/__init__.py b/pulumi/seqera_platform/shared/providers/__init__.py new file mode 100644 index 00000000..39eb2523 --- /dev/null +++ b/pulumi/seqera_platform/shared/providers/__init__.py @@ -0,0 +1,7 @@ +"""Provider configurations for AWS Megatests infrastructure.""" + +from .aws import create_aws_provider +from .github import create_github_provider +from .seqera import create_seqera_provider + +__all__ = ["create_aws_provider", "create_github_provider", "create_seqera_provider"] diff --git a/pulumi/seqera_platform/shared/providers/aws.py b/pulumi/seqera_platform/shared/providers/aws.py new file mode 100644 index 00000000..9c0febdb --- /dev/null +++ b/pulumi/seqera_platform/shared/providers/aws.py @@ -0,0 +1,19 @@ +"""AWS provider configuration for AWS Megatests infrastructure.""" + +import pulumi_aws as aws +from ..utils.constants import AWS_REGION + + +def create_aws_provider() -> aws.Provider: + """Create AWS provider using ESC OIDC authentication. + + The ESC environment should automatically provide AWS credentials + when the environment is imported in Pulumi.prod.yaml. + + Returns: + aws.Provider: Configured AWS provider instance + """ + return aws.Provider( + "aws-provider", + region=AWS_REGION, + ) diff --git a/pulumi/seqera_platform/shared/providers/github.py b/pulumi/seqera_platform/shared/providers/github.py new file mode 100644 index 00000000..03ac9db9 --- /dev/null +++ b/pulumi/seqera_platform/shared/providers/github.py @@ -0,0 +1,16 @@ +"""GitHub provider configuration for AWS Megatests infrastructure.""" + +import pulumi_github as github +from ..utils.constants import GITHUB_ORG + + +def create_github_provider(github_token: str) -> github.Provider: + """Create GitHub provider with token authentication. + + Args: + github_token: GitHub personal access token with org admin permissions + + Returns: + github.Provider: Configured GitHub provider instance + """ + return github.Provider("github-provider", token=github_token, owner=GITHUB_ORG) diff --git a/pulumi/seqera_platform/shared/providers/seqera.py b/pulumi/seqera_platform/shared/providers/seqera.py new file mode 100644 index 00000000..465f728c --- /dev/null +++ b/pulumi/seqera_platform/shared/providers/seqera.py @@ -0,0 +1,38 @@ +"""Seqera provider configuration for AWS Megatests infrastructure.""" + +import pulumi +import pulumi_seqera as seqera +from typing import Dict, Any +from ..utils.constants import SEQERA_API_URL, ERROR_MESSAGES + + +class SeqeraProviderError(Exception): + """Exception raised when Seqera provider initialization fails.""" + + pass + + +def create_seqera_provider(config: Dict[str, Any]) -> seqera.Provider: + """Create and configure the Seqera provider with error handling. + + Args: + config: Configuration dictionary containing tower_access_token + + Returns: + seqera.Provider: Configured Seqera provider instance + + Raises: + SeqeraProviderError: If provider creation fails + ValueError: If required configuration is missing + """ + # Validate required configuration + if not config.get("tower_access_token"): + raise ValueError(ERROR_MESSAGES["missing_tower_token"]) + + pulumi.log.info("Creating Seqera provider with Cloud API endpoint") + + return seqera.Provider( + "seqera-provider", + bearer_auth=config["tower_access_token"], + server_url=SEQERA_API_URL, + ) diff --git a/pulumi/seqera_platform/shared/utils/__init__.py b/pulumi/seqera_platform/shared/utils/__init__.py new file mode 100644 index 00000000..5687405f --- /dev/null +++ b/pulumi/seqera_platform/shared/utils/__init__.py @@ -0,0 +1,31 @@ +"""Utility functions and constants for AWS Megatests.""" + +from .constants import ( + AWS_REGION, + S3_BUCKET_NAME, + SEQERA_API_URL, + COMPUTE_ENV_NAMES, + TOWERFORGE_POLICY_NAMES, +) +from .logging import ( + log_info, + log_error, + log_warning, + log_step, + log_resource_creation, + log_resource_success, +) + +__all__ = [ + "AWS_REGION", + "S3_BUCKET_NAME", + "SEQERA_API_URL", + "COMPUTE_ENV_NAMES", + "TOWERFORGE_POLICY_NAMES", + "log_info", + "log_error", + "log_warning", + "log_step", + "log_resource_creation", + "log_resource_success", +] diff --git a/pulumi/seqera_platform/shared/utils/constants.py b/pulumi/seqera_platform/shared/utils/constants.py new file mode 100644 index 00000000..baf1eefa --- /dev/null +++ b/pulumi/seqera_platform/shared/utils/constants.py @@ -0,0 +1,145 @@ +"""Constants and configuration values for AWS Megatests infrastructure.""" + +# AWS Configuration +AWS_REGION = "eu-west-1" +S3_BUCKET_NAME = "nf-core-awsmegatests" +S3_WORK_DIR = f"s3://{S3_BUCKET_NAME}" + +# Seqera Configuration +SEQERA_API_URL = "https://api.cloud.seqera.io" + +# Compute Environment Names +COMPUTE_ENV_NAMES = { + "cpu": "aws_ireland_fusionv2_nvme_cpu_snapshots", + "gpu": "aws_ireland_fusionv2_nvme_gpu_snapshots", + "arm": "aws_ireland_fusionv2_nvme_cpu_ARM_snapshots", +} + +# Compute Environment Descriptions +COMPUTE_ENV_DESCRIPTIONS = { + "cpu": "CPU compute environment with Fusion v2 and NVMe storage", + "gpu": "GPU compute environment with Fusion v2 and NVMe storage", + "arm": "ARM CPU compute environment with Fusion v2 and NVMe storage", +} + +# Configuration File Paths +CONFIG_FILES = { + "cpu": "seqerakit/current-env-cpu.json", + "gpu": "seqerakit/current-env-gpu.json", + "arm": "seqerakit/current-env-cpu-arm.json", +} + +# Nextflow configuration files for compute environments +NEXTFLOW_CONFIG_FILES = { + "cpu": "seqerakit/configs/nextflow-cpu.config", + "gpu": "seqerakit/configs/nextflow-gpu.config", + "arm": "seqerakit/configs/nextflow-arm.config", +} + +# TowerForge Configuration +TOWERFORGE_USER_NAME = "TowerForge-AWSMegatests" +TOWERFORGE_POLICY_NAMES = { + "forge": "TowerForge-Forge-Policy", + "launch": "TowerForge-Launch-Policy", + "s3": "TowerForge-S3-Policy", +} + +TOWERFORGE_CREDENTIAL_NAME = "TowerForge-AWSMegatests-Dynamic" +TOWERFORGE_CREDENTIAL_DESCRIPTION = ( + "Dynamically created TowerForge credentials for AWS Megatests compute environments" +) + +# GitHub Configuration +GITHUB_ORG = "nf-core" +GITHUB_VARIABLE_NAMES = { + "cpu": "TOWER_COMPUTE_ENV_CPU", + "gpu": "TOWER_COMPUTE_ENV_GPU", + "arm": "TOWER_COMPUTE_ENV_ARM", + "workspace_id": "TOWER_WORKSPACE_ID", + "s3_bucket": "AWS_S3_BUCKET", +} + +# Timeout Configuration (in minutes) +TIMEOUTS = { + "seqera_credential_create": "5m", + "seqera_credential_update": "5m", + "seqera_credential_delete": "2m", + "compute_env_create": "10m", + "compute_env_update": "10m", + "compute_env_delete": "5m", +} + +# Default Compute Environment Settings +DEFAULT_COMPUTE_ENV_CONFIG = { + "region": AWS_REGION, + "workDir": S3_WORK_DIR, + "waveEnabled": True, + "fusion2Enabled": True, + "nvnmeStorageEnabled": True, + "fusionSnapshots": True, + "nextflowConfig": "", +} + +DEFAULT_FORGE_CONFIG = { + "type": "SPOT", + "minCpus": 0, + "maxCpus": 1000, + "gpuEnabled": False, + "instanceTypes": [], + "subnets": [], + "securityGroups": [], + "disposeOnDeletion": True, + "allowBuckets": [], + "efsCreate": False, + "ebsBootSize": 50, + "fargateHeadEnabled": True, + "arm64Enabled": False, +} + +# Error Messages +ERROR_MESSAGES = { + "missing_tower_token": ( + "TOWER_ACCESS_TOKEN is required for Seqera provider. " + "Please ensure it's set in your ESC environment with proper permissions: " + "WORKSPACE_ADMIN or COMPUTE_ENV_ADMIN scope." + ), + "seqera_provider_init_failed": ( + "Seqera provider initialization failed. " + "This usually indicates token permissions issues. " + "Ensure your TOWER_ACCESS_TOKEN has WORKSPACE_ADMIN or COMPUTE_ENV_ADMIN permissions." + ), + "config_file_not_found": "Configuration file not found: {}", + "invalid_json": "Invalid JSON in configuration file {}: {}", + "config_load_failed": "Failed to load configuration file {}: {}", + "invalid_workspace_id": "Invalid or missing workspace ID: {}", + "missing_compute_env_params": "Missing required parameters for compute environment {}", + "missing_config_args": "Configuration arguments are required for compute environment {}", + "compute_env_create_failed": ( + "Failed to create compute environment '{}'. " + "Common causes: " + "1. Seqera API token lacks required permissions (403 Forbidden) " + "2. Invalid credentials_id reference " + "3. Workspace access restrictions " + "4. Network connectivity issues" + ), + "credential_upload_failed": ( + "Failed to upload credentials to Seqera Platform. " + "Common causes: " + "1. Seqera provider not properly configured " + "2. Invalid workspace ID " + "3. Network connectivity issues to api.cloud.seqera.io " + "4. Invalid AWS credentials format" + ), +} + +# Required Environment Variables +REQUIRED_ENV_VARS = [ + "TOWER_ACCESS_TOKEN", + "TOWER_WORKSPACE_ID", + "GITHUB_TOKEN", +] + +# Optional Environment Variables with Defaults +DEFAULT_ENV_VARS = { + "TOWER_WORKSPACE_ID": "59994744926013", # Fallback workspace ID +} diff --git a/pulumi/seqera_platform/shared/utils/logging.py b/pulumi/seqera_platform/shared/utils/logging.py new file mode 100644 index 00000000..2cb342bd --- /dev/null +++ b/pulumi/seqera_platform/shared/utils/logging.py @@ -0,0 +1,68 @@ +"""Logging utilities for AWS Megatests infrastructure.""" + +import pulumi +from typing import Optional + + +def log_info(message: str, context: Optional[str] = None) -> None: + """Log an informational message with optional context. + + Args: + message: The message to log + context: Optional context prefix + """ + formatted_message = f"[{context}] {message}" if context else message + pulumi.log.info(formatted_message) + + +def log_error(message: str, context: Optional[str] = None) -> None: + """Log an error message with optional context. + + Args: + message: The error message to log + context: Optional context prefix + """ + formatted_message = f"[{context}] {message}" if context else message + pulumi.log.error(formatted_message) + + +def log_warning(message: str, context: Optional[str] = None) -> None: + """Log a warning message with optional context. + + Args: + message: The warning message to log + context: Optional context prefix + """ + formatted_message = f"[{context}] {message}" if context else message + pulumi.log.warn(formatted_message) + + +def log_step(step_number: int, step_name: str, description: str) -> None: + """Log a deployment step with consistent formatting. + + Args: + step_number: The step number + step_name: Short name for the step + description: Detailed description of what the step does + """ + log_info(f"Step {step_number}: {step_name} - {description}") + + +def log_resource_creation(resource_type: str, resource_name: str) -> None: + """Log resource creation with consistent formatting. + + Args: + resource_type: Type of resource being created + resource_name: Name of the resource + """ + log_info(f"Creating {resource_type}: {resource_name}", "Resource") + + +def log_resource_success(resource_type: str, resource_name: str) -> None: + """Log successful resource creation. + + Args: + resource_type: Type of resource that was created + resource_name: Name of the resource + """ + log_info(f"Successfully created {resource_type}: {resource_name}", "Resource")