feat: pythonfunction (#11)

* StarPrincipal

* QUILT_ROLE_ARN

* refactor projenrc

* test events

* aws-lambda-python-alpha

* stub packager

* create PythonFunction

* pytest-watcher

* test_event["outputUri"] in result["body"]

* poetry remove flake8

* GATKReport

* QUILT_METADATA vs FASTQ_SENTINEL

* Constants

* drop gsalib dependency

made Lambda crash - not in requirements.txt?!?

* PYTHON_INDEX

* pull config files

instance-scheduler

* fix docker config

* drop quiltcore for quilt3

pyarrow too large - layer?

* test python types

* handler takes LambdaContext

* disable pytest

Hard to run Python/Poetry in GitHub Action
https://github.com/abatilo/actions-poetry

* handle missing client_context.env

.pre-commit-config.yaml

@@ -17,3 +17,8 @@ repos:
         entry: bash -c "npm run eslint"
         language: node
         stages: [pre-push]
+      - id: python-lint
+        name: python-lint
+        entry: bash -c "cd src/packager && make lint"
+        language: system
+        stages: [pre-commit]

.projenrc.ts

@@ -1,13 +1,17 @@
 import { awscdk } from 'projen';
+
+const cdkVersion = '2.114.1';
+const solutionName = 'omics-quilt-demo';
 const project = new awscdk.AwsCdkTypeScriptApp({
-  cdkVersion: '2.114.1',
+  cdkVersion: cdkVersion,
   majorVersion: 1,
   defaultReleaseBranch: 'main',
   description: 'Use CDK to create Quilt packages from AWS HealthOmics',
-  name: 'omics-quilt-demo',
+  name: solutionName,
   projenrcTs: true,
   deps: [
     'aws-lambda',
+    `@aws-cdk/aws-lambda-python-alpha@^${cdkVersion}-alpha.0`,
     '@aws-sdk/client-s3',
     '@aws-sdk/client-sns',
     '@aws-sdk/client-omics',
@@ -25,25 +29,41 @@ const project = new awscdk.AwsCdkTypeScriptApp({
     '.env*',
     '.DS_Store',
     'test/__snapshots__/*',
+    '__pycache__', // Python
+    '*.pyc', // Python
   ],
 });
-project.tryFindObjectFile('.github/workflows/build.yml')!.addOverride('jobs.build.env', {
-  CI: 'true',
-  AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}',
-  AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}',
-  AWS_ACCOUNT_ID: '${{ secrets.AWS_ACCOUNT_ID }}',
-  AWS_DEFAULT_REGION: '${{ secrets.AWS_DEFAULT_REGION }}',
-  CDK_APP_NAME: '${{ secrets.CDK_APP_NAME }}',
-  CDK_DEFAULT_ACCOUNT: '${{ secrets.AWS_ACCOUNT_ID }}',
-  CDK_DEFAULT_REGION: '${{ secrets.AWS_DEFAULT_REGION }}',
-  CDK_DEFAULT_EMAIL: '${{ secrets.CDK_DEFAULT_EMAIL }}',
-  QUILT_CATALOG_DOMAIN: '${{ secrets.QUILT_CATALOG_DOMAIN }}',
+override_file_key('.github/workflows/build.yml', 'jobs.build.env');
+fix_deprecation_warning();
+/*
+const appTestTask = project.addTask('pytest', {
+  cwd: 'src/packager',
+  exec: 'make test',
 });
-// Fix Jest 29 warning about deprecated config in `globals`
-project.jest!.config.transform ??= {};
-project.jest!.config.transform['\\.ts$'] = [
-  'ts-jest',
-  project.jest?.config.globals['ts-jest'],
-];
-delete project.jest!.config.globals['ts-jest'];
+const testTask = project.tasks.tryFind('test');
+testTask?.spawn(appTestTask);
+*/
 project.synth();
+
+
+function override_file_key(file: string, key: string) {
+  const KEYS = 'AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_ACCOUNT_ID AWS_DEFAULT_REGION CDK_APP_NAME CDK_DEFAULT_EMAIL QUILT_CATALOG_DOMAIN'.split(' ');
+  var opts: {[key: string]: string} = { CI: 'true' };
+  for (const k of KEYS) {
+    opts[k] = `\${{ secrets.${k} }}`;
+  }
+  opts.CDK_DEFAULT_ACCOUNT = opts.AWS_ACCOUNT_ID;
+  opts.CDK_DEFAULT_REGION = opts.AWS_DEFAULT_REGION;
+
+  project.tryFindObjectFile(file)?.addOverride(key, opts);
+}
+
+// Fix Jest 29 warning about deprecated config in `globals`
+function fix_deprecation_warning() {
+  project.jest!.config.transform ??= {};
+  project.jest!.config.transform['\\.ts$'] = [
+    'ts-jest',
+    project.jest?.config.globals['ts-jest'],
+  ];
+  delete project.jest!.config.globals['ts-jest'];
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.analysis.typeCheckingMode": "basic"
+}

src/constants.ts

@@ -18,6 +18,8 @@ export class Constants {
     READY2RUN_WORKFLOW_ID: '9500764',
     MANIFEST_ROOT: 'fastq',
     MANIFEST_SUFFIX: '.json',
+    QUILT_METADATA: 'quilt_metadata.json',
+    FASTQ_SENTINEL: 'out/bqsr_report/NA12878.hg38.recal_data.csv',
   };
 
   public static GET(key: string): any {

src/omics-quilt.fastq.ts

@@ -62,10 +62,14 @@ export async function handler(event: any, context: any) {
   return { message: 'Success' };
 }
 
-async function save_input(prefix: string, item: any, cc: Constants) {
-  const input_uri = cc.get('INPUT_S3_LOCATION') + '/' + prefix + '.json';
-  console.info(`Writing input to ${input_uri}`);
-  await Constants.SaveObjectURI(input_uri, item);
+async function save_metadata(item: any, cc: Constants) {
+  const sentinel_file = cc.get('QUILT_METADATA)');
+  if (!sentinel_file) {
+    console.info('No QUILT_METADATA, skipping metadata save');
+    return;
+  }
+  console.info(`Writing input to ${sentinel_file}`);
+  await Constants.SaveObjectURI(sentinel_file, item);
 }
 
 async function run_workflow(
@@ -74,7 +78,6 @@ async function run_workflow(
   cc: Constants,
 ) {
   const _samplename = item.sample_name;
-  await save_input(_samplename, item, cc);
   console.info(`Starting workflow for sample: ${_samplename}`);
   const uuid = cc.get('TEST_UUID') || uuidv4();
   const run_name = `${_samplename}.${uuid}.`;
@@ -95,7 +98,6 @@ async function run_workflow(
     },
     requestId: uuid,
   };
-  await save_input(run_name + 'input', options, cc);
   try {
     console.debug(`Workflow options: ${JSON.stringify(options)}`);
     if (cc.get('debug') === true) {
@@ -104,7 +106,12 @@ async function run_workflow(
       const input: StartRunCommandInput = options;
       const response = await start_omics_run(input);
       console.info(`Workflow response: ${JSON.stringify(response)}`);
-      await save_input(run_name + 'output', response, cc);
+      const run_metadata = {
+        sample: item,
+        run: response,
+        workflow: options,
+      };
+      await save_metadata(run_metadata, cc);
     }
   } catch (e: any) {
     console.error('Error : ' + e.toString());

src/omics-quilt.ts

@@ -1,8 +1,10 @@
+import * as python from '@aws-cdk/aws-lambda-python-alpha';
 import { Duration, RemovalPolicy, Stack, type StackProps } from 'aws-cdk-lib';
 import { Rule } from 'aws-cdk-lib/aws-events';
 import { SnsTopic } from 'aws-cdk-lib/aws-events-targets';
 import {
   AccountPrincipal,
+  ArnPrincipal,
   ManagedPolicy,
   PolicyStatement,
   Role,
@@ -23,12 +25,16 @@ import { StringParameter } from 'aws-cdk-lib/aws-ssm';
 import { type Construct } from 'constructs';
 import { Constants } from './constants';
 
+const PYTHON_FOLDER = `${__dirname}/packager`;
+const PYTHON_INDEX = 'packager/index.py';
+
 export class OmicsQuiltStack extends Stack {
   public readonly inputBucket: Bucket;
   public readonly outputBucket: Bucket;
 
   public readonly manifest_prefix: string;
   public readonly manifest_suffix: string;
+  public readonly packager_sentinel: string;
 
   readonly cc: Constants;
   readonly lambdaRole: Role;
@@ -42,10 +48,12 @@ export class OmicsQuiltStack extends Stack {
     const manifest_root = this.cc.get('MANIFEST_ROOT');
     this.manifest_prefix = `${manifest_root}/${this.cc.region}`;
     this.manifest_suffix = this.cc.get('MANIFEST_SUFFIX');
+    this.packager_sentinel = this.cc.get('FASTQ_SENTINEL');
 
     // Create Input/Output S3 buckets
     this.inputBucket = this.makeBucket('input');
     this.outputBucket = this.makeBucket('output');
+
     this.makeParameter('INPUT_BUCKET_NAME', this.inputBucket.bucketName);
     this.makeParameter('OUTPUT_BUCKET_NAME', this.outputBucket.bucketName);
     // SNS Topic for Workflow notifications
@@ -68,6 +76,16 @@ export class OmicsQuiltStack extends Stack {
       ],
     });
     fastqLambda.addEventSource(fastqTrigger);
+
+    const packagerLambda = this.makePythonLambda('packager', {});
+    // TODO: trigger on Omics completion event, not report file
+    const packagerTrigger = new S3EventSource(this.outputBucket, {
+      events: [EventType.OBJECT_CREATED],
+      filters: [
+        { suffix: this.packager_sentinel },
+      ],
+    });
+    packagerLambda.addEventSource(packagerTrigger);
   }
 
   private makeParameter(name: string, value: any) {
@@ -118,7 +136,7 @@ export class OmicsQuiltStack extends Stack {
           source: ['aws.omics'],
           detailType: ['Run Status Change'],
           detail: {
-            status: ['FAILED', 'COMPLETED', 'CREATED'],
+            status: ['*'],
           },
         },
       },
@@ -156,31 +174,56 @@ export class OmicsQuiltStack extends Stack {
     const bucket = new Bucket(this, name, bucketOptions);
     bucket.grantDelete(this.principal);
     bucket.grantReadWrite(this.principal);
+    const quilt_arn = this.cc.get('QUILT_ROLE_ARN');
+    if (quilt_arn) {
+      const quilt_principal = new ArnPrincipal(quilt_arn);
+      bucket.grantReadWrite(quilt_principal);
+    }
     return bucket;
   }
 
   private makeLambda(name: string, env: object) {
-    const output = ['s3:/', this.outputBucket.bucketName, this.cc.app];
-    const input = ['s3:/', this.inputBucket.bucketName, this.manifest_prefix];
-    const default_env = {
-      OMICS_ROLE: this.omicsRole.roleArn,
-      OUTPUT_S3_LOCATION: output.join('/'),
-      INPUT_S3_LOCATION: input.join('/'),
-      WORKFLOW_ID: this.cc.get('READY2RUN_WORKFLOW_ID'),
-      ECR_REGISTRY: this.cc.getEcrRegistry(),
-      LOG_LEVEL: 'ALL',
-    };
-    // create merged env
-    const final_env = Object.assign(default_env, env);
     return new NodejsFunction(this, name, {
       runtime: Runtime.NODEJS_18_X,
       role: this.lambdaRole,
       timeout: Duration.seconds(60),
       retryAttempts: 1,
-      environment: final_env,
+      environment: this.makeLambdaEnv(env),
     });
   }
 
+  private makePythonLambda(name: string, env: object) {
+    return new python.PythonFunction(this, name, {
+      entry: PYTHON_FOLDER,
+      index: PYTHON_INDEX,
+      runtime: Runtime.PYTHON_3_11,
+      role: this.lambdaRole,
+      timeout: Duration.seconds(60),
+      retryAttempts: 1,
+      environment: this.makeLambdaEnv(env),
+      bundling: {
+        assetExcludes: ['.mypy_cache', '.pytest_cache', '.tox', '__pycache__'],
+      },
+    });
+  }
+
+  private makeLambdaEnv(env: object) {
+    const output = ['s3:/', this.outputBucket.bucketName, this.cc.app];
+    const input = ['s3:/', this.inputBucket.bucketName, this.manifest_prefix];
+    const final_env = {
+      ECR_REGISTRY: this.cc.getEcrRegistry(),
+      INPUT_S3_LOCATION: input.join('/'),
+      LOG_LEVEL: 'ALL',
+      OMICS_ROLE: this.omicsRole.roleArn,
+      OUTPUT_S3_LOCATION: output.join('/'),
+      SENTINEL_FILE: this.packager_sentinel,
+      QUILT_METADATA: this.cc.get('QUILT_METADATA'),
+      WORKFLOW_ID: this.cc.get('READY2RUN_WORKFLOW_ID'),
+      ...env,
+    };
+    return final_env;
+  }
+
   private makeLambdaRole() {
     const lambdaRole = new Role(this, `${this.cc.app}-lambda-role`, {
       assumedBy: new ServicePrincipal('lambda.amazonaws.com'),

src/packager/.coveragerc

@@ -0,0 +1,7 @@
+[run]
+branch = true
+source = instance_scheduler
+
+[report]
+exclude_lines =
+    if TYPE_CHECKING:

src/packager/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.analysis.typeCheckingMode": "basic"
+}

src/packager/Makefile

@@ -0,0 +1,22 @@
+PROJECT=packager
+sinclude ../Makefile.include
+
+.PHONY: all clean lint install test watch
+
+all: install test
+
+clean:
+	rm -rf .pytest_cache
+
+install:
+	poetry install
+
+test: lint
+	poetry run pytest
+
+watch:
+	poetry run ptw --now .
+
+lint:
+	poetry run mypy packager
+	poetry run black .

src/packager/README.md

@@ -0,0 +1,3 @@
+# Packager
+
+Python Lambda to create a Quilt package from an S3 URI