feat: input manifest error (#20)

* fix timeout

* updates

* document DEMO

* makeDirectory if IsLocal

* don't save metadata (test each time)

.projenrc.ts

@@ -31,6 +31,7 @@ const project = new awscdk.AwsCdkTypeScriptApp({
     'test/__snapshots__/*',
     '__pycache__', // Python
     '*.pyc', // Python
+    '*_metadata.json', // Quilt
   ],
 });
 override_file_key('.github/workflows/build.yml', 'jobs.build.env');

README.md

@@ -4,14 +4,51 @@ Use CDK to create Quilt packages from AWS HealthOmics
 
 ## Usage
 
+### Installation
+
+Use CDK to create and deploy the stack.
+
 ```bash
 cp example.env .env # and edit
-aws configure list-profiles # ensure it is setup properly
+aws configure list-profiles # verify AWS credentials exist
+npx npm install # if npm not present (but npx is)
+npm install yarn -g # if yarn not present (but npm is)
 yarn install
 npx cdk bootstrap # if not yet done for this account/region
+# start Docker if not already running
+sudo systemctl start docker # e.g. on Linux (requires large Cloud9 instance!)
 npm run deploy
 ```
 
+You will also need to accept the Subscription from your email client.
+
+### Quilt Integration
+
+Use your Quilt Catalog to browse the inputs and outputs
+
+1. Go to Console and find the Omics-Quilt stack
+2. Copy names of the INPUT and OUTPUT buckets
+3. Copy the RegistryRoleARN
+4. Go your Quilt Catalog
+5. Click "+" on the front page (or Admin Settings -> Buckets)
+6. Click "+" in the upper right corner to add a new bucket
+7. For each bucket, enter the name and ARN
+
+### Run the Workflow
+
+1. Find or create your region's input parameters file: `./workflows/fastq/<region>.json`
+2. Go to Console and find the input bucket
+3. Create Folders `fastq` and, inside that, `<region>`
+4. Upload the JSON file to `s3://<input-bucket>/fastq/<region>/<region>.json`
+
+### View the Results
+
+1. Go to the 'packager' Lambda and view the logs
+2. Grab the 'quilt+uri' from the output
+3. Paste into the URI field of the Quilt Catalog (next to the search bar)
+   1. NOTE: May need to click the package name to stop page reloading
+4. Click "Expand" (and tripledot menu) to interact with result DataGrids
+
 ## Development
 
 Uses [pre-commit](https://pre-commit.com/) to pre-lint files.

src/constants.ts

@@ -1,5 +1,5 @@
 import 'dotenv/config';
-import { readFileSync, writeFileSync } from 'fs';
+import { readFileSync, writeFileSync, mkdirSync } from 'fs';
 import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
 import handlebars from 'handlebars';
 import yaml from 'js-yaml';
@@ -58,10 +58,14 @@ export class Constants {
     return components.join('/');
   }
 
+  public static IsLocal(scheme: string): boolean {
+    return scheme === '' || scheme === 'file' || scheme[0] === '/' || scheme[0] == '.';
+  }
+
   public static async LoadObjectURI(uri: string, env: object = {}): Promise<KeyedConfig> {
     const split = uri.split('://');
     const scheme = split[0];
-    if (!scheme || scheme === '' || scheme === 'file' || scheme[0] === '/' || scheme[0] == '.') {
+    if (Constants.IsLocal(scheme)) {
       return Constants.LoadObjectFile(uri, env);
     }
     if (scheme !== 's3') {
@@ -88,17 +92,30 @@ export class Constants {
     }
   }
 
+  public static makeDirectory(file_path: string): void {
+    const dirs = file_path.split('/');
+    dirs.pop();
+    const dir = dirs.join('/');
+    mkdirSync(dir, { recursive: true });
+  }
+
   public static async SaveObjectURI(uri: string, itme: any): Promise<void> {
-    const data = JSON.stringify(itme, null, 2);
+    const data = JSON.stringify(itme, null, 2) + '\n';
     const split = uri.split('://');
     const scheme = split[0];
-    if (!scheme || scheme === '' || scheme === 'file' || scheme[0] === '/' || scheme[0] == '.') {
-      return writeFileSync(uri, data);
+    if (Constants.IsLocal(scheme)) {
+      Constants.makeDirectory(uri);
+      try {
+        return writeFileSync(uri, data);
+      } catch (error: any) {
+        console.error(error);
+        return;
+      }
     }
+    const paths = split[1].split('/');
     if (scheme !== 's3') {
       throw new Error(`Unsupported scheme: ${scheme}`);
     }
-    const paths = split[1].split('/');
     const s3 = Constants.DefaultS3();
     const bucket = paths[0];
     const file = paths.slice(-1)[0];

src/packager/tests/conftest.py

@@ -4,5 +4,5 @@
     "EVENT": "../../test/events/event-params.json",
     "S3_URI": "s3://data-yaml-spec-tests/quilt_metadata.json",
     "S3_BUCKET": "data-yaml-spec-tests",
-    "TIMEOUT": "1",
+    "TIMEOUT": "5",
 }

test/omics-quilt.fastq.test.ts

@@ -45,7 +45,6 @@ describe('save_metadata', () => {
     const item = { name: 'Test Item' };
     const cc = new Constants(CONTEXT);
 
-    // Call the save_metadata function with the provided parameters
     await save_metadata(id, item, cc);
 
     const root = CONTEXT.OUTPUT_S3_LOCATION + '/' + id;

workflows/fastq/DEMO.md

@@ -0,0 +1,21 @@
+# Fastq Demo Walkthrough
+
+## Intro
+
+ 1. Show Git Repo <https://github.com/quiltdata/omics-quilt-demo>
+ 2. Walk through CDK Usage in README.md
+ 3. Talk through Cloud 9 environment
+
+### Launch Omics Job
+
+ 1. Go to Omics Input bucket in Quilt Catalog
+ 2. Edit Timestamp of /fast/us-east-1/us-east-1.json
+ 3. Show "fastq" Lambda running in AWS Console
+ 4. Show Omics job executing in AWS Console under "Runs"
+
+### View Omics Results
+
+ 1. Go to Omics Output bucket in Quilt Catalog
+ 2. Show test Package (real Package if it completes)
+ 3. Expand CSV
+ 4. Show metadata lineage