Merge pull request #217 from cumulus-nasa/testing-ftp-yay

Added FTP options flag "useList"
nasa · Feb 27, 2018 · d3df8e6 · d3df8e6
2 parents 34a85f1 + 8580022
commit d3df8e6
Show file tree

Hide file tree

Showing 15 changed files with 267 additions and 198 deletions.
diff --git a/cumulus/tasks/discover-granules/package.json b/cumulus/tasks/discover-granules/package.json
@@ -23,7 +23,6 @@
   "ava": {
     "files": "tests",
     "color": false,
-    "serial": true,
     "babel": "inherit",
     "require": [
       "babel-polyfill",

diff --git a/cumulus/tasks/discover-granules/schemas/config.json b/cumulus/tasks/discover-granules/schemas/config.json
@@ -53,6 +53,9 @@
           }
         }
       }
+    },
+    "useList": {
+      "type": "boolean"
     }
   }
 }
diff --git a/cumulus/tasks/discover-granules/tests/index.js b/cumulus/tasks/discover-granules/tests/index.js
@@ -7,7 +7,7 @@ const mur = require('./fixtures/mur.json');
 const { cloneDeep } = require('lodash');
 const { recursivelyDeleteS3Bucket, s3 } = require('@cumulus/common/aws');
 const {
-  findGitRepoRootDirectory,
+  findTmpTestDataDirectory,
   randomString,
   validateConfig,
   validateOutput
@@ -39,8 +39,7 @@ test('discover granules using SFTP', async (t) => {
   const providerPath = randomString();
 
   // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([
@@ -98,8 +97,7 @@ test('discover granules using HTTP', async (t) => {
   const providerPath = randomString();
 
   // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([
@@ -153,8 +151,7 @@ test('discover granules using S3', async (t) => {
   const providerPath = randomString();
 
   // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([

diff --git a/cumulus/tasks/discover-pdrs/index.js b/cumulus/tasks/discover-pdrs/index.js
@@ -29,7 +29,8 @@ function discoverPdrs(event) {
       stack,
       bucket,
       collection,
-      provider
+      provider,
+      config.useList
     );
 
     log.debug('Starting PDR discovery');

diff --git a/cumulus/tasks/discover-pdrs/package.json b/cumulus/tasks/discover-pdrs/package.json
@@ -26,7 +26,6 @@
       "babel-polyfill",
       "babel-register"
     ],
-    "serial": true,
     "color": false
   },
   "babel": {

diff --git a/cumulus/tasks/discover-pdrs/schemas/config.json b/cumulus/tasks/discover-pdrs/schemas/config.json
@@ -39,6 +39,9 @@
         "name": { "type": "string" },
         "granuleIdExtraction": { "type": "string" }
       }
+    },
+    "useList": {
+      "type": "boolean"
     }
   }
 }
diff --git a/cumulus/tasks/discover-pdrs/tests/index.js b/cumulus/tasks/discover-pdrs/tests/index.js
@@ -11,7 +11,8 @@ const input = require('./fixtures/input.json');
 
 const { recursivelyDeleteS3Bucket, s3 } = require('@cumulus/common/aws');
 const {
-  findGitRepoRootDirectory,
+  findTestDataDirectory,
+  findTmpTestDataDirectory,
   randomString,
   validateConfig,
   validateOutput
@@ -21,6 +22,7 @@ test('test pdr discovery with FTP assuming all PDRs are new', async (t) => {
   const event = cloneDeep(input);
   event.config.bucket = randomString();
   event.config.collection.provider_path = '/pdrs';
+  event.config.useList = true;
   event.config.provider = {
     id: 'MODAPS',
     protocol: 'ftp',
@@ -110,6 +112,7 @@ test('test pdr discovery with FTP assuming some PDRs are new', async (t) => {
   };
 
   const newPayload = cloneDeep(input);
+  newPayload.config.useList = true;
   newPayload.config.provider = provider;
   newPayload.config.collection.provider_path = '/pdrs';
   newPayload.input = {};
@@ -153,9 +156,8 @@ test('test pdr discovery with HTTP assuming some PDRs are new', async (t) => {
   const providerPath = randomString();
 
   // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const testDataDirectory = path.join(gitRepoRootDirectory, 'packages', 'test-data', 'pdrs');
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const testDataDirectory = path.join(await findTestDataDirectory(), 'pdrs');
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([
@@ -225,9 +227,8 @@ test('test pdr discovery with SFTP assuming some PDRs are new', async (t) => {
   const providerPath = randomString();
 
   // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const testDataDirectory = path.join(gitRepoRootDirectory, 'packages', 'test-data', 'pdrs');
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const testDataDirectory = path.join(await findTestDataDirectory(), 'pdrs');
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([

diff --git a/cumulus/tasks/discover-pdrs/tests/s3.js b/cumulus/tasks/discover-pdrs/tests/s3.js
@@ -21,7 +21,8 @@ test.beforeEach(async (t) => {
         id: randomString(),
         protocol: 's3'
       },
-      stack: randomString()
+      stack: randomString(),
+      useList: true
     }
   };
 

diff --git a/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js b/cumulus/tasks/parse-pdr/tests/parse_pdrs_test.js
@@ -9,7 +9,8 @@ const test = require('ava');
 const { recursivelyDeleteS3Bucket, s3 } = require('@cumulus/common/aws');
 const { cloneDeep } = require('lodash');
 const {
-  findGitRepoRootDirectory,
+  findTestDataDirectory,
+  findTmpTestDataDirectory,
   randomString,
   validateConfig,
   validateInput,
@@ -19,29 +20,26 @@ const {
 const { parsePdr } = require('../index');
 
 test('parse PDR from FTP endpoint', async (t) => {
-  const provider = {
+  const internalBucketName = randomString();
+
+  const newPayload = cloneDeep(modis);
+
+  newPayload.config.bucket = internalBucketName;
+  newPayload.config.provider = {
     id: 'MODAPS',
     protocol: 'ftp',
     host: 'localhost',
     username: 'testuser',
     password: 'testpass'
   };
 
-  const pdrName = 'MOD09GQ.PDR';
-
-  const newPayload = cloneDeep(modis);
-  newPayload.config.provider = provider;
-
-  const internalBucketName = randomString();
-  newPayload.config.bucket = internalBucketName;
-
   await validateConfig(t, newPayload.config);
 
   return s3().createBucket({ Bucket: internalBucketName }).promise()
     .then(() => parsePdr(newPayload))
     .then((output) => {
       t.is(output.granules.length, output.granulesCount);
-      t.is(output.pdr.name, pdrName);
+      t.is(output.pdr.name, newPayload.input.pdr.name);
       t.is(output.filesCount, 2);
       return output;
     })
@@ -61,8 +59,8 @@ test('parse PDR from HTTP endpoint', async (t) => {
   const providerPath = randomString();
 
   // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const testDataDirectory = path.join(await findTestDataDirectory(), 'pdrs');
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([
@@ -73,7 +71,7 @@ test('parse PDR from HTTP endpoint', async (t) => {
   const pdrName = 'MOD09GQ.PDR';
 
   await fs.copy(
-    path.join(gitRepoRootDirectory, 'packages', 'test-data', 'pdrs', pdrName),
+    path.join(testDataDirectory, pdrName),
     path.join(providerPathDirectory, pdrName));
 
   const newPayload = cloneDeep(modis);
@@ -118,10 +116,7 @@ test('parse PDR from HTTP endpoint', async (t) => {
 test('parse PDR from SFTP endpoint', async (t) => {
   const internalBucketName = randomString();
   const providerPath = randomString();
-
-  // Figure out the directory paths that we're working with
-  const gitRepoRootDirectory = await findGitRepoRootDirectory();
-  const providerPathDirectory = path.join(gitRepoRootDirectory, 'tmp-test-data', providerPath);
+  const providerPathDirectory = path.join(await findTmpTestDataDirectory(), providerPath);
 
   // Create providerPathDirectory and internal bucket
   await Promise.all([
@@ -131,10 +126,6 @@ test('parse PDR from SFTP endpoint', async (t) => {
 
   const pdrName = 'MOD09GQ.PDR';
 
-  await fs.copy(
-    path.join(gitRepoRootDirectory, 'packages', 'test-data', 'pdrs', pdrName),
-    path.join(providerPathDirectory, pdrName));
-
   const newPayload = cloneDeep(modis);
   newPayload.config.bucket = internalBucketName;
   newPayload.config.provider = {
@@ -158,7 +149,14 @@ test('parse PDR from SFTP endpoint', async (t) => {
   await validateConfig(t, newPayload.config);
 
   try {
+    // Stage the file to be downloaded
+    const testDataDirectory = path.join(await findTestDataDirectory(), 'pdrs');
+    await fs.copy(
+      path.join(testDataDirectory, pdrName),
+      path.join(providerPathDirectory, pdrName));
+
     const output = await parsePdr(newPayload);
+
     await validateOutput(t, output);
     t.is(output.granules.length, output.granulesCount);
     t.is(output.pdr.name, pdrName);

diff --git a/cumulus/tasks/sync-granule/tests/.eslintrc.json b/cumulus/tasks/sync-granule/tests/.eslintrc.json
@@ -0,0 +1,5 @@
+{
+  "rules": {
+    "no-param-reassign": "off"
+  }
+}