Merge pull request #218 from cumulus-nasa/UpdateDiscoverGranules

Update discover-granules

.tmp-test-data/.gitignore

@@ -0,0 +1,3 @@
+*
+!README
+!.gitignore

.tmp-test-data/README

@@ -0,0 +1,6 @@
+The http and sftp containers managed by docker-compose.yml will serve out any
+files in this directory.  The intention is that any files being fetched using
+http or sftp should be copied into this directory (probably under a
+subdirectory), fetched, and then deleted.
+
+The docker containers defined in docker-compose.yml are used for testing.

cumulus/tasks/copy-idx-from-s3-to-efs/package.json

@@ -48,7 +48,6 @@
     "babel-polyfill": "^6.23.0",
     "babel-preset-env": "^1.4.0",
     "babel-register": "^6.26.0",
-    "sinon": "^2.0.0-pre.5",
     "webpack": "^1.12.13"
   }
 }

cumulus/tasks/delete-ingest-tracking-data/package.json

@@ -46,7 +46,6 @@
     "babel-loader": "^6.2.4",
     "babel-polyfill": "^6.23.0",
     "babel-preset-es2017": "^6.24.1",
-    "sinon": "^2.0.0-pre.5",
     "webpack": "^1.12.13"
   }
 }

cumulus/tasks/delete-pdr-ftp/package.json

@@ -53,7 +53,6 @@
     "babel-polyfill": "^6.23.0",
     "babel-preset-env": "^1.4.0",
     "babel-preset-es2017": "^6.24.1",
-    "sinon": "^2.0.0-pre.5",
     "webpack": "^1.12.13"
   }
 }

cumulus/tasks/delete-pdr-s3/package.json

@@ -57,7 +57,6 @@
     "babel-polyfill": "^6.23.0",
     "babel-preset-env": "^1.4.0",
     "babel-preset-es2017": "^6.24.1",
-    "sinon": "^2.0.0-pre.5",
     "webpack": "^1.12.13"
   }
 }

cumulus/tasks/discover-cmr-granules/package.json

@@ -43,7 +43,6 @@
     "babel-core": "^6.25.0",
     "babel-loader": "^6.2.4",
     "babel-polyfill": "^6.23.0",
-    "babel-preset-es2017": "^6.24.1",
-    "sinon": "^2.0.0-pre.5"
+    "babel-preset-es2017": "^6.24.1"
   }
 }

cumulus/tasks/discover-granules/index.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const cumulusMessageAdapter = require('@cumulus/cumulus-message-adapter-js');
-const get = require('lodash.get');
 const granule = require('@cumulus/ingest/granule');
 const log = require('@cumulus/common/log');
 
@@ -15,9 +14,8 @@ const log = require('@cumulus/common/log');
 **/
 async function discoverGranules(event) {
   const protocol = event.config.provider.protocol;
-  const useQueue = get(event.config, 'useQueue', true);
 
-  const Discover = granule.selector('discover', protocol, useQueue);
+  const Discover = granule.selector('discover', protocol);
   const discover = new Discover(event);
 
   let granules;

cumulus/tasks/discover-granules/package.json

@@ -54,7 +54,7 @@
     "@ava/babel-preset-stage-4": "^1.1.0",
     "@ava/babel-preset-transform-test-files": "^3.0.0",
     "ava": "^0.21.0",
-    "lodash.clonedeep": "^4.5.0",
-    "sinon": "^2.0.0-pre.5"
+    "fs-extra": "^5.0.0",
+    "lodash.clonedeep": "^4.5.0"
   }
 }

cumulus/tasks/discover-granules/schemas/config.json

@@ -2,12 +2,11 @@
   "title": "DiscoverGranulesConfig",
   "description": "Describes the config used by the discover-granules task",
   "type": "object",
+  "required": [ "provider" ],
   "properties": {
-    "useQueue": {
-      "type": "boolean"
-    },
     "provider": {
       "type": "object",
+      "required": [ "host", "protocol" ],
       "properties": {
         "id": { "type": "string" },
         "username": { "type": "string" },
@@ -17,7 +16,7 @@
         "globalConnectionLimit": { "type": "integer" },
         "protocol": {
           "type": "string",
-          "enum": ["ftp", "sftp", "http", "https"]
+          "enum": ["ftp", "sftp", "http", "https", "s3"]
         }
       }
     },
@@ -56,4 +55,4 @@
       }
     }
   }
-}
+}

cumulus/tasks/discover-granules/schemas/output.json

@@ -2,20 +2,34 @@
   "title": "DiscoverGranulesOutput",
   "description": "Describes the output produced by the discover-granules task",
   "type": "object",
-  "required": ["granules"],
+  "required": [ "granules" ],
   "additionalProperties": false,
   "properties": {
-    "granules_found": { "type": "integer" },
     "granules": {
       "type": "array",
       "items": {
         "type": "object",
-        "additionalProperties": false,
+        "required": [ "granuleId", "files" ],
         "properties": {
-          "name": { "type": "string" },
           "granuleId": { "type": "string" },
-          "bucket": { "type": "string" },
-          "url_path": { "type": "string" }
+          "files": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "required": [ "name", "path" ],
+              "properties": {
+                "name": { "type": "string" },
+                "path": { "type": "string" },
+                "size": { "type": "integer" },
+                "time": {
+                  "description": "The number of milliseconds since January 1, 1970, 00:00:00 UTC",
+                  "type": "integer"
+                },
+                "bucket": { "type": "string" },
+                "url_path": { "type": "string" }
+              }
+            }
+          }
         }
       }
     }

cumulus/tasks/discover-granules/tests/fixtures/mur.json

@@ -1,6 +1,5 @@
 {
   "config": {
-    "useQueue": false,
     "provider": {
       "id": "MUR",
       "globalConnectionLimit": 10,