[CUMULUS-3368]: [User Contribution] Introducing allFilesPresent metadata field for Cumulus Collections. Fixes Issue #3193

CHANGELOG.md

@@ -48,6 +48,11 @@ Users/clients that do not make use of these endpoints will not be impacted.
   - Added optional `maxDownloadTime` field to `provider` schema
   - Added `max_download_time` column to PostgreSQL `providers` table
   - Updated `@cumulus/ingest/lock` to check expired locks based on `provider.maxDownloadTime`
+- **CUMULUS-3368**
+  - Added `allFilesPresent` field for `collection.meta` that defines whether or not all files
+    should be present in a granule before it can be discovered. If `allFilesPresent` is true,
+    the granules missing files will be removed. Otherwise, the default behavior is discover-granules
+    will ignore missing files in granules.
 
 ### Changed
 

docs/workflow_tasks/discover_granules.md

@@ -48,6 +48,10 @@ A list of buckets with types that will be used to assign bucket targets based on
 
 A Cumulus [collection](https://github.com/nasa/cumulus/blob/master/packages/api/lib/schemas.js) object.    Used to define granule file groupings and granule metadata for discovered files.   The collection object utilizes the collection type key to generate types in the output object on discovery.
 
+##### Collection Meta
+
+If the collection is configured with `collection.meta.allFilesPresent` set to `true`, the task will remove granules's missing files. Otherwise, the default behavior ignores missing files in granules.
+
 #### DuplicateGranuleHandling
 
 A string configuration that configures the step to filter the granules discovered:

tasks/discover-granules/index.js

@@ -225,6 +225,62 @@ const checkGranuleHasNoDuplicate = async (granuleId, duplicateHandling) => {
   throw new Error(`Unexpected return from Private API lambda: ${JSON.stringify(response)}`);
 };
 
+/**
+ * Checks if all files from a collection are present for files from a granuleId
+ *
+ *
+ * @param {Object} config - the event config
+ * @param {Object} filesByGranuleId - Object with granuleId for keys with an array of
+ *                                    matching files for each
+ * @param {string} granuleId - the Id of a granule
+ * @returns {Boolean} - returns true if all file in collection config are present for granuleId
+ */
+const checkGranuleHasAllFiles = (config, filesByGranuleId, granuleId) => {
+  const granuleHasAllFiles = filesByGranuleId[granuleId].length >= config.collection.files.length;
+  return granuleHasAllFiles;
+};
+
+/**
+ * Filters out granules that are missing files from a collection
+ *
+ *
+ * @param {Object} params.filesByGranuleId - Object with granuleId for keys with an array of
+ *                                           matching files for each
+ * @param {Object} params.config - the event config
+ * @returns {Array.string} - returns granuleIds that contain all files from a collection
+ */
+const filterGranulesWithoutAllFiles = ({ filesByGranuleId, config }) => {
+  const checkResults = Object.keys(filesByGranuleId).filter(
+    checkGranuleHasAllFiles.bind(this, config, filesByGranuleId)
+  );
+  return checkResults;
+};
+
+/**
+ * Handles granules without all files present according to the allFilesPresent boolean
+ *
+ * allFilesPresent = true : granules missing files will be removed
+ * allFilesPresent = false: ignore missing files in granules
+ *
+ * @param {Object} params.filesByGranuleId - Object with granuleId for keys with an array of
+ *                                           matching files for each
+ * @param {Boolean} params.allFilesPresent - boolean that defines whether or not all files should
+ *                                           be present in a granule before it can be discovered
+ * @param {Object} params.config - the event config
+ * @returns {Object} returns filesByGranuleId with all the granules missing files removed
+ */
+const handleGranulesWithoutAllFilesPresent = ({ filesByGranuleId, allFilesPresent, config }) => {
+  if (!allFilesPresent) {
+    return filesByGranuleId;
+  }
+  const filteredKeys = filterGranulesWithoutAllFiles({
+    granuleIds: Object.keys(filesByGranuleId),
+    filesByGranuleId,
+    config,
+  });
+  return pick(filesByGranuleId, filteredKeys);
+};
+
 /**
  * Filters granule duplicates from a list of granuleIds according to the
  * configuration in duplicateHandling:
@@ -314,6 +370,16 @@ const discoverGranules = async ({ config }) => {
     concurrency: get(config, 'concurrency', 3),
   });
 
+  let allFilesPresent = false;
+  if (config.collection.meta) {
+    allFilesPresent = config.collection.meta.allFilesPresent || false;
+  }
+  filesByGranuleId = handleGranulesWithoutAllFilesPresent({
+    filesByGranuleId,
+    allFilesPresent,
+    config,
+  });
+
   const discoveredGranules = map(filesByGranuleId, buildGranule(config));
 
   logger({ granules: discoveredGranules.map((g) => g.granuleId) }).info(`Discovered ${discoveredGranules.length} granules.`);

tasks/discover-granules/tests/index.js

@@ -190,8 +190,8 @@ const discoverGranulesUsingS3 = (configure, assert = assertDiscoveredGranules) =
     // State sample files
     const files = [
       'granule-1.nc', 'granule-1.nc.md5',
-      'granule-2.nc', 'granule-2.nc.md5',
-      'granule-3.nc', 'granule-3.nc.md5',
+      'granule-2.nc', 'granule-2.nc.md5', 'granule-2.nc.cmr.json',
+      'granule-3.nc', 'granule-3.nc.md5', 'granule-3.nc.cmr.json', 'granule-3.nc.xml',
     ];
 
     config.sourceBucketName = randomString();
@@ -216,6 +216,126 @@ const discoverGranulesUsingS3 = (configure, assert = assertDiscoveredGranules) =
     }
   };
 
+test('discover granules with allFilesPresent set to true',
+  discoverGranulesUsingS3(({ context: { event: { config } } }) => {
+    config.provider = {
+      id: 'MODAPS',
+      protocol: 's3',
+      host: config.sourceBucketName,
+    };
+    // Adds cmr.json files to collection config
+    config.collection.granuleIdExtraction = '^(.*)\\.(nc|nc\\.md5|nc\\.cmr\\.json)$';
+    config.collection.meta = {
+      allFilesPresent: true,
+    };
+    config.collection.files = [
+      {
+        regex: '.*.nc.cmr.json$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.cmr.json',
+        bucket: 'protected',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc',
+        bucket: 'protected',
+        type: 'data',
+      },
+      {
+        regex: '.*.nc.md5$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+    ];
+  }, async (t, output) => {
+    await validateOutput(t, output);
+    t.is(output.granules.length, 2);
+    output.granules.forEach(({ files }) => t.is(files.length, 3));
+  }));
+
+test('discover granules task discovers no granules when allFilesPresent is set to true with files missing',
+  discoverGranulesUsingS3(({ context: { event: { config } } }) => {
+    config.provider = {
+      id: 'MODAPS',
+      protocol: 's3',
+      host: config.sourceBucketName,
+    };
+    // Adds cmr.json files to collection config
+    config.collection.granuleIdExtraction = '^(.*)\\.(nc|nc\\.md5|nc\\.cmr\\.json|nc\\.png)$';
+    config.collection.meta = {
+      allFilesPresent: true,
+    };
+    config.collection.files = [
+      {
+        regex: '.*.nc.cmr.json$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.cmr.json',
+        bucket: 'protected',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc',
+        bucket: 'protected',
+        type: 'data',
+      },
+      {
+        regex: '.*.nc.md5$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc.png$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+    ];
+  }, async (t, output) => {
+    await validateOutput(t, output);
+    t.is(output.granules.length, 0);
+    output.granules.forEach(({ files }) => t.is(files.length, 0));
+  }));
+
+test('discover granules task discovers granules with allFilesPresent set to true with extra files for granule',
+  discoverGranulesUsingS3(({ context: { event: { config } } }) => {
+    config.provider = {
+      id: 'MODAPS',
+      protocol: 's3',
+      host: config.sourceBucketName,
+    };
+    // Adds cmr.json and .xml granuleId
+    config.collection.granuleIdExtraction = '^(.*)\\.(nc|nc\\.md5|nc\\.cmr\\.json|nc\\.xml)$';
+    config.collection.meta = {
+      allFilesPresent: true,
+    };
+    config.collection.files = [
+      {
+        regex: '.*.nc.cmr.json$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.cmr.json',
+        bucket: 'protected',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc',
+        bucket: 'protected',
+        type: 'data',
+      },
+      {
+        regex: '.*.nc.md5$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+    ];
+  }, async (t, output) => {
+    await validateOutput(t, output);
+    t.is(output.granules.length, 2);
+    output.granules.forEach(({ files }) => t.is(files.length, 3));
+  }));
+
 test('discover granules using S3',
   discoverGranulesUsingS3(({ context: { event: { config } } }) => {
     config.provider = {