Modified offsets

* Satellite data has a lag before it becomes available.
* The script has been modified so that the offset days for downloading and
processing of the data can be controlled from environment variables
(VIIRS_OFFSET and MODIS_OFFSET)

.vscode/settings.json

@@ -25,5 +25,6 @@
     "files.trimTrailingWhitespace": true,
     "files.associations": {
         "*.jenkins": "Jenkinsfile"
-    }
+    },
+    "python.pythonPath": "/home/kjnether/proj/rfc/envs/snowpack/bin/python"
 }

admin/constants.py

@@ -57,3 +57,13 @@
 VIIRS_DAILY_20YR = os.path.join(VIIRS_DAILY_NORM, '20yr')
 
 AOI = os.path.join(os.path.dirname(__file__), '..', 'aoi')
+
+# set default values and then override with what is in the
+# env vars if populated with a valid value
+VIIRS_OFFSET = 1
+if ('VIIRS_OFFSET' in os.environ ) and os.environ['VIIRS_OFFSET']:
+    VIIRS_OFFSET=os.environ['VIIRS_OFFSET']
+
+MODIS_OFFSET = 3
+if ('MODIS_OFFSET' in os.environ) and os.environ['MODIS_OFFSET']:
+    MODIS_OFFSET=os.environ['MODIS_OFFSET']

run.py

@@ -167,7 +167,7 @@ def daily_pipeline(envpth: str, date: str, clean: str, days: int = 5):
         date_l = date.split('.')
         target_date = datetime.datetime(int(date_l[0]), int(date_l[1]), int(date_l[2]))
         if target_date.date() == datetime.datetime.now(pst).date():
-            date = datetime.datetime.strftime(target_date - datetime.timedelta(days=4), '%Y.%m.%d')
+            date = datetime.datetime.strftime(target_date - datetime.timedelta(days=const.MODIS_OFFSET), '%Y.%m.%d')
         for sat in ['modis','viirs']:
             logger.info(f'Daily Pipeline running {sat} process')
             dailypipeline(envpth, date, sat, int(days), db_handler)
@@ -179,7 +179,7 @@ def daily_pipeline(envpth: str, date: str, clean: str, days: int = 5):
 def dailypipeline(envpth: str, date: str, sat: str, days: int, db_handler: DBHandler):
     if check_date(date):
         if sat == 'viirs':
-            days = 1
+            days = const.VIIRS_OFFSET #1
         #print(type())
         #logger.debug(f'download granules: {download_granules} - {download_granules.download_granules}')
         download_granules.download_granules.download_granules(envpth, date, sat, int(days))