Update to allow the observation date database to be exported and impo…

…rted.

bin/eoddexportdb.py

@@ -43,8 +43,10 @@
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("-c", "--config", type=str, default="", help="Path to the JSON config file.")
-    parser.add_argument("-s", "--sensor", type=str, required=True, choices=EODATADOWN_SENSORS_LIST,
+    parser.add_argument("-s", "--sensor", type=str, required=False, choices=EODATADOWN_SENSORS_LIST,
                         help='''Specify the sensor you wish to export''')
+    parser.add_argument("--obsdate", action='store_true', default=False,
+                        help="Exports the observation date database.")
     parser.add_argument("-o", "--output", type=str, required=True,
                         help="Specify the JSON file to which the sensors database should be exported to.")
     args = parser.parse_args()
@@ -62,7 +64,12 @@
     t = rsgislib.RSGISTime()
     t.start(True)
 
-    eodatadown.eodatadownrun.export_sensor_database(config_file, args.sensor, args.output)
+    if args.obsdate:
+        eodatadown.eodatadownrun.export_obsdate_database(config_file, args.output)
+    else:
+        if args.sensor is None:
+            raise Exception("A sensor needs to be specified.")
+        eodatadown.eodatadownrun.export_sensor_database(config_file, args.sensor, args.output)
 
     t.end(reportDiff=True, preceedStr='EODataDown export completed ', postStr=' - eoddexportdb.py.')
 

bin/eoddimportdb.py

@@ -43,13 +43,15 @@
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("-c", "--config", type=str, default="", help="Path to the JSON config file.")
-    parser.add_argument("-s", "--sensor", type=str, required=True, choices=EODATADOWN_SENSORS_LIST,
+    parser.add_argument("-s", "--sensor", type=str, required=False, choices=EODATADOWN_SENSORS_LIST,
                         help='''Specify the sensor you wish to export''')
     parser.add_argument("-i", "--input", type=str, required=True,
                         help="Specify the JSON file which should be appended to sensors database.")
     parser.add_argument("-p", "--paths", type=str, required=False,
                         help="""Specify a JSON file will key pairs for file paths which should be updated
                                 (keys are the path to replace and value is the replacement value).""")
+    parser.add_argument("--obsdate", action='store_true', default=False,
+                        help="Exports the observation date database.")
     args = parser.parse_args()
 
     config_file = args.config
@@ -65,7 +67,10 @@
     t = rsgislib.RSGISTime()
     t.start(True)
 
-    eodatadown.eodatadownrun.import_sensor_database(config_file, args.sensor, args.input, args.paths)
+    if args.obsdate:
+        eodatadown.eodatadownrun.import_obsdate_database(config_file, args.input, args.paths)
+    else:
+        eodatadown.eodatadownrun.import_sensor_database(config_file, args.sensor, args.input, args.paths)
 
     t.end(reportDiff=True, preceedStr='EODataDown import completed ', postStr=' - eoddimportdb.py.')
 

eodatadown/__init__.py

@@ -41,7 +41,7 @@
 import json
 
 EODATADOWN_VERSION_MAJOR = 0
-EODATADOWN_VERSION_MINOR = 52
+EODATADOWN_VERSION_MINOR = 53
 EODATADOWN_VERSION_PATCH = 0
 
 # Check is GTIFF Creation Options Flag has been defined and if not then define it.

eodatadown/eodatadownlandsatgoogsensor.py

@@ -2166,8 +2166,7 @@ def export_db_to_json(self, out_json_file):
     def import_sensor_db(self, input_json_file, replace_path_dict=None):
         """
         This function imports from the database records from the specified input JSON file.
-        The database table checks are not made for duplicated as records are just appended
-        to the table with a new PID.
+
         :param input_json_file: input JSON file with the records to be imported.
         :param replace_path_dict: a dictionary of file paths to be updated, if None then ignored.
         """
@@ -2222,6 +2221,7 @@ def import_sensor_db(self, input_json_file, replace_path_dict=None):
             ses = session_sqlalc()
             ses.add_all(db_records)
             ses.commit()
+            ses.close()
 
     def create_gdal_gis_lyr(self, file_path, lyr_name, driver_name='GPKG', add_lyr=False):
         """

eodatadown/eodatadownrun.py

@@ -531,6 +531,59 @@ def import_sensor_database(config_file, sensor, in_json_file, replace_path_jsonf
     edd_usage_db.add_entry("Finished: Import {} sensors database table.".format(sensor), end_block=True)
 
 
+def export_obsdate_database(config_file, out_json_file):
+    """
+    A function to export the observation dates database tables to a JSON file.
+
+    :param config_file: The EODataDown configuration file path.
+    :param out_json_file: the file path of the output JSON file.
+
+    """
+    sys_main_obj = eodatadown.eodatadownsystemmain.EODataDownSystemMain()
+    sys_main_obj.parse_config(config_file)
+    logger.debug("Parsed the system configuration.")
+
+    edd_usage_db = sys_main_obj.get_usage_db_obj()
+    edd_usage_db.add_entry("Starting: Export obsdate database table.", start_block=True)
+
+    obsdates_obj = sys_main_obj.get_obsdates_obj()
+    if obsdates_obj is None:
+        logger.error("Error occurred and the observation date object could not be created.")
+    obsdates_obj.export_db_to_json(out_json_file)
+
+    edd_usage_db.add_entry("Finished: Export obsdate database table.", end_block=True)
+
+
+def import_obsdate_database(config_file, in_json_file, replace_path_jsonfile):
+    """
+    A function to import the observation dates database table from a JSON file.
+
+    :param config_file: The EODataDown configuration file path.
+    :param in_json_file: the file path of the output JSON file.
+    :param replace_path_jsonfile: a JSON file with pairs of paths to be updated during import.
+
+    """
+    if replace_path_jsonfile is None:
+        replace_path_dict = None
+    else:
+        json_file_obj = open(replace_path_jsonfile)
+        replace_path_dict = json.load(json_file_obj)
+
+    sys_main_obj = eodatadown.eodatadownsystemmain.EODataDownSystemMain()
+    sys_main_obj.parse_config(config_file)
+    logger.debug("Parsed the system configuration.")
+
+    edd_usage_db = sys_main_obj.get_usage_db_obj()
+    edd_usage_db.add_entry("Starting: Import obsdate database tables.", start_block=True)
+
+    obsdates_obj = sys_main_obj.get_obsdates_obj()
+    if obsdates_obj is None:
+        logger.error("Error occurred and the observation date object could not be created.")
+    obsdates_obj.import_obsdates_db(in_json_file, replace_path_dict)
+
+    edd_usage_db.add_entry("Finished: Import obsdate database table.", end_block=True)
+
+
 def run_scn_analysis(params):
     """
     This function runs a full per scene analysis and will check whether the scene has been: downloaded,

eodatadown/eodatadownsensor.py

@@ -443,8 +443,8 @@ def process_obsdata(self, sys_main_obj, sensor_id, platform_id, obs_date):
         ses = session_sqlalc()
 
         obs_qry = ses.query(EDDObsDates).filter(EDDObsDates.SensorID == sensor_id,
-                                            EDDObsDates.PlatformID == platform_id,
-                                            EDDObsDates.ObsDate == obs_date).one_or_none()
+                                                EDDObsDates.PlatformID == platform_id,
+                                                EDDObsDates.ObsDate == obs_date).one_or_none()
         if obs_qry is not None:
             obsdate_scns_qry = ses.query(EDDObsDatesScns).filter(EDDObsDatesScns.SensorID == obs_qry.SensorID,
                                                                  EDDObsDatesScns.PlatformID == obs_qry.PlatformID,
@@ -576,4 +576,103 @@ def get_obs_scns(self, start_date, end_date, sensor=None, platform=None, valid=T
                                                                 EDDObsDates.ObsDate.asc()).all()
         return obsdate_qry
 
+    def export_db_to_json(self, out_json_file):
+        """
+        Export the EDDObsDates and EDDObsDateScns databases to a JSON file.
+
+        :param out_json_file: The output JSON text file.
+
+        """
+        db_engine = sqlalchemy.create_engine(self.db_info_obj.dbConn)
+        session_sqlalc = sqlalchemy.orm.sessionmaker(bind=db_engine)
+        ses = session_sqlalc()
+
+        query_result = ses.query(EDDObsDates).all()
+        db_obs_dates_dict = dict()
+        pid = 0
+        for obsdate in query_result:
+            db_obs_dates_dict[pid] = dict()
+            db_obs_dates_dict[pid]["SensorID"] = obsdate.SensorID
+            db_obs_dates_dict[pid]["PlatformID"] = obsdate.PlatformID
+            db_obs_dates_dict[pid]["ObsDate"] = obsdate.ObsDate
+            db_obs_dates_dict[pid]["OverviewCreated"] = obsdate.OverviewCreated
+            db_obs_dates_dict[pid]["NeedUpdate"] = obsdate.NeedUpdate
+            db_obs_dates_dict[pid]["Invalid"] = obsdate.Invalid
+            db_obs_dates_dict[pid]["Overviews"] = obsdate.Overviews
+            pid = pid + 1
+
+        query_result = ses.query(EDDObsDatesScns).all()
+        db_obs_date_scns_dict = dict()
+        pid = 0
+        for obsdatescns in query_result:
+            db_obs_date_scns_dict[pid] = dict()
+            db_obs_date_scns_dict[pid]["SensorID"] = obsdatescns.SensorID
+            db_obs_date_scns_dict[pid]["PlatformID"] = obsdatescns.PlatformID
+            db_obs_date_scns_dict[pid]["ObsDate"] = obsdatescns.ObsDate
+            db_obs_date_scns_dict[pid]["Scene_PID"] = obsdatescns.Scene_PID
+            pid = pid + 1
+        ses.close()
 
+        out_dict = dict()
+        out_dict["EDDObsDates"] = db_obs_dates_dict
+        out_dict["EDDObsDatesScns"] = db_obs_date_scns_dict
+
+        with open(out_json_file, 'w') as outfile:
+            json.dump(out_dict, outfile, indent=4, separators=(',', ': '), ensure_ascii=False)
+
+    def update_overview_file_paths(self, overviews_lst, replace_path_dict=None):
+        """
+        Update the list of overview file paths.
+        :param overviews_lst:
+        :param replace_path_dict:
+        """
+        if replace_path_dict is None:
+            return overviews_lst
+
+        eodd_utils = eodatadown.eodatadownutils.EODataDownUtils()
+        out_overviews_lst = list()
+        for overview_img in overviews_lst:
+            out_overviews_lst.append(eodd_utils.update_file_path(overview_img, replace_path_dict))
+        return out_overviews_lst
+
+    def import_obsdates_db(self, input_json_file, replace_path_dict=None):
+        """
+        This function imports from the database records from the specified input JSON file.
+
+        :param input_json_file: input JSON file with the records to be imported.
+        :param replace_path_dict: a dictionary of file paths to be updated, if None then ignored.
+
+        """
+        db_obsdate_records = list()
+        db_obsdatescn_records = list()
+
+        with open(input_json_file) as json_file_obj:
+            obsdate_db_dict = json.load(json_file_obj)
+            db_obs_dates_dict = obsdate_db_dict["EDDObsDates"]
+            db_obs_date_scns_dict = obsdate_db_dict["EDDObsDatesScns"]
+
+            for pid in db_obs_dates_dict:
+                db_obsdate_records.append(EDDObsDates(SensorID=db_obs_dates_dict[pid]["SensorID"],
+                                                      PlatformID=db_obs_dates_dict[pid]["PlatformID"],
+                                                      ObsDate=db_obs_dates_dict[pid]["ObsDate"],
+                                                      OverviewCreated=db_obs_dates_dict[pid]["OverviewCreated"],
+                                                      NeedUpdate=db_obs_dates_dict[pid]["NeedUpdate"],
+                                                      Invalid=db_obs_dates_dict[pid]["Invalid"],
+                                                      Overviews=self.update_overview_file_paths(
+                                                                                 db_obs_dates_dict[pid]["Overviews"])))
+            for pid in db_obs_dates_dict:
+                db_obsdatescn_records.append(EDDObsDatesScns(SensorID=db_obs_date_scns_dict[pid]["SensorID"],
+                                                             PlatformID=db_obs_date_scns_dict[pid]["PlatformID"],
+                                                             ObsDate=db_obs_date_scns_dict[pid]["ObsDate"],
+                                                             Scene_PID=db_obs_date_scns_dict[pid]["Scene_PID"]))
+
+        if len(db_obsdate_records) > 0:
+            db_engine = sqlalchemy.create_engine(self.db_info_obj.dbConn)
+            session_sqlalc = sqlalchemy.orm.sessionmaker(bind=db_engine)
+            ses = session_sqlalc()
+            ses.add_all(db_obsdate_records)
+            ses.commit()
+            if len(db_obsdatescn_records) > 0:
+                ses.add_all(db_obsdatescn_records)
+                ses.commit()
+            ses.close()

eodatadown/eodatadownsentinel1asf.py

@@ -1834,6 +1834,7 @@ def import_sensor_db(self, input_json_file, replace_path_dict=None):
             ses = session_sqlalc()
             ses.add_all(db_records)
             ses.commit()
+            ses.close()
 
     def create_gdal_gis_lyr(self, file_path, lyr_name, driver_name='GPKG', add_lyr=False):
         """

eodatadown/eodatadownsentinel2googsensor.py

@@ -2062,6 +2062,7 @@ def import_sensor_db(self, input_json_file, replace_path_dict=None):
             ses = session_sqlalc()
             ses.add_all(db_records)
             ses.commit()
+            ses.close()
 
     def create_gdal_gis_lyr(self, file_path, lyr_name, driver_name='GPKG', add_lyr=False):
         """

setup.py

@@ -37,7 +37,7 @@
 import os
 
 setup(name='EODataDown',
-    version='0.52.0',
+    version='0.53.0',
     description='A tool for automating Earth Observation Data Downloading.',
     author='Pete Bunting',
     author_email='pfb@aber.ac.uk',