diff --git a/src/temporal/t.register.local/t.register.local.py b/src/temporal/t.register.local/t.register.local.py index de7f28b0..2658e820 100644 --- a/src/temporal/t.register.local/t.register.local.py +++ b/src/temporal/t.register.local/t.register.local.py @@ -66,7 +66,7 @@ # %end # %option G_OPT_M_DIR -# % required: yes +# % required: no # % multiple: no # %end @@ -79,11 +79,10 @@ # % description: Comma separated list of files to register # %end -# %option GOPT_F_INPUT -# % key: file +# %option +# % key: register_file +# % type: string # % required: no -# % multiple: yes -# % key_desc: CSV-like file with files to import containing map_name,start,stop,semantic_label,file_name,metadata_json # % description: CSV-like file with files to import containing map_name,start,stop,semantic_label,file_name,metadata_json # %end @@ -180,7 +179,7 @@ # %option # % key: units # % type: string -# % required: yes +# % required: no # % multiple: no # % description: Units of the values in the input raster maps # %end @@ -193,10 +192,11 @@ # %rules # % excludes: file_pattern,files -# % excludes: file_pattern,file -# % excludes: files,file +# % excludes: file_pattern,register_file +# % excludes: files,register_file # % excludes: semantic_labels,semantic_label_pattern -# % required: file_pattern,files,file +# % required: file_pattern,files,register_file +# % required: input,register_file # % collective: file_pattern,suffix # %end @@ -598,7 +598,8 @@ def import_data_from_file_row( Link (import) external raster data and set relevant metadata Implemented as a sequence of modules to run Returns a line for registering the linked map in a STRDS - :param import_tuple: tuple containing Path to the raster data file to link/import, band/subdataset, semantic_label + :param file_row: delimited string containing Path to the raster data file to + link/import, band/subdataset, semantic_label :param metadata_dict: dictionary containing relevant metadata for the link/import :param modules_dict: Dictionary containing pyGRASS modules to run for link/import :return: str with one line for registering the linked raster map in an STRDS @@ -609,16 +610,22 @@ def import_data_from_file_row( map_name = register_row[0] start_time = register_row[1] - end_time = register_row[2] - semantic_label = register_row[3] + try: + end_time = register_row[2] + except Exception: + end_time = None + semantic_label = register_row[-2] file_path = Path(register_row[-1]) + if not file_path.exists(): + gs.warning(_("File <{}> not found.").format(str(file_path))) + return None mods = deepcopy(modules_dict) mods["import"].inputs.input = str(file_path) mods["import"].outputs.output = map_name mods["metadata"].inputs.map = map_name - mods["metadata"].inputs.units = metadata_dict["units"] - mods["metadata"].inputs.title = metadata_dict["long_name"] + mods["metadata"].inputs.units = metadata_dict.get("units", None) + mods["metadata"].inputs.title = metadata_dict.get("long_name", None) mods["metadata"].inputs.semantic_label = semantic_label mods["timestamp"].inputs.map = map_name mods["timestamp"].inputs.date = datetime.fromisoformat(start_time).strftime( @@ -629,8 +636,9 @@ def import_data_from_file_row( except: gs.warning(_("Cannot register file <{}>").format(str(file_path))) return None - - return f"{map_name},{start_time},{end_time},{semantic_label}" + if end_time: + return f"{map_name},{start_time},{end_time},{semantic_label}" + return f"{map_name},{start_time},{semantic_label}" def main(): @@ -686,15 +694,13 @@ def main(): for raster_file in files_input if (Path(options["input"]) / raster_file).exists() ] - else: - # Register file mode - files_input = Path(files_input[0]) - try: - ds = gdal.Open(str(files_input)) - del ds - except Exception: - register_from_file = True - files_input = files_input.read_text(encoding="UTF8").strip().split() + elif options["register_file"]: + # Register file mode + input_file = Path(options["register_file"]) + register_from_file = True + files_input = input_file.read_text(encoding="UTF8").strip().split("\n") + print(files_input) + raster_files = None else: # Directory mode @@ -794,14 +800,21 @@ def main(): input_option_dict=options, ) - # Pre-define kwargs for import-function - run_import = partial( - import_data, - metadata_dict=options, - modules_dict=modules, - second_is_stop=flags["s"], - ) - + # Pre-define kwargs for import-function + run_import = partial( + import_data, + metadata_dict=options, + modules_dict=modules, + second_is_stop=flags["s"], + ) + else: + # Pre-define kwargs for import-function + run_import = partial( + import_data_from_file_row, + metadata_dict=options, + modules_dict=modules, + ) + import_tuples = [files_input] # Get GRASS GIS environment info options.update(dict(gs.gisenv())) @@ -819,15 +832,10 @@ def main(): import_tuples = pool.map(match_semantic_labels, raster_files) gs.verbose(_("Semantic labels matched")) - # Import / Link raster maps into mapset - with Pool(processes=nprocs) as pool: - register_string = pool.map(run_import, chain(*import_tuples)) - gs.verbose(_("Maps imported")) - else: - # Import / Link raster maps into mapset - with Pool(processes=nprocs) as pool: - register_string = pool.map(run_import, chain(*import_tuples)) - gs.verbose(_("Maps imported")) + # Import / Link raster maps into mapset + with Pool(processes=nprocs) as pool: + register_string = pool.map(run_import, chain(*import_tuples)) + gs.verbose(_("Maps imported")) # Register imported maps in STRDS using register file map_file = gs.tempfile() diff --git a/src/temporal/t.register.local/testsuite/test_t_register_local.py b/src/temporal/t.register.local/testsuite/test_t_register_local.py new file mode 100644 index 00000000..fc0979b9 --- /dev/null +++ b/src/temporal/t.register.local/testsuite/test_t_register_local.py @@ -0,0 +1,317 @@ +"""Test t.rast.reclass + +(C) 2022 by the GRASS GIS Development Team +This program is free software under the GNU General Public +License (>=v2). Read the file COPYING that comes with GRASS +for details. + +:authors: Stefan Blumentrath +""" + +import os + +import grass.temporal as tgis +from grass.gunittest.case import TestCase +from grass.gunittest.gmodules import SimpleModule + + +class TestRegistrationLocal(TestCase): + @classmethod + def setUpClass(cls): + """Initiate the temporal GIS and set the region""" + os.putenv("GRASS_OVERWRITE", "1") + tgis.init() + cls.use_temp_region() + cls.runModule("g.region", s=0, n=80, w=0, e=120, b=0, t=50, res=10, res3=10) + cls.runModule("r.mapcalc", expression="a1 = 100", overwrite=True) + cls.runModule("r.mapcalc", expression="a2 = 200", overwrite=True) + cls.runModule("r.mapcalc", expression="a3 = 300", overwrite=True) + cls.runModule("r.mapcalc", expression="a4 = 400", overwrite=True) + cls.runModule("r.mapcalc", expression="a5 = 500", overwrite=True) + cls.runModule("r.mapcalc", expression="a6 = 600", overwrite=True) + cls.runModule("r.mapcalc", expression="a7 = null()", overwrite=True) + + cls.runModule( + "t.create", + type="strds", + temporaltype="absolute", + output="A", + title="A test", + description="A test", + overwrite=True, + ) + + cls.runModule( + "t.register", + flags="i", + type="raster", + input="A", + maps="a1,a2,a3,a4,a5,a6,a7", + start="2001-01-15 12:05:45", + increment="14 days", + overwrite=True, + ) + + @classmethod + def tearDownClass(cls): + """Remove the temporary region and data""" + cls.del_temp_region() + cls.runModule("t.remove", flags="df", type="strds", inputs="A") + + def tearDown(self): + """Remove generated data""" + self.runModule("t.remove", flags="df", type="strds", inputs="B") + + def test_register_maps_from_file(self): + """Reclassify and register also maps with only NoData""" + self.assertModule( + "t.register.local", + flags="n", + input="./", + output="A", + file="./register.txt", + title="B", + description="B", + nprocs=2, + units="m2", + semantic_label="label_a", + overwrite=True, + ) + info = SimpleModule( + "t.info", + flags="g", + input="B", + ).run() + print(info.outputs.stdout) + + list_mod1 = SimpleModule( + "g.list", + type="raster", + pattern="*_label_a", + ).run() + + def test_reclass_no_null_maps(self): + """Reclassify and not register maps with only NoData""" + self.assertModule( + "t.rast.reclass", + input="A", + output="B", + rules="-", + title="B", + description="B", + nprocs=1, + stdin_="100 = 1\n200 = 2\n* = NULL", + overwrite=True, + semantic_label="label_b", + ) + info = SimpleModule( + "t.info", + flags="g", + input="B", + ).run() + print(info.outputs.stdout) + + list_mod2 = SimpleModule( + "g.list", + type="raster", + pattern="*_label_b", + ).run() + + def test_reclass_extend_strds(self): + """Reclassify and not register maps with only NoData""" + self.assertModule( + "t.rast.reclass", + input="A", + output="B", + rules="-", + title="B", + description="B", + nprocs=1, + stdin_="100 = 1\n200 = 2\n* = NULL", + overwrite=True, + semantic_label="label_b", + ) + self.assertModule( + "t.rast.reclass", + flags="e", + input="A", + output="B", + rules="-", + nprocs=1, + stdin_="300 = 3\n300 = 4\n* = NULL", + overwrite=True, + semantic_label="label_c", + ) + info = SimpleModule( + "t.info", + flags="g", + input="B", + ).run() + print(info.outputs.stdout) + + # tinfo_string = """start_time='2001-01-15 00:00:00' + # end_time='2001-04-25 00:00:00' + # granularity='2 days' + # aggregation_type=average + # number_of_maps=50 + # map_time=interval + # min_min=100.0 + # min_max=600.0 + # max_min=100.0 + # max_max=600.0""" + + # info = SimpleModule("t.info", flags="g", input="B") + # # info.run() + # # print info.outputs.stdout + # self.assertModuleKeyValue( + # module=info, reference=tinfo_string, precision=2, sep="=" + # ) + + # def test_aggregation_1month(self): + # """Aggregation one month""" + # self.assertModule( + # "t.rast.aggregate", + # input="A", + # output="B", + # basename="b", + # granularity="1 months", + # method="maximum", + # sampling=["contains"], + # file_limit=0, + # nprocs=3, + # ) + + # tinfo_string = """start_time='2001-01-01 00:00:00' + # end_time='2001-04-01 00:00:00' + # granularity='1 month' + # map_time=interval + # aggregation_type=maximum + # number_of_maps=3 + # min_min=100.0 + # min_max=500.0 + # max_min=100.0 + # max_max=500.0""" + + # info = SimpleModule("t.info", flags="g", input="B") + # # info.run() + # # print info.outputs.stdout + # self.assertModuleKeyValue( + # module=info, reference=tinfo_string, precision=2, sep="=" + # ) + + # # Check the map names are correct + # lister = SimpleModule("t.rast.list", input="B", columns="name", flags="u") + # self.runModule(lister) + # # print lister.outputs.stdout + # maps = ( + # "b_2001_01" + # + os.linesep + # + "b_2001_02" + # + os.linesep + # + "b_2001_03" + # + os.linesep + # ) + # self.assertEqual(maps, lister.outputs.stdout) + + # def test_aggregation_1month_time(self): + # """Aggregation one month time suffix""" + # self.assertModule( + # "t.rast.aggregate", + # input="A", + # output="B", + # basename="b", + # granularity="1 months", + # method="maximum", + # sampling=["contains"], + # file_limit=0, + # nprocs=3, + # suffix="time", + # ) + # self.assertRasterExists("b_2001_01_01T00_00_00") + + # def test_aggregation_2months(self): + # """Aggregation two month""" + # self.assertModule( + # "t.rast.aggregate", + # input="A", + # output="B", + # basename="b", + # granularity="2 months", + # method="minimum", + # sampling=["contains"], + # nprocs=4, + # offset=10, + # suffix="num%02", + # ) + + # tinfo_string = """start_time='2001-01-01 00:00:00' + # end_time='2001-05-01 00:00:00' + # granularity='2 months' + # map_time=interval + # aggregation_type=minimum + # number_of_maps=2 + # min_min=100.0 + # min_max=500.0 + # max_min=100.0 + # max_max=500.0""" + + # info = SimpleModule("t.info", flags="g", input="B") + # # info.run() + # # print info.outputs.stdout + # self.assertModuleKeyValue( + # module=info, reference=tinfo_string, precision=2, sep="=" + # ) + + # # Check the map names are correct + # lister = SimpleModule("t.rast.list", input="B", columns="name", flags="u") + # self.runModule(lister) + # # print lister.outputs.stdout + # maps = "b_11" + os.linesep + "b_12" + os.linesep + # self.assertEqual(maps, lister.outputs.stdout) + + # def test_aggregation_3months(self): + # """Aggregation three month""" + # self.assertModule( + # "t.rast.aggregate", + # input="A", + # output="B", + # basename="b", + # granularity="3 months", + # method="sum", + # sampling=["contains"], + # file_limit=0, + # nprocs=9, + # offset=100, + # suffix="num%03", + # ) + + # tinfo_string = """start_time='2001-01-01 00:00:00' + # end_time='2001-04-01 00:00:00' + # granularity='3 months' + # map_time=interval + # aggregation_type=sum + # number_of_maps=1 + # min_min=1500.0 + # min_max=1500.0 + # max_min=1500.0 + # max_max=1500.0""" + + # info = SimpleModule("t.info", flags="g", input="B") + # # info.run() + # # print info.outputs.stdout + # self.assertModuleKeyValue( + # module=info, reference=tinfo_string, precision=2, sep="=" + # ) + + # # Check the map names are correct + # lister = SimpleModule("t.rast.list", input="B", columns="name", flags="u") + # self.runModule(lister) + # # print lister.outputs.stdout + # maps = "b_101" + os.linesep + # self.assertEqual(maps, lister.outputs.stdout) + + +if __name__ == "__main__": + from grass.gunittest.main import test + + test()