From c4cf6dc081bee5ea7631369c5ae432aa67ea3e0d Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 25 Jan 2024 09:39:44 -0800 Subject: [PATCH 1/5] update configuration generation with palette option --- CHANGELOG.md | 2 ++ podaac/tig/generate_hitide_config.py | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4618d50..0d8a258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - update tig to be able to run with arm architecture - ** Add image uploading ** - update tig to upload tig image to ecr +- ** Add palette support to generating configuration ** + - [issue/44] (https://github.com/podaac/hitide/issues/44): Add support to palette in csv in generating configuration ### Changed ### Deprecated ### Removed diff --git a/podaac/tig/generate_hitide_config.py b/podaac/tig/generate_hitide_config.py index e846485..a646636 100644 --- a/podaac/tig/generate_hitide_config.py +++ b/podaac/tig/generate_hitide_config.py @@ -104,9 +104,9 @@ def generate_hitide_config(granule, dataset_id, include_image_variables, longitu 's2': '0:*,*:*' } - vars_min_max = {} + vars_data = {} if include_image_variables: - vars_min_max = read_min_max_csv(include_image_variables) + vars_data = read_min_max_csv(include_image_variables) with nc.Dataset(granule, 'r') as dataset: # pylint: disable=no-member @@ -115,27 +115,31 @@ def generate_hitide_config(granule, dataset_id, include_image_variables, longitu try: for data_var in data_var_names: - if vars_min_max and data_var in vars_min_max: + if vars_data and data_var in vars_data: variable = dataset[data_var] units = variable.units if 'units' in variable.ncattrs() else '' long_name = variable.long_name if 'long_name' in variable.ncattrs() else '' - if data_var in vars_min_max: - min_max = vars_min_max[data_var] - min_val = float(min_max['min']) - max_val = float(min_max['max']) + palette = 'paletteMedspirationIndexed' + if data_var in vars_data: + min_val = float(vars_data[data_var]['min']) + max_val = float(vars_data[data_var]['max']) + palette = vars_data[data_var].get('palette') else: min_val = variable.valid_min if 'valid_min' in variable.ncattrs() else '' max_val = variable.valid_max if 'valid_max' in variable.ncattrs() else '' + if not palette: + palette = 'paletteMedspirationIndexed' + dataset_config['imgVariables'].append({ 'id': data_var, 'title': long_name, 'units': units, 'min': min_val, 'max': max_val, - 'palette': 'paletteMedspirationIndexed' + 'palette': palette }) except Exception as ex: # pylint: disable=broad-exception-caught From 5c6a11ff574649e4e6d5be2d8c76fd0ab748564f Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Mon, 29 Jan 2024 12:12:16 -0800 Subject: [PATCH 2/5] update to make fill_missing and ppd optional for each variable --- CHANGELOG.md | 2 +- README.md | 7 +++++++ example_vars.csv | 6 +++--- podaac/tig/generate_hitide_config.py | 18 ++++++++++++++++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d8a258..584bcfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ** Add image uploading ** - update tig to upload tig image to ecr - ** Add palette support to generating configuration ** - - [issue/44] (https://github.com/podaac/hitide/issues/44): Add support to palette in csv in generating configuration + - [issue/44] (https://github.com/podaac/hitide/issues/44): Add support to palette, ppd, fill_missing in csv in generating configuration ### Changed ### Deprecated ### Removed diff --git a/README.md b/README.md index c052c55..427103a 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,13 @@ latitude: latitude variable include the group if they're in a group defaults to time: time variable include the group if they're in a group defaults to time footprint_strategy: strategy to generate footprint will default to None options should be ["periodic", "linestring", "polar", "swot_linestring", "polarsides", "smap"] +### CSV Columns + +variable: name of variable +min: min value for variable +max: max value for variable +fill_missing (optional): if the generated images have missing pixel in images most likely resolution is to big, either lower resolution or we can fill in the pixels with surrounding pixel +ppd (optional): resolution of the variable, must be an integer ## How to load and use tig module diff --git a/example_vars.csv b/example_vars.csv index 0b534ca..fff970c 100644 --- a/example_vars.csv +++ b/example_vars.csv @@ -1,3 +1,3 @@ -variable,min,max -data_01/ku/ssha,-0.2,0.2 -data_01/ku/swh_ocean,0,30 +variable,min,max,fill_missing,ppd +data_01/ku/ssha,-0.2,0.2,TRUE,20 +data_01/ku/swh_ocean,0,30,FALSE,15 \ No newline at end of file diff --git a/podaac/tig/generate_hitide_config.py b/podaac/tig/generate_hitide_config.py index a646636..cc9176d 100644 --- a/podaac/tig/generate_hitide_config.py +++ b/podaac/tig/generate_hitide_config.py @@ -122,10 +122,15 @@ def generate_hitide_config(granule, dataset_id, include_image_variables, longitu long_name = variable.long_name if 'long_name' in variable.ncattrs() else '' palette = 'paletteMedspirationIndexed' + fill_missing = False + ppd = 16 + if data_var in vars_data: min_val = float(vars_data[data_var]['min']) max_val = float(vars_data[data_var]['max']) palette = vars_data[data_var].get('palette') + fill_missing = vars_data[data_var].get('fill_missing', False) + ppd = vars_data[data_var].get('ppd', 16) else: min_val = variable.valid_min if 'valid_min' in variable.ncattrs() else '' max_val = variable.valid_max if 'valid_max' in variable.ncattrs() else '' @@ -133,14 +138,23 @@ def generate_hitide_config(granule, dataset_id, include_image_variables, longitu if not palette: palette = 'paletteMedspirationIndexed' - dataset_config['imgVariables'].append({ + dataset_dict = { 'id': data_var, 'title': long_name, 'units': units, 'min': min_val, 'max': max_val, 'palette': palette - }) + } + + if fill_missing: + fill_missing = fill_missing.lower().strip() + dataset_dict['fill_missing'] = fill_missing == "true" + + if ppd != 16 and ppd.isdigit(): + dataset_dict['ppd'] = int(ppd) + + dataset_config['imgVariables'].append(dataset_dict) except Exception as ex: # pylint: disable=broad-exception-caught print(f"Error: Failed on variable {data_var}, exception: " + str(ex)) From 7768c30b2a2078a6f86a2c3eb94dd21f89e871a0 Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Mon, 29 Jan 2024 13:58:26 -0800 Subject: [PATCH 3/5] fix pytoml version for develop --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 77c631e..7ea4821 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "podaac-tig" -version = "0.9.0-rc.0" +version = "0.9.0-alpha.1" description = "Tool for Image Generation (TIG)" authors = ["podaac-tva "] license = "Apache-2.0" From 09a7e6e3116e932b7152a3d06c66bcdcfe59b50b Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Tue, 30 Jan 2024 12:26:52 -0800 Subject: [PATCH 4/5] show example of palette in csv --- README.md | 1 + example_vars.csv | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 427103a..2289a7c 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ footprint_strategy: strategy to generate footprint will default to None options variable: name of variable min: min value for variable max: max value for variable +palette (optional): the palette to be used for the variable fill_missing (optional): if the generated images have missing pixel in images most likely resolution is to big, either lower resolution or we can fill in the pixels with surrounding pixel ppd (optional): resolution of the variable, must be an integer diff --git a/example_vars.csv b/example_vars.csv index fff970c..389042e 100644 --- a/example_vars.csv +++ b/example_vars.csv @@ -1,3 +1,3 @@ -variable,min,max,fill_missing,ppd -data_01/ku/ssha,-0.2,0.2,TRUE,20 -data_01/ku/swh_ocean,0,30,FALSE,15 \ No newline at end of file +variable,min,max,palette,fill_missing,ppd +data_01/ku/ssha,-0.2,0.2,paletteMedspirationIndexed,TRUE,20 +data_01/ku/swh_ocean,0,30,palette_AQUARIUS,FALSE,15 \ No newline at end of file From 0149ff74a24a3a307c5035c5c5d5197b7ddcaa1e Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Tue, 30 Jan 2024 12:50:45 -0800 Subject: [PATCH 5/5] fix version to 0.9.0-alpha.4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7ea4821..76237c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "podaac-tig" -version = "0.9.0-alpha.1" +version = "0.9.0-alpha.4" description = "Tool for Image Generation (TIG)" authors = ["podaac-tva "] license = "Apache-2.0"