#81: Make sure that data can be shifted given WGS84 coordinates as re…

…ference point (#82)
Deltares-research · Apr 19, 2024 · f5ec8e0 · f5ec8e0
1 parent 4c68484
commit f5ec8e0
Show file tree

Hide file tree

Showing 26 changed files with 2,175 additions and 162 deletions.
diff --git a/README.md b/README.md
@@ -5,22 +5,29 @@
 # Contents
 
 - [Contents](#contents)
-- [D-HYDRO netCDF output to glTF converter](#d-hydro-netcdf-output-to-gltf-converter)
+- [Disclaimer](#disclaimer)
+- [NetCDF to glTF converter](#netcdf-to-gltf-converter)
   - [Why use glTF](#why-use-gltf)
+  - [Supported models](#supported-models)
 - [User guide](#user-guide)
   - [Requirements](#requirements)
   - [Installation](#installation)
   - [Usage](#usage)
   - [Configuration file](#configuration-file)
+  - [Logging](#logging)
   - [View results](#view-results)
 - [Methodology](#methodology)
 - [Limitations](#limitations)
 - [Contributing](#contributing)
 - [Acknowledgments](#acknowledgments)
 
-# D-HYDRO netCDF output to glTF converter
+# Disclaimer
 
-This is a tool that converts D-HYDRO map output results that are stored in the netCDF file format to the glTF file format. The goal is to allow users who work with D-HYDRO results to view their data in 3D renderers using the glTF format.
+This tool is a proof-of-concept, which means it has been tested in a limited scope and may not handle all real-world cases effectively. As many cases have not been validated, conversion may fail in certain scenarios.
+
+# NetCDF to glTF converter
+
+This is a tool that converts hydrodynamical model output results that are stored in the netCDF file format to the glTF file format. The goal is to allow users to view their data in 3D renderers using the glTF format.
 
 <p align="center">
   <img src="docs/readme/img/example.gif" alt="animated" />
@@ -30,6 +37,13 @@ This is a tool that converts D-HYDRO map output results that are stored in the n
 [glTF (GL Transmission Format)](https://www.khronos.org/gltf/) is an open-standard file format developed by [The Khronos Group](https://www.khronos.org/). The glTF file format is used for 3D scenes and models designed for efficient transmission and loading of 3D content on the web and other real-time applications. This file format can store geometry, materials, textures, animations, and other scene data.
 glTF is used in a variety of industries and applications, including gaming, virtual and augmented reality, education, and more. It is particularly well-suited for web-based applications, as it allows 3D content to be easily and efficiently delivered over the internet, and can be rendered in real-time on a wide range of devices. 
 
+## Supported models
+
+Currently, this tool supports the conversion of the following hydrodynamical models:
+
+- D-HYDRO (map)
+- XBEACH
+
 # User guide
 ## Requirements
 - Python >=3.9, <3.12
@@ -82,7 +96,14 @@ These steps will ensure that the converter is installed within a virtual environ
 
 - `times_per_frame`: An integer value specifying the number of time steps to be included in each frame of the glTF animation. This option is useful if you want to adjust the time resolution of the animation.
 
-- `shift_coordinates`: A boolean value indicating whether to shift the coordinates of the data during conversion. When set to `true`, the converter will shift the coordinates such that the smallest x and y become the origin (0,0).
+- `shift_coordinates` (optional): A value indicating how to shift the coordinates of the data during conversion. When set to `min`, the converter will shift the coordinates such that the smallest x and y become the origin (0,0); variable values remain unchanged. It is also possible to provide custom shift values for the x- and y-coordinates and the variable values (z-coordinates):
+
+  - `crs_transformation` (optional): The configuration settings for transforming the provided shift values from one coordinate system to another. The target coordinate system should be the coordinate system of the model.
+    - `source_epsg`: EPSG code of the source coordinate system.
+    - `target_epsg`: EPSG code of the target coordinate system.
+  - `shift_x`: A floating value containing the value that should be subtracted from all x-coordinates.
+   - `shift_y`: A floating value containing the value that should be subtracted from all y-coordinates.
+   - `shift_z`: A floating value containing the value that should be subtracted from all variable values (z-coordinates).
 
 - `scale_horizontal`: A floating value indicating the scale factor for the x- and y-coordinates. It determines the scaling of the converted geometry. A value of 1.0 results in the original geometry size.
 
@@ -109,28 +130,28 @@ These steps will ensure that the converter is installed within a virtual environ
 {
    "file_version":"0.1.0",
    "model_type": "D-HYDRO",
-   "time_index_start":50,
-   "time_index_end":100,
-   "times_per_frame":3,
-   "shift_coordinates":true,
-   "scale_horizontal":0.5,
-   "scale_vertical":0.5,
+   "time_index_start": 50,
+   "time_index_end": 100,
+   "times_per_frame": 3,
+   "shift_coordinates": "min",
+   "scale_horizontal": 0.5,
+   "scale_vertical": 0.5,
    "variables":[
       {
-         "name":"Mesh2d_waterdepth",
-         "color":[0.372, 0.635, 0.8, 1.0],
-         "metallic_factor":0.0,
-         "roughness_factor":0.15,
-         "use_threshold":false,
-         "threshold_height":0.01,
-         "threshold_color":[1.0, 1.0, 1.0, 1.0]
+         "name": "Mesh2d_waterdepth",
+         "color": [0.372, 0.635, 0.8, 1.0],
+         "metallic_factor": 0.0,
+         "roughness_factor": 0.15,
+         "use_threshold": false,
+         "threshold_height": 0.01,
+         "threshold_color": [1.0, 1.0, 1.0, 1.0]
       },
       {
-         "name":"Mesh2d_s1",
-         "color":[0.686, 0.831, 0.937, 1.0],
-         "metallic_factor":0.0,
-         "roughness_factor":0.15,
-         "use_threshold":false
+         "name": "Mesh2d_s1",
+         "color": [0.686, 0.831, 0.937, 1.0],
+         "metallic_factor": 0.0,
+         "roughness_factor": 0.15,
+         "use_threshold": false
       }
    ]
 }
@@ -140,14 +161,17 @@ In the above example, we render two variables from a D-HYDRO output map netCDF f
 
 For the `Mesh2d_waterdepth` variable, an additional threshold mesh is rendered at a height of 0.01. Each mesh is assigned its own color, specified by the normalized red, green, blue and alpha (RGBA) values.
 
+## Logging
+During conversion, a log file is written to the output folder (folder of the resulting glTF file) with the name: `gltf_converter_<date>_<time>_<gltf-name>.log`
+
 ## View results
  Several glTF viewers exist that can be used to view the produced glTF file. Simply drag and drop the file, and the glTF file will be rendered.
  * [glTF Sample Viewer](https://github.khronos.org/glTF-Sample-Viewer-Release/)
  * [Babylon.js Sandbox](https://sandbox.babylonjs.com/)
 
 # Methodology
 The converter operates through the following steps:
-1. If specified in the configuration file, the x- and y-coordinates are shifted such that the smallest x and y become the origin (0,0).
+1. If specified in the configuration file, the x- and y-coordinates and possibly z-coordinates (variable values) are shifted such that the dataset contains an origin point.
 2. If specified in the configuration file, the geometry is scaled with the desired scaling factor.
 3. The 2D grid from the netCDF file is triangulated, allowing it to be passed to glTF. In order to render a mesh, glTF requires a geometry definition that consists of triangles.
 4. For each specified variable in the configuration file:
@@ -188,7 +212,7 @@ If you encounter any issues or have good ideas for this project please [create a
 # Acknowledgments
 [Connec2](https://connec2.nl/) is a company specialized in cross reality (XR) technology that guided us to setup this project. 
 
-This tool was developed as part of the [D-HYDRO GUI Visualisatie & Cloud project](https://tkideltatechnologie.invoermodule.nl/project/d-hydro-gui-visualisatie-cloud/), which was funded by the fifth Top consortium for Knowledge and Innovation (TKI) programme.
+This tool was initially developed as part of the [D-HYDRO GUI Visualisatie & Cloud project](https://tkideltatechnologie.invoermodule.nl/project/d-hydro-gui-visualisatie-cloud/), which was funded by the fifth Top consortium for Knowledge and Innovation (TKI) programme.
 
 
 

diff --git a/netcdf_to_gltf_converter/config.py b/netcdf_to_gltf_converter/config.py
@@ -1,7 +1,7 @@
 import logging
 from abc import ABC
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Type, TypeVar
+from typing import Any, Dict, List, Optional, Type, TypeVar, Union
 
 from packaging.version import Version
 from pydantic import BaseModel as PydanticBaseModel
@@ -99,6 +99,36 @@ class ModelType(StrEnum):
     XBEACH = "XBEACH"
     """Output from an XBEACH model (regular grid)."""
 
+class ShiftType(StrEnum):
+    """The method to shift the coordinates."""
+
+    MIN = "min"
+    """The smalles x- and -y coordinate become the origin (0,0,z)."""
+
+class CrsTransformation(BaseModel):
+    """The configuration settings for transforming the coordinates."""
+
+    source_epsg: int
+    """int: EPSG code of the source coordinate system."""
+
+    target_epsg: int
+    """int: EPSG code of the target coordinate system."""
+
+class CrsShifting(BaseModel):
+    """The configuration settings for shifting the coordinates."""
+
+    crs_transformation: Optional[CrsTransformation]
+    """Optional[CrsTransformation]: The configuration settings for transforming the provided shift values from one coordinate system to another (e.g. from WGS84 to the model CRS)."""
+
+    shift_x: float
+    """float: Value to shift the x-coordinates with. All x-coordinates will be subtracted with this value."""
+
+    shift_y: float
+    """float: Value to shift the y-coordinates with. All y-coordinates will be subtracted with this value."""
+
+    shift_z: float
+    """float: Value to shift the variables values (z-coordinates) with. All variable values will be subtracted with this value."""
+
 class Variable(BaseModel):
     """Configuration properties of a variable."""
 
@@ -180,8 +210,8 @@ class Config(AbstractJsonConfigFile, AbstractFileVersionFile):
     times_per_frame: int
     """int: The number of time steps per animation frame."""
 
-    shift_coordinates: bool
-    """bool: Whether or not to shift the x- and y-coordinates, such that the smallest x and y become the origin (0,0)."""
+    shift_coordinates: Optional[Union[ShiftType, CrsShifting]]
+    """Optional[Union[ShiftType, CrsShifting]]: The options how to shift the x-, y- and optionally z-coordinates. Typically used to create a reference point, such that an x-, y- and z-coordinate become the origin (0,0,0)."""
 
     scale_horizontal: float
     """float: The horizontal scaling factor of the mesh coordinates compared to the coordinates from file."""

diff --git a/netcdf_to_gltf_converter/converter.py b/netcdf_to_gltf_converter/converter.py
@@ -1,3 +1,5 @@
+import logging
+from datetime import datetime
 from pathlib import Path
 
 from netcdf_to_gltf_converter.config import Config
@@ -32,7 +34,27 @@ def __init__(self, netcdf: Path, gltf: Path, config: Path) -> None:
 
         self._importer = Importer()
         self._exporter = Exporter()
+
+        self._configure_logging()
 
+    def _configure_logging(self) -> None:
+        self._reset_logger()
+
+        time_stamp = datetime.now().strftime("%y%m%d_%H%M%S")
+        log_file = self._gltf.parent / f"gltf_converter_{time_stamp}_{self._gltf.stem}.log"
+
+        logging.basicConfig(
+            level=logging.DEBUG,
+            filename=log_file,
+            filemode="w",
+            format="%(asctime)s %(levelname)-8s %(filename)-20s %(funcName)-20s %(message)s",
+            datefmt="%H:%M:%S",
+        )
+
+    def _reset_logger(self):
+        for handler in logging.root.handlers[:]:
+            logging.root.removeHandler(handler)
+
     def run(self):
         """Run the conversion."""
 

diff --git a/netcdf_to_gltf_converter/data/mesh.py b/netcdf_to_gltf_converter/data/mesh.py
@@ -3,7 +3,8 @@
 import numpy as np
 
 from netcdf_to_gltf_converter.typing.custom_types import Color
-from netcdf_to_gltf_converter.utils.arrays import float32_array, validate_2d_array
+from netcdf_to_gltf_converter.utils.arrays import (float32_array,
+                                                   validate_2d_array)
 
 
 class MeshAttributes:

diff --git a/netcdf_to_gltf_converter/data/vector.py b/netcdf_to_gltf_converter/data/vector.py
@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Vec3:
+    """Data class representing a vector with 3 values."""
+
+    x: float = 0.0
+    """float: x. Defaults to 0.0."""
+
+    y: float = 0.0
+    """float: y. Defaults to 0.0."""
+
+    z: float = 0.0
+    """float: z. Defaults to 0.0."""
diff --git a/netcdf_to_gltf_converter/gltf/builder.py b/netcdf_to_gltf_converter/gltf/builder.py
@@ -2,34 +2,15 @@
 from typing import Any, List
 
 import numpy as np
-from pygltflib import (
-    ANIM_LINEAR,
-    ARRAY_BUFFER,
-    DATA_URI_HEADER,
-    ELEMENT_ARRAY_BUFFER,
-    FLOAT,
-    GLTF2,
-    SCALAR,
-    UNSIGNED_INT,
-    VEC3,
-    VEC4,
-    Accessor,
-    Animation,
-    AnimationChannel,
-    AnimationChannelTarget,
-    AnimationSampler,
-    Attributes,
-    Buffer,
-    BufferView,
-    Material,
-    Mesh,
-    Node,
-    PbrMetallicRoughness,
-    Primitive,
-    Scene,
-)
-
-from netcdf_to_gltf_converter.data.mesh import MeshAttributes, TriangularMesh
+from pygltflib import (ANIM_LINEAR, ARRAY_BUFFER, DATA_URI_HEADER,
+                       ELEMENT_ARRAY_BUFFER, FLOAT, GLTF2, SCALAR,
+                       UNSIGNED_INT, VEC3, VEC4, Accessor, Animation,
+                       AnimationChannel, AnimationChannelTarget,
+                       AnimationSampler, Attributes, Buffer, BufferView,
+                       Material, Mesh, Node, PbrMetallicRoughness, Primitive,
+                       Scene)
+
+from netcdf_to_gltf_converter.data.mesh import TriangularMesh
 from netcdf_to_gltf_converter.utils.arrays import float32_array
 
 PADDING_BYTE = b"\x00"