Missing or Invalid metadata placeholders fix.

CMakeLists.txt

@@ -51,13 +51,15 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-Wextra")
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr")
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --Werror all-warnings")
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --extended-lambda")
-if (NOT DEFINED ENV{CUDAARCHS})
-    message(STATUS "CUDAARCHS not set, defaulting")
-    # https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html
-    if (CMAKE_VERSION VERSION_LESS "3.24")
-        set(CMAKE_CUDA_ARCHITECTURES 60 CACHE STRING "CUDA architectures" FORCE)
-    else()
-        set(CMAKE_CUDA_ARCHITECTURES native CACHE STRING "CUDA architectures" FORCE)
+if (NOT CMAKE_CUDA_ARCHITECTURES)
+    if (NOT DEFINED ENV{CUDAARCHS})
+        message(STATUS "CUDAARCHS not set, defaulting")
+        # https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html
+        if (CMAKE_VERSION VERSION_LESS "3.24")
+            set(CMAKE_CUDA_ARCHITECTURES 60 CACHE STRING "CUDA architectures" FORCE)
+        else()
+            set(CMAKE_CUDA_ARCHITECTURES native CACHE STRING "CUDA architectures" FORCE)
+        endif()
     endif()
 endif()
 message(STATUS "CUDA binary code will be generated for the following architecture(s) - ${CMAKE_CUDA_ARCHITECTURES}")

RELEASE.md

@@ -1,3 +1,40 @@
+# Release 0.3.1
+
+## Breaking changes
+
+## Known Caveats
+* Only IMS and IMP product processing
+* Azimuth compression windowing is yet to be done - https://github.com/cgi-estonia-space/asar-focus/issues/2
+* Processing speed (Vr) and Doppler centroid changes in azimuth direction yet to be done - https://github.com/cgi-estonia-space/asar-focus/issues/2
+* Packets' ISP sensing time handling might not be 100% correct
+  * It is observed for the reference products that it calculates new ISP sensing times based on PRI
+  * Therefore products by this processor differ in sensing start/stop and first/last line times (always inside the specified sensing filter)
+  * Best knowledge/effort basis changes has been implemented - https://github.com/cgi-estonia-space/asar-focus/issues/17 and https://github.com/cgi-estonia-space/asar-focus/issues/16
+* Various metadata fields needs further work (Some were adressed during this release)
+* Final results' scaling is yet to be determined, currently it is not matching exactly the reference processor
+  * With the current experience/knowledge there is a "best guess" implemented
+
+## Major Features and Improvements
+* Following metadata fields implemented:
+  * SQ ADS missing fields (output statistics, thresholds)
+  * REL_ORBIT
+  * ABS_ORBIT
+  * PHASE, CYCLE
+  * LEVEL 0 PRODUCT
+  * LEAP_XYZ
+  * CLOCK_STEP, UTC_SBT_TIME, SAT_BINARY_TIME
+
+## Bug Fixes and Other Changes
+* `CMAKE_CUDA_ARCHITECTURES` is not ignored anymore when generating via cmake
+* SUM consists of development 101
+
+## Thanks to our Contributors
+
+Kajal Haria, Fabiano Costantini from Telespazio UK\
+Sabrina Pinori, Marco Galli from SERCO\
+Andrea Recchia from aresys
+
+
 # Release 0.3.0
 
 ## Breaking changes

VERSION

@@ -1,7 +1,7 @@
 #define VERSION_MAJOR 0
 #define VERSION_MINOR 3
-#define VERSION_PATCH 0
+#define VERSION_PATCH 1
 
 #define STRINGIZE(s) #s
 #define STRINGIZE_VAL(s) STRINGIZE(s)
-#define VERSION_STRING STRINGIZE_VAL(VERSION_MAJOR) "." STRINGIZE_VAL(VERSION_MINOR) "." STRINGIZE_VAL(VERSION_PATCH)
+#define VERSION_STRING STRINGIZE_VAL(VERSION_MAJOR) "." STRINGIZE_VAL(VERSION_MINOR) "." STRINGIZE_VAL(VERSION_PATCH)

cuda_util/device_padded_image.cu

@@ -311,7 +311,7 @@ namespace {
 
     }
 
-    cufftComplex DevicePaddedImage::CalcStdDev(float i_mean, float q_mean, size_t total_samples)
+    cufftComplex DevicePaddedImage::CalcStdDev(float i_mean, float q_mean, size_t total_samples) const
     {
         size_t n_blocks = 50;
         size_t byte_size = n_blocks * sizeof(float);
@@ -342,6 +342,9 @@ namespace {
         r.x = std::accumulate(h_i_sum.begin(), h_i_sum.end(), 0.0);
         r.y = std::accumulate(h_q_sum.begin(), h_q_sum.end(), 0.0);
 
+        if (total_samples == 0) {
+            total_samples = x_size_ * y_size_;
+        }
         // variance
         r.x /= total_samples;
         r.y /= total_samples;
@@ -353,7 +356,7 @@ namespace {
         return r;
     }
 
-    cufftComplex DevicePaddedImage::CalcMean(size_t total_samples)
+    cufftComplex DevicePaddedImage::CalcMean(size_t total_samples) const
     {
         size_t n_blocks = 50;
         size_t byte_size = n_blocks * sizeof(float);
@@ -383,6 +386,10 @@ namespace {
         r.x = std::accumulate(h_i_sum.begin(), h_i_sum.end(), 0.0);
         r.y = std::accumulate(h_q_sum.begin(), h_q_sum.end(), 0.0);
 
+        if (total_samples == 0) {
+            total_samples = x_size_ * y_size_;
+        }
+
         r.x /= total_samples;
         r.y /= total_samples;
 

cuda_util/device_padded_image.cuh

@@ -125,8 +125,8 @@
         double CalcTotalIntensity(size_t sm_count);
 
         // calculate statistics, return value contains I and Q channel results
-        cufftComplex CalcStdDev(float i_mean, float q_mean, size_t total_samples);
-        cufftComplex CalcMean(size_t total_samples);
+        cufftComplex CalcStdDev(float i_mean, float q_mean, size_t total_samples = 0) const;
+        cufftComplex CalcMean(size_t total_samples = 0) const;
 
         void ZeroNaNs();
 

doc/add/add_esa.md

@@ -18,7 +18,7 @@
 <br>
 <br>
 <br>
-<p style="text-align: center; font-weight: bold">CGI Estonia<br>Jan 8th 2024<br>Issue/Revision: 1/A</p>
+<p style="text-align: center; font-weight: bold">CGI Estonia<br>Jan 8th 2024<br>Issue/Revision: 1/B</p>
 <br>
 
 <p style="text-align: center">Published (including this document) at <a href="https://github.com/cgi-estonia-space/asar-focus">Github<br>https://github.com/cgi-estonia-space/asar-focus</a></p>
@@ -136,7 +136,7 @@ This processor's general system has been kept simple which has been validated by
 systems with possibility to add convenient functionality like graphical user interface or data fetching. Below are
 general blocks that define the approach.
 
-![asar_focus system context](https://github.com/kautlenbachs/bulpp_diagrams/blob/main/asar-focus-system-context-a.drawio.png?raw=true)
+![asar_focus system context](https://private-user-images.githubusercontent.com/98521380/379805603-d07659d5-48f7-4fbf-9661-ba81db2a49ba.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3Mjk3NzkzMDIsIm5iZiI6MTcyOTc3OTAwMiwicGF0aCI6Ii85ODUyMTM4MC8zNzk4MDU2MDMtZDA3NjU5ZDUtNDhmNy00ZmJmLTk2NjEtYmE4MWRiMmE0OWJhLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNDEwMjQlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjQxMDI0VDE0MTAwMlomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTUzMTIzOTNmNDFlZTkwZjcxODFjNzAwZDkyZTM1ODAzYzdjMGU5NGYwZjNlZDVkNWVlODNjZWQ5MmZmYTEzZmYmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.IFwEzsQBKHy3lNfmevUhQI7lUfCxDs-p9v3Nr3gxUG8)
 
 *Figure 3.0 - General system block scheme*
 
@@ -165,12 +165,13 @@ The `asar_focus` processor:
 Main focus is on the efficient GPU processing pipeline where the time spent on I/O should be minimized or parallelized
 while processing calculations. Below is the general pipeline of the processor.
 
-![general processing pipeline](https://github.com/kautlenbachs/bulpp_diagrams/blob/main/asar_meta_flow.drawio.png?raw=true)
+![general processing pipeline](https://private-user-images.githubusercontent.com/98521380/380160723-e622e536-894c-4670-8e7a-16be2fe54ca5.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3Mjk4NTk4MDEsIm5iZiI6MTcyOTg1OTUwMSwicGF0aCI6Ii85ODUyMTM4MC8zODAxNjA3MjMtZTYyMmU1MzYtODk0Yy00NjcwLThlN2EtMTZiZTJmZTU0Y2E1LnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNDEwMjUlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjQxMDI1VDEyMzE0MVomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTljMGRiYmZjNzliMTBlZGQ0MWVlN2FiNmM4YTUwNDEyZGY1N2E5NmIwNzllNjYwYWZjYzQ3MDMzYzZhYTExNDcmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.Yq_JobrN-bzrbgXMB58ZI_hW7vDr9jo_LF7MyROJ1w4)
 
 *Figure 4.0 - General processing pipeline*
 
 The **measurements processing** step is generic focussing pipeline that is represented by the figure below.
-![focussing processing pipeline](https://github.com/kautlenbachs/bulpp_diagrams/blob/main/sar_focussing.drawio.png?raw=true)
+
+![focussing processing pipeline](https://private-user-images.githubusercontent.com/98521380/380164487-f083e9b2-ef14-40b7-825a-3bd5891b1a6a.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3Mjk4NjAzODgsIm5iZiI6MTcyOTg2MDA4OCwicGF0aCI6Ii85ODUyMTM4MC8zODAxNjQ0ODctZjA4M2U5YjItZWYxNC00MGI3LTgyNWEtM2JkNTg5MWIxYTZhLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNDEwMjUlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjQxMDI1VDEyNDEyOFomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWIxZWRjMmIyYTNkZjE5NzBkZjM0NDI3OWE5Y2MxYzk2ODgwYzZiNTUyMWU1NDkwNmFlODhlMzAwZmY1OGVjM2ImWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.Mn8HRrhAYwnSF04WCOMTH1qqCOvxJJa4bmeiWjgH1zI)
 
 *Figure 4.1 - Generic focussing pipeline*
 
@@ -275,7 +276,7 @@ call.
 ## 6.1 Profiling analyze
 
 Example GPU profiling chart is presented below
-![profiling chart](https://sar-focusing.s3.eu-central-1.amazonaws.com/pages/asar-focus_profile_add.png)
+![profiling chart](https://private-user-images.githubusercontent.com/98521380/380165156-46fad159-30ce-458c-9cea-55dad33d99ef.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3Mjk4NjA0ODIsIm5iZiI6MTcyOTg2MDE4MiwicGF0aCI6Ii85ODUyMTM4MC8zODAxNjUxNTYtNDZmYWQxNTktMzBjZS00NThjLTljZWEtNTVkYWQzM2Q5OWVmLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNDEwMjUlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjQxMDI1VDEyNDMwMlomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTFiYjFiZWE4OGVhMWU5NTk5MzNjYWZlM2Q0YmFhNjQxNGNjZGMyMjQ4ODRhNzQ0YTgyOGFlY2E2MjRhMzkwNjImWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.7GKJIMwKXX58cbJUodDfI76qqUUfOxZYFl3TWn46m08)
 
 *Figure 6.1 - Processor profiling*
 

doc/sum/sum.md

@@ -20,7 +20,7 @@ version 8 (codename Ootpa). There are 2 set of requirements lists - for developm
 Common components are NVIDIA GPU driver and CUDA SDK components that should be installed on the processing environment
 for both scenarios.
 
-The list of packages and installation scripts are available at [ALUS-platform Github repo](https://github.com/cgi-estonia-space/ALUs-platform).
+The list of packages and installation scripts are available at [ALUS-platform GitHub repo](https://github.com/cgi-estonia-space/ALUs-platform).
 Based on target platform one can conveniently choose intended scripts. Subchapters consist summary of those.
 
 ## 2.1 Common components
@@ -37,7 +37,7 @@ There are many options to choose from which are all summarized on Nvidia's offic
 The scenario could vary from the type of operating system to orchestration framework used. This means there could be
 different engines used - docker, containerd, CRI-O and podman - which all need customized setup.
 
-Currently utilized and tested approaches within this project are represented below:
+Currently, utilized and tested approaches within this project are represented below:
 * [podman on Ootpa](https://github.com/cgi-estonia-space/ALUs-platform/blob/main/rhel/8/ootpa/install_container_toolkit.sh)
 * [docker on Jammy](https://github.com/cgi-estonia-space/ALUs-platform/blob/main/ubuntu/22_04/jammy-aws/install_nvidia_container_base.sh)
 
@@ -94,7 +94,22 @@ There is no distinction between ERS and ENVISAT instrument datasets. When there
 present for given instrument then this is reported. Same goes for datasets - the processor makes decisions
 based on the given input arguments.
 
-## 3.1 CLI arguments
+## 3.1 Development
+
+The project can be built using CMake. The arguments are:\
+`cmake -B <build files dir> <CMakeLists.txt location>` e.g. `cmake -B build .`\
+Next run `make` inside the build directory.
+
+Additionally one can open the `CMakeLists.txt` file with an editor/IDE that supports CMake (e.g. CLion).
+
+The build script supports two options:
+* Specify CUDA architecture target(s) for the binary - `CMAKE_CUDA_ARCHITECTURES` or environment variable `CUDAARCHS`
+* Build tests - `ALUS_ENABLE_TEST`
+
+For example:\
+`cmake -B my_build_dir -DCMAKE_CUDA_ARCHITECTURES=75 -DALUS_ENABLE_TESTS=ON asar-focus/`
+
+## 3.2 CLI arguments
 
 All the processing input is given by the program arguments. No environment variables or configuration files are used.
 Somewhat exceptions are mission and instrument specific auxiliary files which are supplied with the directory from
@@ -103,7 +118,7 @@ specific DORIS orbit file path could be given. When not specifying sensing time
 input packets if there is sufficient GPU memory.
 
 ```
-asar_focus/0.2.0
+asar_gpu/0.3.1
   -h [ --help ]         Print help
   -i [ --input ] arg    Level 0 ERS or ENVISAT dataset
   --aux arg             Auxiliary files folder path
@@ -121,10 +136,10 @@ asar_focus/0.2.0
                         verbose|debug|info|warning|error
 ```
 
-As of release version 0.3 `type` supports `IMS` or `IMP`. Any other value would result in termination
+As of release version 0.3.x `type` supports `IMS` or `IMP`. Any other value would result in termination
 of processing with accompanying message on console.
 
-### 3.1.1 Hidden CLI features
+### 3.2.1 Hidden CLI features
 
 For debugging and research possibilities couple of 'unofficial' command line arguments are present which are given below.
 * `--plot` - create HTML files consisting graphs of the following processing properties - processing velocity,
@@ -134,9 +149,9 @@ For debugging and research possibilities couple of 'unofficial' command line arg
 All the created files would be saved in the same directory as the final product given by the `output` argument. They are
 prefixed with the level 0 dataset name pattern.
 
-### 3.1.3 Examples
+### 3.2.2 Examples
 
-During the production and internal tests there are so called test datasets (TDS) packages used. This is not required
+During the production and internal tests there are so-called test datasets (TDS) packages used. This is not required
 and one could simply have manually assembled the collection of suitable orbit files and auxiliary files downloaded as
 a bundle on online at available ESA sites (see [ERS](https://earth.esa.int/eogateway/instruments/sar-ers/auxiliary-data)
 and [ENVISAT](https://earth.esa.int/eogateway/instruments/asar/auxiliary-data)).
@@ -161,14 +176,14 @@ DOR_VOR_AXVF-P20120424_125300_20040228_215528_20040301_002328
 --sensing_stop 20040229_212523002000
 ```
 
-## 3.2 Input datasets
+## 3.3 Input datasets
 
-Single level 0 **envisat format** ERS-1/2 and Envisat mission dataset is required. As of version 0.3 only imaging mode
+Single level 0 **envisat format** ERS-1/2 and Envisat mission dataset is required. As of version 0.3.x only imaging mode
 datasets are supported. The specification is given in document `PO-RS-MDA-GS-2009 4/C`.
 
-## 3.3 Auxiliary files
+## 3.4 Auxiliary files
 
-As of version 0.3 the following auxiliary files are used:
+As of version 0.3.x the following auxiliary files are used:
 * DORIS orbit files in envisat format, see [DOR_VOR_AX](https://earth.esa.int/eogateway/catalog/envisat-doris-precise-orbit-state-vectors-dor-vor_ax-)
 * Processor configuration file (ASA_CON_AX/ER_CON_AX)
 * Instrument characterization file (AUX_INS_AX/ER_INS_AX)
@@ -178,12 +193,12 @@ As of version 0.3 the following auxiliary files are used:
 For auxiliary file access and documentation please visit [ERS aux](https://earth.esa.int/eogateway/instruments/sar-ers/auxiliary-data)
 and [ENVISAT aux](https://earth.esa.int/eogateway/instruments/asar/auxiliary-data) pages at ESA.
 
-# 4 Requirements
+# 4 Hardware requirements
 
 Below are specified minimum requirements for hardware. It is based on the resources needed to focus 16 seconds of
 level 0 imaging mode dataset. Resource like CPU is out of scope, because all modern CPUs are suitable. Disk storage
 resource is dismal, since only level 1 dataset storage is required also no intermediate files are stored
-(except for debugging features discussed in [3.1.1 Hidden CLI features](#311-hidden-cli-features)).
+(except for debugging features discussed in [3.2.1 Hidden CLI features](#321-hidden-cli-features)).
 
 | Property               | Value  |
 |------------------------|--------|