Az rg window prepare

.github/workflows/e2e.yml

@@ -59,6 +59,8 @@ jobs:
           ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "CUDAARCHS=\"50;52\" CMAKE_BUILD_TYPE=Release /tmp/asar-focus/build-automation/create_exe_in_container.sh /tmp/asar-focus cgialus/alus-ootpa-devel-extra"
           # To RAM disk
           ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "cp -r /home/$INSTANCE_USERNAME/e2e /dev/shm/"
+          # Do not use previous run results here. When skipping/failing would possibly use valid results for validation.
+          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "rm -rf /dev/shm/e2e/last_run/*"
       - uses: actions/checkout@v4
       - name: End-to-end tests
         id: e2e-run

envisat_format/include/asar_constants.h

@@ -1,18 +1,17 @@
 /**
-* ENVISAT and ERS ASAR instrument focusser for QA4EO activity (c) by CGI Estonia AS
-*
-* ENVISAT and ERS ASAR instrument focusser for QA4EO activity is licensed under a
-* Creative Commons Attribution-ShareAlike 4.0 International License.
-*
-* You should have received a copy of the license along with this
-* work. If not, see http://creativecommons.org/licenses/by-sa/4.0/
-*/
+ * ENVISAT and ERS ASAR instrument focusser for QA4EO activity (c) by CGI Estonia AS
+ *
+ * ENVISAT and ERS ASAR instrument focusser for QA4EO activity is licensed under a
+ * Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * You should have received a copy of the license along with this
+ * work. If not, see http://creativecommons.org/licenses/by-sa/4.0/
+ */
 #pragma once
 
 #include <cstdint>
 #include <string_view>
 
-
 /**
  * ASAR specific constants use for converting raw data to physical values
  */
@@ -24,19 +23,14 @@ float AuxTxMonitorConversion(uint16_t aux_tx);
 
 namespace alus::asar::specification {
 
-    enum ProductTypes{
-        SAR_IM0,
-        ASA_IM0,
-        SAR_IMS,
-        SAR_IMP,
-        ASA_IMS,
-        ASA_IMP,
-        UNIDENTIFIED
-    };
-
-    constexpr std::string_view PRODUCT_NAME_SAR_IM0{"SAR_IM__0"};
-    constexpr std::string_view PRODUCT_NAME_ASA_IM0{"ASA_IM__0"};
-
-    ProductTypes GetProductTypeFrom(std::string_view product_name);
-    ProductTypes TryDetermineTargetProductFrom(ProductTypes in_product, std::string_view user_defined_target_type);
-}
+enum ProductTypes { SAR_IM0, ASA_IM0, SAR_IMS, SAR_IMP, ASA_IMS, ASA_IMP, UNIDENTIFIED };
+
+enum Instrument { SAR, ASAR };
+
+constexpr std::string_view PRODUCT_NAME_SAR_IM0{"SAR_IM__0"};
+constexpr std::string_view PRODUCT_NAME_ASA_IM0{"ASA_IM__0"};
+
+ProductTypes GetProductTypeFrom(std::string_view product_name);
+ProductTypes TryDetermineTargetProductFrom(ProductTypes in_product, std::string_view user_defined_target_type);
+Instrument GetInstrumentFrom(ProductTypes product);
+}  // namespace alus::asar::specification

envisat_format/include/envisat_aux_file.h

@@ -10,10 +10,12 @@
 #pragma once
 
 #include <filesystem>
+#include <string_view>
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 
 #include "bswap_util.h"
+#include "envisat_aux_xca.h"
 
 /**
  * Parser for ASA_CON and ASA_INS files, auxiliary files used for metadata extractions
@@ -710,3 +712,7 @@ static_assert(sizeof(ConfigurationFile) == 4096);
 void FindINSFile(std::string aux_root, boost::posix_time::ptime start, InstrumentFile& ins_file, std::string& filename);
 void FindCONFile(std::string aux_root, boost::posix_time::ptime start, ConfigurationFile& ins_file,
                  std::string& filename);
+
+namespace alus::asar::envformat::aux {
+void GetXca(std::string_view aux_root, boost::posix_time::ptime start, ExternalCalibration& xca, std::string& filename);
+}

envisat_format/include/envisat_aux_xca.h

@@ -0,0 +1,131 @@
+/**
+ * ENVISAT and ERS ASAR instrument focusser for QA4EO activity (c) by CGI Estonia AS
+ *
+ * ENVISAT and ERS ASAR instrument focusser for QA4EO activity is licensed under a
+ * Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * You should have received a copy of the license along with this
+ * work. If not, see http://creativecommons.org/licenses/by-sa/4.0/
+ */
+#pragma once
+
+#include "bswap_util.h"
+#include "envisat_types.h"
+
+namespace alus::asar::envformat::aux {
+
+// PO-RS-MDA-GS-2009 table 8.6.2.1-1
+struct ExternalCalibration {
+    mjd creation_time;
+    ul dsr_length;
+    // 7 values stand for swaths IS1 to IS7.
+    fl scaling_factor_im_slc_hh[7];
+    fl scaling_factor_im_slc_vv[7];
+    fl scaling_factor_im_precision_hh[7];
+    fl scaling_factor_im_precision_vv[7];
+    fl scaling_factor_im_geocoded_hh[7];
+    fl scaling_factor_im_geocoded_vv[7];
+    fl scaling_factor_im_mediumres_hh[7];
+    fl scaling_factor_im_mediumres_vv[7];
+    fl scaling_factor_ap_slc_hh[7];
+    fl scaling_factor_ap_slc_vv[7];
+    fl scaling_factor_ap_slc_hv[7];
+    fl scaling_factor_ap_slc_vh[7];
+    fl scaling_factor_ap_precision_hh[7];
+    fl scaling_factor_ap_precision_vv[7];
+    fl scaling_factor_ap_precision_hv[7];
+    fl scaling_factor_ap_precision_vh[7];
+    fl scaling_factor_ap_geocoded_hh[7];
+    fl scaling_factor_ap_geocoded_vv[7];
+    fl scaling_factor_ap_geocoded_hv[7];
+    fl scaling_factor_ap_geocoded_vh[7];
+    fl scaling_factor_ap_mediumres_hh[7];
+    fl scaling_factor_ap_mediumres_vv[7];
+    fl scaling_factor_ap_mediumres_hv[7];
+    fl scaling_factor_ap_mediumres_vh[7];
+    fl scaling_factor_wv_hh[7];
+    fl scaling_factor_wv_vv[7];
+    fl scaling_factor_ws_hh;
+    fl scaling_factor_ws_vv;
+    fl scaling_factor_gm_hh;
+    fl scaling_factor_gm_vv;
+
+    fl ref_elev_angle_is1;
+    fl ref_elev_angle_is2;
+    fl ref_elev_angle_is3_ss2;
+    fl ref_elev_angle_is4_ss3;
+    fl ref_elev_angle_is5_ss4;
+    fl ref_elev_angle_is6_ss5;
+    fl ref_elev_angle_is7;
+    fl ref_elev_angle_ss1;
+
+    // pattern is defined from reference elevation angle - 5 deg. to reference
+    // elevation angle + 5 deg. in 0.05 degree steps.
+    fl two_way_ant_elev_pat_gain_lut_is1[804];
+    fl two_way_ant_elev_pat_gain_lut_is2[804];
+    fl two_way_ant_elev_pat_gain_lut_is3_ss2[804];
+    fl two_way_ant_elev_pat_gain_lut_is4_ss3[804];
+    fl two_way_ant_elev_pat_gain_lut_is5_ss4[804];
+    fl two_way_ant_elev_pat_gain_lut_is6_ss5[804];
+    fl two_way_ant_elev_pat_gain_lut_is7[804];
+    fl two_way_ant_elev_pat_gain_lut_ss1[804];
+    uc spare[32];
+
+    void Bswap() {
+        creation_time = bswap(creation_time);
+        dsr_length = bswap(dsr_length);
+        BSWAP_ARR(scaling_factor_im_slc_hh);
+        BSWAP_ARR(scaling_factor_im_slc_vv);
+        BSWAP_ARR(scaling_factor_im_precision_hh);
+        BSWAP_ARR(scaling_factor_im_precision_vv);
+        BSWAP_ARR(scaling_factor_im_geocoded_hh);
+        BSWAP_ARR(scaling_factor_im_geocoded_vv);
+        BSWAP_ARR(scaling_factor_im_mediumres_hh);
+        BSWAP_ARR(scaling_factor_im_mediumres_vv);
+        BSWAP_ARR(scaling_factor_ap_slc_hh);
+        BSWAP_ARR(scaling_factor_ap_slc_vv);
+        BSWAP_ARR(scaling_factor_ap_slc_hv);
+        BSWAP_ARR(scaling_factor_ap_slc_vh);
+        BSWAP_ARR(scaling_factor_ap_precision_hh);
+        BSWAP_ARR(scaling_factor_ap_precision_vv);
+        BSWAP_ARR(scaling_factor_ap_precision_hv);
+        BSWAP_ARR(scaling_factor_ap_precision_vh);
+        BSWAP_ARR(scaling_factor_ap_geocoded_hh);
+        BSWAP_ARR(scaling_factor_ap_geocoded_vv);
+        BSWAP_ARR(scaling_factor_ap_geocoded_hv);
+        BSWAP_ARR(scaling_factor_ap_geocoded_vh);
+        BSWAP_ARR(scaling_factor_ap_mediumres_hh);
+        BSWAP_ARR(scaling_factor_ap_mediumres_vv);
+        BSWAP_ARR(scaling_factor_ap_mediumres_hv);
+        BSWAP_ARR(scaling_factor_ap_mediumres_vh);
+        BSWAP_ARR(scaling_factor_wv_hh);
+        BSWAP_ARR(scaling_factor_wv_vv);
+        scaling_factor_ws_hh = bswap(scaling_factor_ws_hh);
+        scaling_factor_ws_vv = bswap(scaling_factor_ws_vv);
+        scaling_factor_gm_hh = bswap(scaling_factor_gm_hh);
+        scaling_factor_gm_vv = bswap(scaling_factor_gm_vv);
+
+        ref_elev_angle_is1 = bswap(ref_elev_angle_is1);
+        ref_elev_angle_is2 = bswap(ref_elev_angle_is2);
+        ref_elev_angle_is3_ss2 = bswap(ref_elev_angle_is3_ss2);
+        ref_elev_angle_is4_ss3 = bswap(ref_elev_angle_is4_ss3);
+        ref_elev_angle_is5_ss4 = bswap(ref_elev_angle_is5_ss4);
+        ref_elev_angle_is6_ss5 = bswap(ref_elev_angle_is6_ss5);
+        ref_elev_angle_is7 = bswap(ref_elev_angle_is7);
+        ref_elev_angle_ss1 = bswap(ref_elev_angle_ss1);
+
+        // pattern is defined from reference elevation angle - 5 deg. to reference
+        // elevation angle + 5 deg. in 0.05 degree steps.
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is1);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is2);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is3_ss2);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is4_ss3);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is5_ss4);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is6_ss5);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_is7);
+        BSWAP_ARR(two_way_ant_elev_pat_gain_lut_ss1);
+    }
+};
+
+static_assert(sizeof(ExternalCalibration) == 26552);
+}  // namespace alus::asar::envformat::aux

envisat_format/include/envisat_im_parse.h

@@ -20,9 +20,14 @@
 
 namespace alus::asar::envformat {
 
-void ParseEnvisatLevel0ImPackets(const std::vector<char>& file_data, const DSD_lvl0& mdsr, SARMetadata& sar_meta,
-                                 ASARMetadata& asar_meta, cufftComplex** d_parsed_packets, InstrumentFile& ins_file,
-                                 boost::posix_time::ptime packets_start_filter = boost::posix_time::not_a_date_time,
-                                 boost::posix_time::ptime packets_stop_filter = boost::posix_time::not_a_date_time);
+std::vector<ForecastMeta> FetchEnvisatL0ImForecastMeta(const std::vector<char>& file_data, const DSD_lvl0& mdsr,
+                                                       boost::posix_time::ptime packets_start_filter,
+                                                       boost::posix_time::ptime packets_stop_filter,
+                                                       size_t& packets_before_start, size_t& packets_after_stop);
+
+RawSampleMeasurements ParseEnvisatLevel0ImPackets(const std::vector<char>& file_data, size_t mdsr_offset_bytes,
+                                                  const std::vector<ForecastMeta>& entries_to_be_parsed,
+                                                  InstrumentFile& ins_file,
+                                                  std::vector<CommonPacketMetadata>& common_metadata);
 
 }  // namespace alus::asar::envformat

envisat_format/include/envisat_lvl0_parser.h

@@ -28,16 +28,13 @@
 #include "sar/orbit_state_vector.h"
 #include "sar/sar_metadata.h"
 
-/**
- * Envisat LVL0 IM file parser, ex ASA_IM__0PNPDE20040111_085939_000000182023_00179_09752_2307.N1
- * Extracts RAW echo data and relevant metadata needed for SAR focussing
- */
-
 struct ASARMetadata {
     std::string lvl0_file_name;
     std::string product_name;
     std::string instrument_file;
     std::string configuration_file;
+    std::string external_calibration_file;
+    std::string external_characterization_file;
     std::string orbit_dataset_name;
     std::string acquistion_station;
     std::string processing_station;
@@ -126,11 +123,59 @@ struct ASARMetadata {
 
 namespace alus::asar::envformat {
 
+int SwathIdx(const std::string& swath);
+
+struct ForecastMeta {
+    size_t packet_start_offset_bytes;
+    mjd isp_sensing_time;
+};
+
+struct CommonPacketMetadata {
+    mjd sensing_time;
+    uint16_t swst_code;
+    uint16_t pri_code;
+    uint64_t onboard_time;
+    uint16_t sample_count;                    // Count of measurement I/Q samples, not equal to bytes.
+    uint16_t measurement_array_length_bytes;  // Including any block information etc...
+
+    struct {
+        uint8_t chirp_pulse_bw_code;
+        uint8_t upconverter_raw;
+        uint8_t downconverter_raw;
+        uint16_t tx_pulse_code;
+        uint8_t antenna_beam_set_no;
+        uint8_t beam_adj_delta;
+        uint16_t resampling_factor;
+    } asar;
+};
+
+struct RawSampleMeasurements {
+    std::unique_ptr<uint8_t[]> raw_samples;
+    size_t single_entry_length; // Length in bytes of the maximum sample measurement record, including block id etc.
+    size_t entries_total; // How many lines in azimuth direction
+    size_t max_samples; // Maximum samples that the entries will consist. Can fluctuate.
+    size_t total_samples; // Total IQ samples collected over all of the range and azimuth directions.
+    size_t no_of_product_errors_compensated;
+};
+
+DSD_lvl0 ParseSphAndGetMdsr(ASARMetadata& asar_meta, const SARMetadata& sar_meta, const std::vector<char>& file_data);
+
+/* Current silent contract is that every packet of the returned list of ForecastMeta shall be parsed by the
+ * ParseLevel0Packets. That also means that any duplicates and/or missing packets shall be compensated already
+ * during FetchMeta. Parsing packets would just respect the offsets. But it can throw if there are discrepancies,
+ * which would indicate programming error. Current FetchMeta is simple byte offsets starting from MDSR (not L0 ds)
+ * and sensing time. Later more fields could be added and parsed to implement more comprehensive parsing scheme.
+ */
+std::vector<ForecastMeta> FetchMeta(const std::vector<char>& file_data, boost::posix_time::ptime packets_start_filter,
+                                    boost::posix_time::ptime packets_stop_filter,
+                                    specification::ProductTypes product_type, size_t& packets_before_start,
+                                    size_t& packets_after_stop);
+
 void ParseLevel0Header(const std::vector<char>& file_data, ASARMetadata& asar_meta);
-void ParseLevel0Packets(const std::vector<char>& file_data, SARMetadata& sar_meta, ASARMetadata& asar_meta,
-                        cufftComplex** d_parsed_packets, alus::asar::specification::ProductTypes product_type,
-                        InstrumentFile& ins_file,
-                        boost::posix_time::ptime packets_start_filter = boost::posix_time::not_a_date_time,
-                        boost::posix_time::ptime packets_stop_filter = boost::posix_time::not_a_date_time);
+
+RawSampleMeasurements ParseLevel0Packets(const std::vector<char>& file_data, size_t mdsr_offset_bytes,
+                                         const std::vector<ForecastMeta>& entries_to_be_parsed,
+                                         alus::asar::specification::ProductTypes product_type, InstrumentFile& ins_file,
+                                         std::vector<CommonPacketMetadata>& common_metadata);
 
 }  // namespace alus::asar::envformat

envisat_format/include/envisat_types.h

@@ -36,6 +36,10 @@ struct mjd {
         return days == o.days && seconds == o.seconds && micros == o.micros;
     }
 
+    bool operator !=(const mjd& o) const{
+        return days != o.days || seconds != o.seconds || micros != o.micros;
+    }
+
     bool operator <(const mjd& o) const{
         if (days == o.days) {
             if (seconds == o.seconds) {

envisat_format/include/ers_im_parse.h

@@ -19,6 +19,16 @@
 
 namespace alus::asar::envformat {
 
+std::vector<ForecastMeta> FetchErsL0ImForecastMeta(const std::vector<char>& file_data, const DSD_lvl0& mdsr,
+                                                   boost::posix_time::ptime packets_start_filter,
+                                                   boost::posix_time::ptime packets_stop_filter,
+                                                   size_t& packets_before_start, size_t& packets_after_stop);
+
+RawSampleMeasurements ParseErsLevel0ImPackets(const std::vector<char>& file_data, size_t mdsr_offset_bytes,
+                                              const std::vector<ForecastMeta>& entries_to_be_parsed,
+                                              InstrumentFile& ins_file,
+                                              std::vector<CommonPacketMetadata>& common_metadata);
+
 void ParseErsLevel0ImPackets(const std::vector<char>& file_data, const DSD_lvl0& mdsr, SARMetadata& sar_meta,
                              ASARMetadata& asar_meta, cufftComplex** d_parsed_packets, InstrumentFile& ins_file,
                              boost::posix_time::ptime packets_start_filter = boost::posix_time::not_a_date_time,

envisat_format/src/asar_aux.cc

@@ -75,7 +75,8 @@ std::string GetPathFrom(std::string aux_root, boost::posix_time::ptime start, Ty
         }
 
         if (current_sat_id != satellite_id) {
-            ERROR_EXIT("There are mixed satellite auxiliary files in the supplied aux folder - " + aux_root);
+            throw std::runtime_error("There are mixed satellite " + std::string(id_string) +
+                                     " auxiliary files in the supplied aux folder - " + aux_root);
         }
 
         if (items.at(1) != id_string) {

envisat_format/src/asar_constants.cc

@@ -81,7 +81,19 @@ ProductTypes TryDetermineTargetProductFrom(ProductTypes in_product, std::string_
     }
 
     throw std::runtime_error("Focussing '" + std::string(user_defined_target_type) +
-                             "' product from given input is not supported.");
+                             "' product from given input is not supported or valid.");
+}
+
+Instrument GetInstrumentFrom(ProductTypes product) {
+    if (product == ProductTypes::SAR_IM0 || product == ProductTypes::SAR_IMS) {
+        return Instrument::SAR;
+    }
+
+    if (product == ProductTypes::ASA_IM0 || product == ProductTypes::ASA_IMS) {
+        return Instrument::ASAR;
+    }
+
+    throw std::runtime_error("Unsupported product type supplied to " + std::string(__FUNCTION__));
 }
 
 }  // namespace alus::asar::specification