Added a ReadPartialData() function that only reads requested samples …

…(min/max). #83

include/mdf/ichannelobserver.h

@@ -254,4 +254,9 @@ bool IChannelObserver::GetEngValue(uint64_t sample, V& value, uint64_t array_ind
   return valid;
 }
 
+/** \brief Returns the sample engineering (channel) value as a byte array. */
+template <>
+bool IChannelObserver::GetEngValue(uint64_t sample,
+                                       std::vector<uint8_t>& value,
+                                       uint64_t array_index) const;
 }  // namespace mdf

include/mdf/idatagroup.h

@@ -81,7 +81,7 @@ class IDataGroup : public IBlock {
   /** \brief Detaches all observers from the measurement. */
   void DetachAllSampleObservers() const;
   /** \brief Notifies the observer that a new sample record have been read.*/
-  void NotifySampleObservers(size_t sample, uint64_t record_id,
+  bool NotifySampleObservers(size_t sample, uint64_t record_id,
                              const std::vector<uint8_t>& record) const;
 
   /** \brief Clear all temporary sample and data buffers.

include/mdf/isampleobserver.h

@@ -47,12 +47,19 @@ class ISampleObserver {
    *
    * The function may be override by an inheritance for more complex
    * implementations but in simpler use cases the DoOnSample function
-   * should be used instead,
+   * should be used instead.
+   *
+   * The function normally returns true indicating that more reading is
+   * expected. If any observer want to stop further parsing, the function
+   * should return false.
+   *
    * @param sample Sample number.
    * @param record_id Record ID (channel group identity).
    * @param record Sample record (excluding the record ID.
+   * @return If this function returns false it indicate that reading should be
+   * aborted.
    */
-  virtual void OnSample(uint64_t sample, uint64_t record_id,
+  virtual bool OnSample(uint64_t sample, uint64_t record_id,
                         const std::vector<uint8_t>& record);
 
   /**
@@ -71,7 +78,7 @@ class ISampleObserver {
    * The function object is typically assigned to a lambda function.
    * See also OnSample() function.
    */
-  std::function<void(uint64_t sample, uint64_t record_id,
+  std::function<bool(uint64_t sample, uint64_t record_id,
       const std::vector<uint8_t>& record)> DoOnSample;
 
   /** \brief The function returns a channel value.

include/mdf/mdfreader.h

@@ -118,10 +118,32 @@ class MdfReader {
    * The attached observers also consumes memory, so remember to delete
    * them when they are no more needed.
    * @param data_group Reference to the data group (DG) object.
-   * @return True if the red was successful.
+   * @return True if the read was successful.
    */
   bool ReadData(IDataGroup& data_group);
 
+  /** \brief Reads a range of samples.
+   *
+   * Reads in a range of samples a data group (DG). The function
+   * reads in sample data (DT..) blocks, sample reduction (RD/RV/RI) blocks
+   * and signal data (SD) blocks. The function is faster that reading all data
+   * bytes, skipping records it doesn't need to read.
+   *
+   * Note that this function still may consume a lot
+   * of memory, so remember to call the IDataGroup::ClearData() function
+   * when data not are needed anymore.
+   *
+   * The attached observers also consumes memory, so remember to delete
+   * them when they are no more needed.
+   *
+   * @param data_group Reference to the data group (DG) object.
+   * @param min_sample First sample index to read.
+   * @param max_sample Last sample index to read.
+   * @return True if the read was successful.
+   */
+  bool ReadPartialData(IDataGroup& data_group, size_t min_sample,
+                       size_t max_sample);
+
   /** \brief Reads in data bytes to a sample reduction (SR) block.
    *
    * To minimíze the use of time and memory, this function reads in
@@ -132,10 +154,30 @@ class MdfReader {
    */
   bool ReadSrData(ISampleReduction& sr_group);
 
+  /** \brief Read in partial variable length data with an offset list.
+   *
+   * This function reads in VLSD stored data according to an offset list.
+   * For smaller MDF files that fits in the primary memory, this function is not
+   * needed but sometimes the VLSD data sample holds a large amount of data
+   * bytes typical a video stream. These files tends to be huge so the
+   * application runs out of memory.
+   *
+   * This function reads in VLSD stored data in smaller batcher. The
+   * application first reads in all offsets to the raw data. Using these offsets
+   * the application can read in typically one sample (offset) at a time. This
+   * tactic saves primary memory.
+   *
+   * @param data_group Data group to read VLSD data from
+   * @param vlsd_channel Which channel that stores the VLSD data
+   * @param offset_list List of offsets (samples) to read.
+   * @param callback Callback function for each offset/sample data
+   * @return Returns true if the read was successful
+   */
   bool ReadVlsdData(IDataGroup &data_group,
                     IChannel &vlsd_channel,
                     const std::vector<uint64_t>& offset_list,
-                    std::function<void(uint64_t, const std::vector<uint8_t>&)>& callback);
+                    std::function<void(uint64_t,
+                                       const std::vector<uint8_t>&)>& callback);
 
 
  private:

mdflib/CMakeLists.txt

@@ -93,6 +93,10 @@ add_library(mdf
         src/isampleobserver.cpp  ../include/mdf/isampleobserver.h
         src/isamplereduction.cpp ../include/mdf/isamplereduction.h
         src/readcache.cpp src/readcache.h
+        src/dgrange.cpp
+        src/dgrange.h
+        src/cgrange.cpp
+        src/cgrange.h
 )
 
 if(VCPKG)
@@ -117,6 +121,8 @@ target_include_directories(
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   set_target_properties(mdf PROPERTIES COMPILE_FLAGS "-fPIC")
+elseif(MSVC)
+  target_compile_definitions(mdf PRIVATE _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS _CRT_SECURE_NO_WARNINGS)
 endif()
 
 set(MDF_PUBLIC_HEADERS

mdflib/src/cg3block.cpp

@@ -54,9 +54,8 @@ const IChannel *Cg3Block::GetXChannel(const IChannel &) const {
   // Search for the master channel in the group
   auto master = std::find_if(cn_list_.cbegin(), cn_list_.cend(),
                              [&](const auto &cn3) {
-    return !cn3 ? false
-                : cn3->Type() == ChannelType::Master ||
-                      cn3->Type() == ChannelType::VirtualMaster;
+    return cn3 && (cn3->Type() == ChannelType::Master ||
+                         cn3->Type() == ChannelType::VirtualMaster);
   });
 
   return master != cn_list_.cend() ? master->get() : nullptr;
@@ -236,21 +235,66 @@ void Cg3Block::PrepareForWriting() {
 
 size_t Cg3Block::ReadDataRecord(std::FILE *file,
                                 const IDataGroup &notifier) const {
-  size_t count = 0;
+  size_t count;
 
   // Normal fixed length records
   size_t record_size = size_of_data_record_;
   std::vector<uint8_t> record(record_size, 0);
   count = std::fread(record.data(), 1, record.size(), file);
   size_t sample = Sample();
   if (sample < NofSamples()) {
-    notifier.NotifySampleObservers(sample, RecordId(), record);
+    const bool continue_reading = notifier.NotifySampleObservers(sample, RecordId(), record);
     IncrementSample();
+    if (!continue_reading) {
+      return 0; // Indicating no more reading
+    }
   }
 
   return count;
 }
 
+size_t Cg3Block::ReadRangeDataRecord(std::FILE *file,
+                                const IDataGroup &notifier,
+                                     DgRange& range) const {
+  size_t count;
+  const size_t sample = Sample();
+  const size_t next_sample = sample + 1;
+  const size_t record_size = size_of_data_record_;
+  CgRange* cg_range = range.GetCgRange(RecordId());
+  if (cg_range == nullptr) {
+    return 0;
+  }
+
+  if (!cg_range->IsUsed() ||
+      next_sample < range.MinSample() ||
+      next_sample > range.MaxSample()) {
+    count = StepFilePosition(file, record_size);
+
+    if (sample < NofSamples()) {
+      IncrementSample();
+    }
+    if (next_sample > range.MaxSample()) {
+      cg_range->IsUsed(false);
+    }
+  } else {
+    // Normal fixed length records
+
+    std::vector<uint8_t> record(record_size, 0);
+
+    count = std::fread(record.data(), 1, record.size(), file);
+
+    if (sample < NofSamples()) {
+      const bool continue_reading =
+          notifier.NotifySampleObservers(sample, RecordId(), record);
+      IncrementSample();
+      if (!continue_reading) {
+        return 0;  // Indicating no more reading
+      }
+    }
+  }
+
+  return count;
+}
 IChannel *Cg3Block::CreateChannel() {
   auto cn3 = std::make_unique<detail::Cn3Block>();
   cn3->Init(*this);

mdflib/src/cg3block.h

@@ -10,6 +10,7 @@
 #include "mdfblock.h"
 #include "sr3block.h"
 #include "tx3block.h"
+#include "dgrange.h"
 namespace mdf::detail {
 class Cg3Block : public MdfBlock, public IChannelGroup {
  public:
@@ -61,6 +62,8 @@ class Cg3Block : public MdfBlock, public IChannelGroup {
     return sample_buffer_;
   }
   size_t ReadDataRecord(std::FILE* file, const IDataGroup& notifier) const;
+  size_t ReadRangeDataRecord(std::FILE* file, const IDataGroup& notifier,
+                             DgRange& range) const;
   void PrepareForWriting();
 
   void ClearData() override;

mdflib/src/cg4block.cpp

@@ -4,12 +4,17 @@
  */
 #include "cg4block.h"
 
+
+
 #include <algorithm>
+
 #include <codecvt>
 #include <climits>
 #include "cn4block.h"
 #include "sr4block.h"
 
+
+
 namespace {
 
 constexpr size_t kIndexNext = 0;
@@ -307,8 +312,12 @@ size_t Cg4Block::ReadDataRecord(std::FILE *file,
     }
     const size_t sample = Sample();
     if (sample < NofSamples()) {
-      notifier.NotifySampleObservers(sample, RecordId(), record);
+      const bool continue_reading = notifier.NotifySampleObservers(sample,
+                                                 RecordId(), record);
       IncrementSample();
+      if (!continue_reading) {
+        return 0;
+      }
     }
   } else {
     // Normal fixed length records
@@ -317,8 +326,12 @@ size_t Cg4Block::ReadDataRecord(std::FILE *file,
     count = std::fread(record.data(), 1, record.size(), file);
     const size_t sample = Sample();
     if (sample < NofSamples()) {
-      notifier.NotifySampleObservers(sample, RecordId(), record);
+      const bool continue_reading = notifier.NotifySampleObservers(sample, RecordId(), record);
       IncrementSample();
+      if (!continue_reading) {
+        return 0;
+      }
+
     }
   }
   return count;

mdflib/src/cgrange.cpp

@@ -0,0 +1,21 @@
+/*
+* Copyright 2024 Ingemar Hedvall
+* SPDX-License-Identifier: MIT
+ */
+/** \file
+ * This file implement the channel group range support class.
+ */
+
+#include "cgrange.h"
+
+namespace mdf {
+
+CgRange::CgRange(const IChannelGroup& channel_group)
+: channel_group_(channel_group) {
+
+}
+uint64_t CgRange::RecordId() const {
+  return channel_group_.RecordId();
+}
+
+}  // namespace mdf

mdflib/src/cgrange.h

@@ -0,0 +1,33 @@
+/*
+* Copyright 2024 Ingemar Hedvall
+* SPDX-License-Identifier: MIT
+ */
+/** \file
+ * This file define the channel group range support class.
+ */
+
+#include "mdf/ichannelgroup.h"
+
+namespace mdf {
+
+class CgRange final {
+ public:
+  CgRange() = delete;
+  explicit CgRange(const IChannelGroup& channel_group);
+  [[nodiscard]] uint64_t RecordId() const;
+
+  void IsUsed(bool used) { is_used_ = used;}
+  [[nodiscard]] bool IsUsed() const { return is_used_;}
+
+  [[nodiscard]] const IChannelGroup& ChannelGroup() const {
+    return channel_group_;
+  }
+
+ private:
+  const IChannelGroup& channel_group_;
+  bool is_used_ = false;
+};
+
+}  // namespace mdf
+
+

mdflib/src/channelobserver.h

@@ -68,7 +68,7 @@ class ChannelObserver : public IChannelObserver {
     return group_.NofSamples();
   }
 
-  void OnSample(uint64_t sample, uint64_t record_id,
+  bool OnSample(uint64_t sample, uint64_t record_id,
                 const std::vector<uint8_t>& record) override {
     const auto* channel_array = channel_.ChannelArray();
     auto array_size = channel_array != nullptr ? channel_array->NofArrayValues() : 1;
@@ -155,7 +155,9 @@ class ChannelObserver : public IChannelObserver {
         }
         break;
     }
+    return true;
   }
+
 };