Initial POC for MCAP Support with c++ and python

CMakeLists.txt

@@ -209,6 +209,35 @@ install(TARGETS ${PROJECT_NAME}
         EXPORT ${PROJECT_NAME}_targets
         DESTINATION "${OSI_INSTALL_LIB_DIR}" COMPONENT lib)
 
+
+##### TODO: WORK IN PROGRESS START
+
+# mcap support
+set(TRACEFILE_MCAP_SOURCES mcap/osi_tracefile_mcap_reader.cpp mcap/osi_tracefile_mcap_writer.cpp)
+add_library(${PROJECT_NAME}_mcap_static STATIC ${TRACEFILE_MCAP_SOURCES} mcap/include/osi_tracefile_mcap_reader.h mcap/include/osi_tracefile_mcap_writer.h)
+add_library(${PROJECT_NAME}::${PROJECT_NAME}_mcap_static ALIAS ${PROJECT_NAME}_mcap_static)
+add_dependencies(${PROJECT_NAME}_mcap_static ${PROJECT_NAME})
+target_include_directories(${PROJECT_NAME}_mcap_static
+        PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/mcap/include>
+        $<INSTALL_INTERFACE:${OSI_INSTALL_INCLUDE_DIR}>
+        PRIVATE
+        mcap/mcap/include
+)
+set_target_properties(${PROJECT_NAME}_mcap_static PROPERTIES CXX_STANDARD 17)
+
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(lz4 REQUIRED IMPORTED_TARGET liblz4)
+pkg_check_modules(ZSTD REQUIRED libzstd)
+target_link_libraries(${PROJECT_NAME}_mcap_static PUBLIC ${lz4_LIBRARIES} ${ZSTD_LIBRARIES} ${PROJECT_NAME}_pic)
+
+install(TARGETS ${PROJECT_NAME}_mcap_static
+        EXPORT ${PROJECT_NAME}_targets
+        ARCHIVE DESTINATION "${OSI_INSTALL_LIB_DIR}" COMPONENT lib
+        INCLUDES DESTINATION "${OSI_INSTALL_INCLUDE_DIR}")
+
+##### TODO: WORK IN PROGRESS END
+
 # Copy proto headers to where they are expected by the package config file
 add_custom_command(
     TARGET ${PROJECT_NAME} POST_BUILD
@@ -246,7 +275,7 @@ install(FILES
     COMPONENT dev)
 
 # Header files
-install(FILES ${PROTO_HEADERS} ${FLAT_HEADERS}
+install(FILES ${PROTO_HEADERS} ${FLAT_HEADERS} mcap/include/osi_tracefile_mcap_reader.h mcap/include/osi_tracefile_mcap_writer.h
         DESTINATION "${OSI_INSTALL_INCLUDE_DIR}")
 
 # Install the export set for use with the install-tree

mcap/README.MD

@@ -0,0 +1,6 @@
+# Mcap Tracefile Support 
+This folder contains the official upstream MCAP c++ library as well as convince classes for c++ OSI trace file writer and reader support.
+
+# Work in Progress
+- This is a work in progress and just a proof of concept.
+- Major cleaning and refactoring is needed.

mcap/include/osi_tracefile_mcap_reader.h

@@ -0,0 +1,24 @@
+#ifndef MCAP_TRACEFILE_H
+#define MCAP_TRACEFILE_H
+#include <../mcap/include/mcap/mcap.hpp>
+#include <filesystem>
+
+namespace osi3
+{
+class TracefileMcapReader {
+public:
+    explicit TracefileMcapReader(std::filesystem::path mcap_file);
+    ~TracefileMcapReader() { reader_.close(); };
+    bool has_next() const;
+    std::pair<mcap::Message, mcap::ChannelPtr> read_next();
+private:
+    mcap::McapReader reader_;
+    std::unique_ptr<mcap::LinearMessageView> message_view_ = nullptr;
+    std::optional<mcap::LinearMessageView::Iterator> message_it_;
+};
+
+
+} // namespace osi3
+
+
+#endif //MCAP_TRACEFILE_H

mcap/include/osi_tracefile_mcap_writer.h

@@ -0,0 +1,27 @@
+#ifndef OSI_TRACEFILE_MCAP_WRITER_H
+#define OSI_TRACEFILE_MCAP_WRITER_H
+#include <../mcap/include/mcap/mcap.hpp>
+#include <filesystem>
+
+namespace osi3
+{
+class TracefileMcapWriter {
+public:
+    explicit TracefileMcapWriter(std::filesystem::path mcap_file);
+    ~TracefileMcapWriter() { writer_.close(); };
+    template <typename T> void write(T top_level_message);
+    void close() { writer_.close(); }
+
+private:
+    void addOSITopLevelMessagesAsSchemata();
+    void addCommonMetadata();
+    mcap::McapWriter writer_;
+    std::map<std::string, int> name_to_schema_ids_;
+
+};
+
+
+} // namespace osi3
+
+
+#endif //OSI_TRACEFILE_MCAP_WRITER_H

mcap/mcap/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Foxglove Technologies Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mcap/mcap/conanfile.py

@@ -0,0 +1,28 @@
+from conans import ConanFile, tools
+
+
+class McapConan(ConanFile):
+    name = "mcap"
+    version = "1.4.0"
+    url = "https://github.com/foxglove/mcap"
+    homepage = "https://github.com/foxglove/mcap"
+    description = "A C++ implementation of the MCAP file format"
+    license = "MIT"
+    topics = ("mcap", "serialization", "deserialization", "recording")
+
+    settings = ("os", "compiler", "build_type", "arch")
+    requires = ("lz4/1.9.4", "zstd/1.5.2")
+    generators = "cmake"
+
+    def validate(self):
+        tools.check_min_cppstd(self, "17")
+
+    def configure(self):
+        pass
+
+    def package(self):
+        self.copy(pattern="LICENSE", dst="licenses")
+        self.copy("include/*")
+
+    def package_id(self):
+        self.info.header_only()

mcap/mcap/include/mcap/crc32.hpp

@@ -0,0 +1,108 @@
+#include <array>
+#include <cstddef>
+#include <cstdint>
+
+namespace mcap::internal {
+
+/**
+ * Compute CRC32 lookup tables as described at:
+ * https://github.com/komrad36/CRC#option-6-1-byte-tabular
+ *
+ * An iteration of CRC computation can be performed on 8 bits of input at once. By pre-computing a
+ * table of the values of CRC(?) for all 2^8 = 256 possible byte values, during the final
+ * computation we can replace a loop over 8 bits with a single lookup in the table.
+ *
+ * For further speedup, we can also pre-compute the values of CRC(?0) for all possible bytes when a
+ * zero byte is appended. Then we can process two bytes of input at once by computing CRC(AB) =
+ * CRC(A0) ^ CRC(B), using one lookup in the CRC(?0) table and one lookup in the CRC(?) table.
+ *
+ * The same technique applies for any number of bytes to be processed at once, although the speed
+ * improvements diminish.
+ *
+ * @param Polynomial The binary representation of the polynomial to use (reversed, i.e. most
+ * significant bit represents x^0).
+ * @param NumTables The number of bytes of input that will be processed at once.
+ */
+template <size_t Polynomial, size_t NumTables>
+struct CRC32Table {
+private:
+  std::array<uint32_t, 256 * NumTables> table = {};
+
+public:
+  constexpr CRC32Table() {
+    for (uint32_t i = 0; i < 256; i++) {
+      uint32_t r = i;
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      r = ((r & 1) * Polynomial) ^ (r >> 1);
+      table[i] = r;
+    }
+    for (size_t i = 256; i < table.size(); i++) {
+      uint32_t value = table[i - 256];
+      table[i] = table[value & 0xff] ^ (value >> 8);
+    }
+  }
+
+  constexpr uint32_t operator[](size_t index) const {
+    return table[index];
+  }
+};
+
+inline uint32_t getUint32LE(const std::byte* data) {
+  return (uint32_t(data[0]) << 0) | (uint32_t(data[1]) << 8) | (uint32_t(data[2]) << 16) |
+         (uint32_t(data[3]) << 24);
+}
+
+static constexpr CRC32Table<0xedb88320, 8> CRC32_TABLE;
+
+/**
+ * Initialize a CRC32 to all 1 bits.
+ */
+static constexpr uint32_t CRC32_INIT = 0xffffffff;
+
+/**
+ * Update a streaming CRC32 calculation.
+ *
+ * For performance, this implementation processes the data 8 bytes at a time, using the algorithm
+ * presented at: https://github.com/komrad36/CRC#option-9-8-byte-tabular
+ */
+inline uint32_t crc32Update(const uint32_t prev, const std::byte* const data, const size_t length) {
+  // Process bytes one by one until we reach the proper alignment.
+  uint32_t r = prev;
+  size_t offset = 0;
+  for (; (uintptr_t(data + offset) & alignof(uint32_t)) != 0 && offset < length; offset++) {
+    r = CRC32_TABLE[(r ^ uint8_t(data[offset])) & 0xff] ^ (r >> 8);
+  }
+  if (offset == length) {
+    return r;
+  }
+
+  // Process 8 bytes (2 uint32s) at a time.
+  size_t remainingBytes = length - offset;
+  for (; remainingBytes >= 8; offset += 8, remainingBytes -= 8) {
+    r ^= getUint32LE(data + offset);
+    uint32_t r2 = getUint32LE(data + offset + 4);
+    r = CRC32_TABLE[0 * 256 + ((r2 >> 24) & 0xff)] ^ CRC32_TABLE[1 * 256 + ((r2 >> 16) & 0xff)] ^
+        CRC32_TABLE[2 * 256 + ((r2 >> 8) & 0xff)] ^ CRC32_TABLE[3 * 256 + ((r2 >> 0) & 0xff)] ^
+        CRC32_TABLE[4 * 256 + ((r >> 24) & 0xff)] ^ CRC32_TABLE[5 * 256 + ((r >> 16) & 0xff)] ^
+        CRC32_TABLE[6 * 256 + ((r >> 8) & 0xff)] ^ CRC32_TABLE[7 * 256 + ((r >> 0) & 0xff)];
+  }
+
+  // Process any remaining bytes one by one.
+  for (; offset < length; offset++) {
+    r = CRC32_TABLE[(r ^ uint8_t(data[offset])) & 0xff] ^ (r >> 8);
+  }
+  return r;
+}
+
+/** Finalize a CRC32 by inverting the output value. */
+inline uint32_t crc32Final(uint32_t crc) {
+  return crc ^ 0xffffffff;
+}
+
+}  // namespace mcap::internal

mcap/mcap/include/mcap/errors.hpp

@@ -0,0 +1,120 @@
+#pragma once
+
+#include <string>
+
+namespace mcap {
+
+/**
+ * @brief Status codes for MCAP readers and writers.
+ */
+enum class StatusCode {
+  Success = 0,
+  NotOpen,
+  InvalidSchemaId,
+  InvalidChannelId,
+  FileTooSmall,
+  ReadFailed,
+  MagicMismatch,
+  InvalidFile,
+  InvalidRecord,
+  InvalidOpCode,
+  InvalidChunkOffset,
+  InvalidFooter,
+  DecompressionFailed,
+  DecompressionSizeMismatch,
+  UnrecognizedCompression,
+  OpenFailed,
+  MissingStatistics,
+  InvalidMessageReadOptions,
+  NoMessageIndexesAvailable,
+  UnsupportedCompression,
+};
+
+/**
+ * @brief Wraps a status code and string message carrying additional context.
+ */
+struct [[nodiscard]] Status {
+  StatusCode code;
+  std::string message;
+
+  Status()
+      : code(StatusCode::Success) {}
+
+  Status(StatusCode code)
+      : code(code) {
+    switch (code) {
+      case StatusCode::Success:
+        break;
+      case StatusCode::NotOpen:
+        message = "not open";
+        break;
+      case StatusCode::InvalidSchemaId:
+        message = "invalid schema id";
+        break;
+      case StatusCode::InvalidChannelId:
+        message = "invalid channel id";
+        break;
+      case StatusCode::FileTooSmall:
+        message = "file too small";
+        break;
+      case StatusCode::ReadFailed:
+        message = "read failed";
+        break;
+      case StatusCode::MagicMismatch:
+        message = "magic mismatch";
+        break;
+      case StatusCode::InvalidFile:
+        message = "invalid file";
+        break;
+      case StatusCode::InvalidRecord:
+        message = "invalid record";
+        break;
+      case StatusCode::InvalidOpCode:
+        message = "invalid opcode";
+        break;
+      case StatusCode::InvalidChunkOffset:
+        message = "invalid chunk offset";
+        break;
+      case StatusCode::InvalidFooter:
+        message = "invalid footer";
+        break;
+      case StatusCode::DecompressionFailed:
+        message = "decompression failed";
+        break;
+      case StatusCode::DecompressionSizeMismatch:
+        message = "decompression size mismatch";
+        break;
+      case StatusCode::UnrecognizedCompression:
+        message = "unrecognized compression";
+        break;
+      case StatusCode::OpenFailed:
+        message = "open failed";
+        break;
+      case StatusCode::MissingStatistics:
+        message = "missing statistics";
+        break;
+      case StatusCode::InvalidMessageReadOptions:
+        message = "message read options conflict";
+        break;
+      case StatusCode::NoMessageIndexesAvailable:
+        message = "file has no message indices";
+        break;
+      case StatusCode::UnsupportedCompression:
+        message = "unsupported compression";
+        break;
+      default:
+        message = "unknown";
+        break;
+    }
+  }
+
+  Status(StatusCode code, const std::string& message)
+      : code(code)
+      , message(message) {}
+
+  bool ok() const {
+    return code == StatusCode::Success;
+  }
+};
+
+}  // namespace mcap