Dev version of protobuf market order book protobuf timed data exports

.dockerignore

@@ -9,6 +9,7 @@ LICENSE
 **/*secret.json
 data/log
 data/secret
+data/serialized
 !data/secret/secret_test.json
 monitoring
 resources

.github/workflows/ubuntu-monitoring.yml → .github/workflows/ubuntu-special.yml

@@ -1,4 +1,4 @@
-name: Monitoring
+name: Special
 
 on:
   push:
@@ -7,14 +7,14 @@ on:
   pull_request:
 
 jobs:
-  ubuntu-monitoring-build:
-    name: Build on Ubuntu with monitoring support
+  ubuntu-special-build:
+    name: Build on Ubuntu with monitoring / protobuf support
     runs-on: ubuntu-latest
     strategy:
       matrix:
         compiler: [g++-11]
         buildmode: [Debug]
-        build-prometheus-from-source: [0, 1]
+        build-special-from-source: [0, 1]
 
     steps:
       - name: Checkout repository code
@@ -38,15 +38,15 @@ jobs:
           ninja
 
           sudo cmake --install .
-        if: matrix.build-prometheus-from-source == 0
+        if: matrix.build-special-from-source == 0
 
       - name: Create Build Environment
         run: cmake -E make_directory ${{github.workspace}}/build
 
       - name: Configure CMake
         working-directory: ${{github.workspace}}/build
         shell: bash
-        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-prometheus-from-source}} -GNinja
+        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{matrix.buildmode}} -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCCT_BUILD_PROMETHEUS_FROM_SRC=${{matrix.build-special-from-source}} -DCCT_ENABLE_PROTO=${{matrix.build-special-from-source}} -GNinja
 
       - name: Build
         working-directory: ${{github.workspace}}/build

.gitignore

@@ -66,6 +66,7 @@ cscope*
 data/cache
 data/log
 data/secret
+data/serialized
 data/static/exchangeconfig.json
 data/static/generalconfig.json
 monitoring/data/grafana/*

CMakeLists.txt

@@ -36,6 +36,7 @@ option(CCT_ENABLE_TESTS "Build the unit tests" ${MAIN_PROJECT})
 option(CCT_BUILD_EXEC "Build an executable instead of a static library" ${MAIN_PROJECT})
 option(CCT_ENABLE_ASAN "Compile with AddressSanitizer" ${CCT_ASAN_BUILD})
 option(CCT_ENABLE_CLANG_TIDY "Compile with clang-tidy checks" OFF)
+option(CCT_ENABLE_PROTO "Compile with protobuf support (to export data to the outside world)" ON)
 option(CCT_BUILD_PROMETHEUS_FROM_SRC "Fetch and build from prometheus-cpp sources" OFF)
 
 set(CCT_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/data" CACHE PATH "Needed data directory for coincenter. Can also be overriden at runtime with this environment variable")
@@ -83,6 +84,7 @@ if(CCT_ENABLE_TESTS)
   enable_testing()
 endif()
 
+# nlohmann_json - coincenter json library
 find_package(nlohmann_json CONFIG)
 if(NOT nlohmann_json_FOUND)
   FetchContent_Declare(
@@ -94,6 +96,7 @@ if(NOT nlohmann_json_FOUND)
   FetchContent_MakeAvailable(nlohmann_json)
 endif()
 
+# spdlog - coincenter logging library
 find_package(spdlog CONFIG)
 if(NOT spdlog_FOUND)
   FetchContent_Declare(
@@ -133,6 +136,34 @@ else()
   endif()
 endif()
 
+if(CCT_ENABLE_PROTO)
+  find_package(Protobuf CONFIG)
+  if(protobuf_FOUND)
+    message(STATUS "Linking with protobuf ${protobuf_VERSION}")
+  else()
+    set(PROTOBUF_VERSION v25.2)
+    if (MSVC)
+      # protobuf v25.0 does not compile yet with MSVC: https://github.com/protocolbuffers/protobuf/issues/14602
+      set(PROTOBUF_VERSION v24.4)
+    endif()
+
+    message(STATUS "Compiling protobuf ${PROTOBUF_VERSION} from sources")
+
+    set(protobuf_BUILD_TESTS OFF)
+    set(ABSL_PROPAGATE_CXX_STD ON)
+
+    FetchContent_Declare(
+      protobuf
+      GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+      GIT_TAG ${PROTOBUF_VERSION}
+    )
+    FetchContent_MakeAvailable(protobuf)
+
+    include(${protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+
+  endif()
+endif()
+
 # Unit Tests
 
 #[[ Create an executable
@@ -248,12 +279,18 @@ if(CCT_ENABLE_PROMETHEUS)
   add_compile_definitions(CCT_ENABLE_PROMETHEUS)
 endif()
 
+if(CCT_ENABLE_PROTO)
+  add_compile_definitions(CCT_ENABLE_PROTO)
+  add_compile_definitions("CCT_PROTOBUF_VERSION=\"${PROTOBUF_VERSION}\"")
+endif()
+
 # Link to sub folders CMakeLists.txt, from the lowest level to the highest level for documentation
 # (beware of cyclic dependencies)
 add_subdirectory(src/tech)
 add_subdirectory(src/monitoring)
 add_subdirectory(src/http-request)
 add_subdirectory(src/objects)
+add_subdirectory(src/serialization)
 add_subdirectory(src/api-objects)
 add_subdirectory(src/api)
 add_subdirectory(src/engine)

CONFIG.md

@@ -150,6 +150,7 @@ Refer to the hardcoded default json example as a model in case of doubt.
 | *query*     | **updateFrequency.depositWallet**  | Duration string (ex: `1min`)                                                     | Minimum duration between two consecutive requests of deposit information (including wallet)                                                                                                                                                                                                                                                                                                                              |
 | *query*     | **updateFrequency.currencyInfo**   | Duration string (ex: `4h`)                                                       | Minimum duration between two consecutive requests of dynamic currency info retrieval on Bithumb only (used for place order)                                                                                                                                                                                                                                                                                              |
 | *query*     | **placeSimulateRealOrder**         | Boolean (`true` or `false`)                                                      | If `true`, in trade simulation mode (with `--sim`) exchanges which do not support simulated mode in place order will actually place a real order, with the following characteristics: <ul><li>trade strategy forced to `maker`</li><li>price will be changed to a maximum for a sell, to a minimum for a buy</li></ul> This will allow place of a 'real' order that cannot be matched in practice (if it is, lucky you!) |
+| *query*     | **marketDataSerialization**        | Boolean (`true` or `false`)                                                      | If `true` and `coincenter` is compiled with **protobuf** support, some market data will automatically be exported in the `data/serialization` directory (`orderbook` and `last-trades`) for a long term storage                                                                                                                                                                                                          |
 | *query*     | **multiTradeAllowedByDefault**     | Boolean (`true` or `false`)                                                      | If `true`, [multi-trade](README.md#multi-trade) will be allowed by default for `trade`, `buy` and `sell`. It can be overridden at command line level with `--no-multi-trade` and `--multi-trade`.                                                                                                                                                                                                                        |
 | *query*     | **validateApiKey**                 | Boolean (`true` or `false`)                                                      | If `true`, each loaded private key will be tested at start of the program. In case of a failure, it will be removed from the list of private accounts loaded by `coincenter`, so that later queries do not consider it instead of raising a runtime exception. The downside is that it will make an additional check that will make startup slower.                                                                      |  |
 | *tradefees* | **maker**                          | String as decimal number representing a percentage (for instance, "0.15")        | Trade fees occurring when a maker order is matched                                                                                                                                                                                                                                                                                                                                                                       |

Dockerfile

@@ -23,12 +23,14 @@ ARG BUILD_MODE=Release
 ARG BUILD_TEST=0
 ARG BUILD_ASAN=0
 ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
 
 # Build and launch tests if any
 RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
     -DCCT_ENABLE_TESTS=${BUILD_TEST} \
     -DCCT_ENABLE_ASAN=${BUILD_ASAN} \
     -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+    -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
     -GNinja .. && \
     ninja && \
     if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \

alpine.Dockerfile

@@ -2,7 +2,7 @@
 FROM alpine:3.19.0 AS build
 
 # Install base & build dependencies, needed certificates for curl to work with https
-RUN apk add --update --upgrade --no-cache g++ libc-dev curl-dev cmake ninja git ca-certificates
+RUN apk add --update --upgrade --no-cache g++ linux-headers libc-dev curl-dev cmake ninja git ca-certificates
 
 # Set default directory for application
 WORKDIR /app
@@ -18,12 +18,14 @@ ARG BUILD_MODE=Release
 ARG BUILD_TEST=0
 ARG BUILD_ASAN=0
 ARG BUILD_WITH_PROMETHEUS=1
+ARG BUILD_WITH_PROTOBUF=1
 
 # Build and launch tests if any
 RUN cmake -DCMAKE_BUILD_TYPE=${BUILD_MODE} \
     -DCCT_ENABLE_TESTS=${BUILD_TEST} \
     -DCCT_ENABLE_ASAN=${BUILD_ASAN} \
     -DCCT_BUILD_PROMETHEUS_FROM_SRC=${BUILD_WITH_PROMETHEUS} \
+    -DCCT_ENABLE_PROTO=${BUILD_WITH_PROTOBUF} \
     -GNinja .. && \
     ninja && \
     if [ "$BUILD_TEST" = "1" -o "$BUILD_TEST" = "ON" ]; then \

src/api-objects/include/order.hpp

@@ -44,7 +44,7 @@ class Order {
   Market market() const { return Market(_matchedVolume.currencyCode(), _price.currencyCode()); }
 
   /// default ordering by place time first, then matched volume, etc
-  auto operator<=>(const Order &) const = default;
+  std::strong_ordering operator<=>(const Order &) const noexcept = default;
 
   using trivially_relocatable = is_trivially_relocatable<OrderId>::type;
 

src/api/exchanges/src/binancepublicapi.cpp

@@ -495,12 +495,12 @@ MarketOrderBook BinancePublic::OrderBookFunc::operator()(Market mk, int depth) {
   if (asksIt != asksAndBids.end() && bidsIt != asksAndBids.end()) {
     orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksIt->size() + bidsIt->size()));
     for (const auto& asksOrBids : {asksIt, bidsIt}) {
-      const bool isAsk = asksOrBids == asksIt;
+      const auto type = asksOrBids == asksIt ? OrderBookLine::Type::kAsk : OrderBookLine::Type::kBid;
       for (const auto& priceQuantityPair : *asksOrBids) {
         MonetaryAmount amount(priceQuantityPair.back().get<std::string_view>(), mk.base());
         MonetaryAmount price(priceQuantityPair.front().get<std::string_view>(), mk.quote());
 
-        orderBookLines.emplace_back(amount, price, isAsk);
+        orderBookLines.emplace_back(amount, price, type);
       }
     }
   }

src/api/exchanges/src/bithumbpublicapi.cpp

@@ -235,12 +235,12 @@ MarketOrderBookMap GetOrderbooks(CurlHandle& curlHandle, const CoincenterInfo& c
         OrderBookVec orderBookLines;
         orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksBids[0]->size() + asksBids[1]->size()));
         for (const json* asksOrBids : asksBids) {
-          const bool isAsk = asksOrBids == asksBids[0];
+          const auto type = asksOrBids == asksBids[0] ? OrderBookLine::Type::kAsk : OrderBookLine::Type::kBid;
           for (const json& priceQuantityPair : *asksOrBids) {
             MonetaryAmount amount(priceQuantityPair["quantity"].get<std::string_view>(), baseCurrencyCode);
             MonetaryAmount price(priceQuantityPair["price"].get<std::string_view>(), quoteCurrencyCode);
 
-            orderBookLines.emplace_back(amount, price, isAsk);
+            orderBookLines.emplace_back(amount, price, type);
           }
         }
         Market market(baseCurrencyCode, quoteCurrencyCode);

src/api/exchanges/src/huobipublicapi.cpp

@@ -389,15 +389,16 @@ MarketOrderBook HuobiPublic::OrderBookFunc::operator()(Market mk, int depth) {
     orderBookLines.reserve(bidsIt->size() + asksIt->size());
     for (const auto& asksOrBids : {bidsIt, asksIt}) {
       int currentDepth = 0;
-      bool isBid = asksOrBids == bidsIt;
+      const auto type = asksOrBids == asksIt ? OrderBookLine::Type::kAsk : OrderBookLine::Type::kBid;
       for (const auto& priceQuantityPair : *asksOrBids) {
         MonetaryAmount amount(priceQuantityPair.back().get<double>(), mk.base());
         MonetaryAmount price(priceQuantityPair.front().get<double>(), mk.quote());
 
-        orderBookLines.emplace_back(amount, price, !isBid);
+        orderBookLines.emplace_back(amount, price, type);
         if (++currentDepth == depth) {
           if (depth < static_cast<int>(asksOrBids->size())) {
-            log::debug("Truncate number of {} prices in order book to {}", isBid ? "bid" : "ask", depth);
+            log::debug("Truncate number of {} prices in order book to {}",
+                       type == OrderBookLine::Type::kAsk ? "ask" : "bid", depth);
           }
           break;
         }

src/api/exchanges/src/krakenpublicapi.cpp

@@ -309,15 +309,15 @@ MarketOrderBook KrakenPublic::OrderBookFunc::operator()(Market mk, int count) {
 
     orderBookLines.reserve(static_cast<OrderBookVec::size_type>(asksIt->size() + bidsIt->size()));
     for (const auto& asksOrBids : {asksIt, bidsIt}) {
-      const bool isAsk = asksOrBids == asksIt;
+      const auto type = asksOrBids == asksIt ? OrderBookLine::Type::kAsk : OrderBookLine::Type::kBid;
       for (const auto& priceQuantityTuple : *asksOrBids) {
         std::string_view priceStr = priceQuantityTuple[0].get<std::string_view>();
         std::string_view amountStr = priceQuantityTuple[1].get<std::string_view>();
 
         MonetaryAmount amount(amountStr, mk.base());
         MonetaryAmount price(priceStr, mk.quote());
 
-        orderBookLines.emplace_back(amount, price, isAsk);
+        orderBookLines.emplace_back(amount, price, type);
       }
     }
   }

src/api/exchanges/src/kucoinpublicapi.cpp

@@ -261,16 +261,18 @@ MarketOrderBookMap KucoinPublic::AllOrderBooksFunc::operator()(int depth) {
 
 namespace {
 template <class InputIt>
-void FillOrderBook(Market mk, int depth, bool isAsk, InputIt beg, InputIt end, vector<OrderBookLine>& orderBookLines) {
+void FillOrderBook(Market mk, int depth, OrderBookLine::Type type, InputIt beg, InputIt end,
+                   vector<OrderBookLine>& orderBookLines) {
   int currentDepth = 0;
   for (auto it = beg; it != end; ++it) {
     MonetaryAmount price((*it)[0].template get<std::string_view>(), mk.quote());
     MonetaryAmount amount((*it)[1].template get<std::string_view>(), mk.base());
 
-    orderBookLines.emplace_back(amount, price, isAsk);
+    orderBookLines.emplace_back(amount, price, type);
     if (++currentDepth == depth) {
       if (++it != end) {
-        log::debug("Truncate number of {} prices in order book to {}", isAsk ? "ask" : "bid", depth);
+        log::debug("Truncate number of {} prices in order book to {}",
+                   type == OrderBookLine::Type::kAsk ? "ask" : "bid", depth);
       }
       break;
     }
@@ -299,8 +301,8 @@ MarketOrderBook KucoinPublic::OrderBookFunc::operator()(Market mk, int depth) {
   if (asksIt != asksAndBids.end() && bidsIt != asksAndBids.end()) {
     orderBookLines.reserve(asksIt->size() + bidsIt->size());
     // Reverse iterate as bids are received in descending order
-    FillOrderBook(mk, depth, false, bidsIt->rbegin(), bidsIt->rend(), orderBookLines);
-    FillOrderBook(mk, depth, true, asksIt->begin(), asksIt->end(), orderBookLines);
+    FillOrderBook(mk, depth, OrderBookLine::Type::kBid, bidsIt->rbegin(), bidsIt->rend(), orderBookLines);
+    FillOrderBook(mk, depth, OrderBookLine::Type::kAsk, asksIt->begin(), asksIt->end(), orderBookLines);
   }
 
   return MarketOrderBook(Clock::now(), mk, orderBookLines);

src/api/exchanges/src/upbitpublicapi.cpp

@@ -201,8 +201,8 @@ MarketOrderBookMap ParseOrderBooks(const json& result, int depth) {
       MonetaryAmount askVol(orderbookDetails["ask_size"].get<double>(), base);
       MonetaryAmount bidVol(orderbookDetails["bid_size"].get<double>(), base);
 
-      orderBookLines.emplace_back(askVol, askPri, true /* isAsk */);
-      orderBookLines.emplace_back(bidVol, bidPri, false /* isAsk */);
+      orderBookLines.emplace_back(askVol, askPri, OrderBookLine::Type::kAsk);
+      orderBookLines.emplace_back(bidVol, bidPri, OrderBookLine::Type::kBid);
 
       if (static_cast<int>(orderBookLines.size() / 2) == depth) {
         // Upbit does not have a depth parameter, the only thing we can do is to truncate it manually

src/api/interface/CMakeLists.txt

@@ -3,6 +3,7 @@ aux_source_directory(src API_INTERFACE_SRC)
 
 add_library(coincenter_api-interface STATIC ${API_INTERFACE_SRC})
 target_link_libraries(coincenter_api-interface PUBLIC coincenter_api-exchange)
+target_link_libraries(coincenter_api-interface PUBLIC coincenter_serialization)
 target_link_libraries(coincenter_api-interface PRIVATE coincenter_monitoring)
 
 target_include_directories(coincenter_api-interface PUBLIC include)

src/api/interface/include/exchange.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <memory>
 #include <optional>
 #include <string_view>
+#include <type_traits>
 
 #include "cct_exception.hpp"
 #include "currencycode.hpp"
@@ -17,17 +19,20 @@
 #include "monetaryamountbycurrencyset.hpp"
 
 namespace cct {
+class AbstractMarketDataSerializer;
 class Exchange {
  public:
   using ExchangePublic = api::ExchangePublic;
 
   /// Builds a Exchange without private exchange. All private requests will be forbidden.
-  Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic);
+  Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic);
 
   /// Build a Exchange with both private and public exchanges
-  Exchange(const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+  Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
            api::ExchangePrivate &exchangePrivate);
 
+  ~Exchange();
+
   std::string_view name() const { return _exchangePublic.name(); }
   std::string_view keyName() const { return apiPrivate().keyName(); }
 
@@ -86,16 +91,12 @@ class Exchange {
     return _exchangePublic.queryAllApproximatedOrderBooks(depth);
   }
 
-  MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth) {
-    return _exchangePublic.queryOrderBook(mk, depth);
-  }
+  MarketOrderBook queryOrderBook(Market mk, int depth = ExchangePublic::kDefaultDepth);
 
   MonetaryAmount queryLast24hVolume(Market mk) { return _exchangePublic.queryLast24hVolume(mk); }
 
   /// Retrieve an ordered vector of recent last trades
-  LastTradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault) {
-    return _exchangePublic.queryLastTrades(mk, nbTrades);
-  }
+  LastTradesVector queryLastTrades(Market mk, int nbTrades = ExchangePublic::kNbLastTradesDefault);
 
   /// Retrieve the last price of given market.
   MonetaryAmount queryLastPrice(Market mk) { return _exchangePublic.queryLastPrice(mk); }
@@ -110,9 +111,15 @@ class Exchange {
 
   void updateCacheFile() const;
 
+  using trivially_relocatable = std::true_type;
+
  private:
+  Exchange(std::string_view dataDir, const ExchangeConfig &exchangeConfig, api::ExchangePublic &exchangePublic,
+           api::ExchangePrivate *pExchangePrivate);
+
   api::ExchangePublic &_exchangePublic;
   api::ExchangePrivate *_pExchangePrivate = nullptr;
   const ExchangeConfig &_exchangeConfig;
+  std::unique_ptr<AbstractMarketDataSerializer> _marketDataSerializerPtr;
 };
 }  // namespace cct