HPACK lazy headers (#51)

-> lazy_headers_prepare -> separate encoder/decoder -> header_view -> separating HPACK into files -> HPACK optimized out
bot-motherlib · Oct 17, 2024 · d913634 · d913634
1 parent f1ba504
commit d913634
Show file tree

Hide file tree

Showing 22 changed files with 258 additions and 2,226 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -47,7 +47,6 @@ endif()
 
 set(TGBM_SRC_LIST
     src/net/http2_client.cpp
-    src/net/http2/huffman.cpp
     src/Api.cpp
     # src/api_types_cmp.cpp
     src/Bot.cpp
@@ -84,6 +83,7 @@ target_link_libraries(tgbmlib PUBLIC
     ssl crypto
     anyanylib
     kelcorolib
+    hpacklib
     Boost::system
     Boost::asio
     Boost::property_tree
@@ -133,3 +133,21 @@ endif()
 if (TGBM_ENABLE_BENCHS)
     add_subdirectory(gbench)
 endif()
+
+if (DEFINED TGBM_SSL_KEYS_FILE)
+  message (STATUS "[tgbm] TGBM_SSL_KEYS_FILE: ${TGBM_SSL_KEYS_FILE}")
+  target_compile_definitions(tgbmlib PUBLIC TGBM_SSL_KEYS_FILE="${TGBM_SSL_KEYS_FILE}")
+endif()
+
+if (DEFINED TGBM_SSL_ADDITIONAL_CERTIFICATE_PATH)
+  message (STATUS "[tgbm] TGBM_SSL_ADDITIONAL_CERTIFICATE_PATH: ${TGBM_SSL_ADDITIONAL_CERTIFICATE_PATH}")
+  target_compile_definitions(tgbmlib PUBLIC TGBM_SSL_ADDITIONAL_CERTIFICATE_PATH="${TGBM_SSL_ADDITIONAL_CERTIFICATE_PATH}")
+endif()
+
+if (WIN32)
+  if (NOT DEFINED TGBM_WINDOWS_VERSION) 
+      set(TGBM_WINDOWS_VERSION 0x0A00 CACHE STRING "windows version for boost asio")
+  endif()
+  message (STATUS "[tgbm] TGBM_WINDOWS_VERSION: ${TGBM_WINDOWS_VERSION}")
+  target_compile_definitions(tgbmlib PUBLIC _WIN32_WINNT=${TGBM_WINDOWS_VERSION})
+endif()
diff --git a/cmake/deps.cmake b/cmake/deps.cmake
@@ -48,6 +48,13 @@ CPMAddPackage(
   GIT_TAG        origin/master
 )
 
+CPMAddPackage(
+  NAME HPACK
+  GIT_REPOSITORY https://github.com/kelbon/HPACK
+  GIT_TAG        v1.0.0
+  OPTIONS "HPACK_ENABLE_TESTING ON"
+)
+
 if (TGBM_ENABLE_TESTS)
   CPMAddPackage(
     NAME googletest

diff --git a/include/tgbm/net/asio_awaiters.hpp b/include/tgbm/net/asio_awaiters.hpp
@@ -81,7 +81,7 @@ struct asio_awaiter<void, CallbackUser> {
   io_error_code& _ec;
 
   template <typename... Args>
-  explicit asio_awaiter(io_error_code& ec [[clang::lifetimebound]], Args&&... args) noexcept
+  explicit asio_awaiter(io_error_code& ec KELCORO_LIFETIMEBOUND, Args&&... args) noexcept
       : _cb_user(std::forward<Args>(args)...), _ec(ec) {
   }
 
@@ -173,26 +173,26 @@ struct read_operation {
   }
 };
 
-template <typename Stream, typename Buffer>
+template <typename Stream>
 struct read_some_operation {
   Stream& stream;
-  Buffer& buffer;
+  std::span<byte_t> buffer;
 
   template <typename T>
   void operator()(T&& cb) {
-    stream.async_read_some(buffer, std::forward<T>(cb));
+    stream.async_read_some(asio::buffer(buffer.data(), buffer.size()), std::forward<T>(cb));
   }
 };
 
 template <typename Stream, typename Buffer>
 struct read_until_operation {
   Stream& stream;
-  Buffer& buffer;
+  std::span<byte_t> buffer;
   std::string_view delim;
 
   template <typename T>
   void operator()(T&& cb) {
-    asio::async_read_until(stream, buffer, delim, std::forward<T>(cb));
+    asio::async_read_until(stream, asio::buffer(buffer.data(), buffer.size()), delim, std::forward<T>(cb));
   }
 };
 
@@ -218,9 +218,26 @@ struct shutdown_operation {
   }
 };
 
+template <typename Protocol>
+struct accept_operation {
+  asio::basic_socket_acceptor<Protocol>& acceptor;
+
+  template <typename T>
+  void operator()(T&& cb) const {
+    acceptor.async_accept(std::forward<T>(cb));
+  }
+};
+
 // boost asio grabs all possible good names for functions and creates 3042350 overloads
 // with all possible combinations of template arguments, tags etc, so this ignores adl
 struct net_t {
+  template <typename Protocol>
+  KELCORO_CO_AWAIT_REQUIRED static auto accept(asio::basic_socket_acceptor<Protocol>& acceptor
+                                                   KELCORO_LIFETIMEBOUND,
+                                               io_error_code& ec KELCORO_LIFETIMEBOUND) {
+    return asio_awaiter<asio::basic_stream_socket<Protocol>, accept_operation<Protocol>>(ec, acceptor);
+  }
+
   // returns resolve results
   template <typename Protocol>
   KELCORO_CO_AWAIT_REQUIRED static auto resolve(
@@ -278,11 +295,11 @@ struct net_t {
 
   // returns count of bytes transmitted
   // TODO return span to readen bytes? And what to accept? dynbuf + temp buf?
-  template <typename Stream, typename Buffer>
+  template <typename Stream>
   KELCORO_CO_AWAIT_REQUIRED static auto read_some(Stream& stream KELCORO_LIFETIMEBOUND,
-                                                  Buffer&& buffer KELCORO_LIFETIMEBOUND,
+                                                  std::span<byte_t> buffer KELCORO_LIFETIMEBOUND,
                                                   io_error_code& ec KELCORO_LIFETIMEBOUND) {
-    return asio_awaiter<size_t, read_some_operation<Stream, Buffer>>(ec, stream, buffer);
+    return asio_awaiter<size_t, read_some_operation<Stream>>(ec, stream, buffer);
   }
 
   // appends readen bytes to 'buffer'
@@ -294,16 +311,16 @@ struct net_t {
     assert(!needle.empty());
     assert(!ec);
     // TODO better
-    char buf[128];
+    byte_t buf[128];
     const size_t init_sz = buffer.size();
     size_t transfered = 0;
     size_t checked_bytes = 0;
     for (;;) {
-      size_t read_bytes = co_await net_t{}.read_some(stream, asio::buffer(buf), ec);
+      size_t read_bytes = co_await net_t{}.read_some(stream, buf, ec);
       if (ec)
         co_return needle.npos;
       transfered += read_bytes;
-      buffer += std::string_view(buf, read_bytes);  // TODO better
+      buffer += std::string_view((char*)buf, read_bytes);  // TODO better
       if (transfered < needle.size())
         continue;
       size_t s = needle.size() - 1;
@@ -322,9 +339,9 @@ struct net_t {
                                              auto condition, io_error_code& ec KELCORO_LIFETIMEBOUND) {
     assert(!ec);
     // TODO better
-    unsigned char buf[128];
+    byte_t buf[128];
     for (;;) {
-      size_t read_bytes = co_await net_t{}.read_some(stream, asio::buffer(buf), ec);
+      size_t read_bytes = co_await net_t{}.read_some(stream, buf, ec);
       if (ec)
         co_return;
       buffer.insert(buffer.end(), buf, buf + read_bytes);  // TODO better

diff --git a/include/tgbm/net/http11_client.hpp b/include/tgbm/net/http11_client.hpp
@@ -18,14 +18,14 @@ struct http11_client : http_client {
   asio::io_context io_ctx;
   tcp_connection_options tcp_options;
   pool_t<tcp_connection_ptr> connections;
-  KELCORO_NO_UNIQUE_ADDRESS dd::this_thread_executor_t exe;
   size_t requests_in_progress = 0;
   bool stop_requested = false;
 
  public:
   explicit http11_client(size_t connections_max_count = 64, std::string_view host = "api.telegram.org",
                          tcp_connection_options = {});
 
+  // note: this client ignores timeouts, its propose to just work in any case
   dd::task<int> send_request(on_header_fn_ptr on_header, on_data_part_fn_ptr on_data_part, http_request,
                              duration_t timeout) override;