Merge pull request #606 from david-cermak/feat/mosquitto

[mosq]: Support for TCP listener mode of mosquitto broker on ESP32

.github/workflows/mosq__build.yml

@@ -0,0 +1,72 @@
+name: "mosq: build-tests"
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  build_mosq:
+    if: contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push'
+    name: Mosquitto build
+    strategy:
+      matrix:
+        idf_ver: ["latest", "release-v5.3"]
+    runs-on: ubuntu-22.04
+    container: espressif/idf:${{ matrix.idf_ver }}
+    env:
+      TEST_DIR: components/mosquitto/examples/broker
+      TARGET_TEST_DIR: build_esp32_default
+    steps:
+      - name: Checkout esp-protocols
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Build with IDF-${{ matrix.idf_ver }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          pip install idf-component-manager idf-build-apps --upgrade
+          python ci/build_apps.py ${TEST_DIR}
+          cd ${TEST_DIR}
+          ${GITHUB_WORKSPACE}/ci/clean_build_artifacts.sh `pwd`/${TARGET_TEST_DIR}
+          zip -qur artifacts.zip ${TARGET_TEST_DIR}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: mosq_target_esp32_${{ matrix.idf_ver }}
+          path: ${{ env.TEST_DIR }}/artifacts.zip
+          if-no-files-found: error
+
+  test_mosq:
+    # Skip running on forks since it won't have access to secrets
+    if: |
+      github.repository == 'espressif/esp-protocols' &&
+      ( contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push' )
+    name: Mosquitto target test
+    needs: build_mosq
+    strategy:
+      matrix:
+        idf_ver: ["latest", "release-v5.3"]
+    runs-on:
+      - self-hosted
+      - ESP32-ETHERNET-KIT
+    env:
+      TEST_DIR: components/mosquitto/examples/broker
+      TARGET_TEST_DIR: build_esp32_default
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: mosq_target_esp32_${{ matrix.idf_ver }}
+          path: ${{ env.TEST_DIR }}/ci/
+      - name: Run Test
+        working-directory: ${{ env.TEST_DIR }}
+        run: |
+          unzip ci/artifacts.zip -d ci
+          for dir in `ls -d ci/build_*`; do
+          rm -rf build sdkconfig.defaults
+          mv $dir build
+          python -m pytest --log-cli-level DEBUG --junit-xml=./results_esp32_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=esp32
+          done

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "components/asio/asio"]
 	path = components/asio/asio
 	url = https://github.com/espressif/asio
+[submodule "components/mosq/mosquitto"]
+	path = components/mosquitto/mosquitto
+	url = https://github.com/eclipse/mosquitto

.pre-commit-config.yaml

@@ -62,7 +62,7 @@ repos:
     hooks:
       - id: commit message scopes
         name: "commit message must be scoped with: mdns, modem, websocket, asio, mqtt_cxx, console, common, eppp, wifi_remote, tls_cxx"
-        entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|console|websocket|asio|mqtt_cxx|examples|eppp|wifi_remote|tls_cxx)\)\:)'
+        entry: '\A(?!(feat|fix|ci|bump|test|docs)\((mdns|modem|common|console|websocket|asio|mqtt_cxx|examples|eppp|wifi_remote|tls_cxx|mosq)\)\:)'
         language: pygrep
         args: [--multiline]
         stages: [commit-msg]

ci/check_copyright_config.yaml

@@ -47,6 +47,13 @@ asio_component:
     - Apache-2.0
     - BSL-1.0
 
+mosquitto_component:
+  include:
+    - 'components/mosquitto/port/**'
+  allowed_licenses:
+    - EPL-2.0
+    - Apache-2.0
+
 slim_modem_examples:
   include:
     - 'examples/esp_netif/slip_custom_netif/**'

components/mosquitto/.cz.yaml

@@ -0,0 +1,7 @@
+commitizen:
+  bump_message: 'bump(mosq): $current_version -> $new_version'
+  pre_bump_hooks: python ../../ci/changelog.py mosq
+  tag_format: mosq-v$version
+  version: 2.0.27
+  version_files:
+    - idf_component.yml

components/mosquitto/CMakeLists.txt

@@ -0,0 +1,92 @@
+set(m_dir mosquitto)
+set(m_src_dir ${m_dir}/src)
+set(m_incl_dir ${m_dir}/include)
+set(m_lib_dir ${m_dir}/lib)
+set(m_deps_dir ${m_dir}/deps)
+
+set(m_srcs
+        ${m_lib_dir}/memory_mosq.c
+        ${m_lib_dir}/util_mosq.c
+        ${m_lib_dir}/net_mosq.c
+        ${m_lib_dir}/will_mosq.c
+        ${m_lib_dir}/alias_mosq.c
+        ${m_lib_dir}/send_mosq.c
+        ${m_lib_dir}/strings_mosq.c
+        ${m_lib_dir}/packet_mosq.c
+        ${m_lib_dir}/packet_datatypes.c
+        ${m_lib_dir}/property_mosq.c
+        ${m_lib_dir}/util_topic.c
+        ${m_lib_dir}/send_publish.c
+        ${m_lib_dir}/send_disconnect.c
+        ${m_lib_dir}/handle_pubackcomp.c
+        ${m_lib_dir}/handle_pubrec.c
+        ${m_lib_dir}/handle_pubrel.c
+        ${m_lib_dir}/handle_ping.c
+        ${m_lib_dir}/time_mosq.c
+        ${m_lib_dir}/utf8_mosq.c
+
+        ${m_src_dir}/bridge.c
+        ${m_src_dir}/bridge_topic.c
+        ${m_src_dir}/conf_includedir.c
+        ${m_src_dir}/context.c
+        ${m_src_dir}/control.c
+        ${m_src_dir}/database.c
+        ${m_src_dir}/handle_auth.c
+        ${m_src_dir}/handle_connack.c
+        ${m_src_dir}/handle_connect.c
+        ${m_src_dir}/handle_disconnect.c
+        ${m_src_dir}/handle_publish.c
+        ${m_src_dir}/handle_subscribe.c
+        ${m_src_dir}/handle_unsubscribe.c
+        ${m_src_dir}/keepalive.c
+        ${m_src_dir}/logging.c
+        ${m_src_dir}/loop.c
+        ${m_src_dir}/memory_public.c
+        ${m_src_dir}/mosquitto.c
+        ${m_src_dir}/mux.c
+        ${m_src_dir}/mux_epoll.c
+        ${m_src_dir}/mux_poll.c
+        ${m_src_dir}/net.c
+        ${m_src_dir}/password_mosq.c
+        ${m_src_dir}/persist_read.c
+        ${m_src_dir}/persist_read_v234.c
+        ${m_src_dir}/persist_read_v5.c
+        ${m_src_dir}/persist_write.c
+        ${m_src_dir}/persist_write_v5.c
+        ${m_src_dir}/property_broker.c
+        ${m_src_dir}/read_handle.c
+        ${m_src_dir}/retain.c
+        ${m_src_dir}/security.c
+        ${m_src_dir}/security_default.c
+        ${m_src_dir}/send_auth.c
+        ${m_src_dir}/send_connack.c
+        ${m_src_dir}/send_suback.c
+        ${m_src_dir}/send_unsuback.c
+        ${m_src_dir}/service.c
+        ${m_src_dir}/session_expiry.c
+        ${m_src_dir}/signals.c
+        ${m_src_dir}/subs.c
+        ${m_src_dir}/sys_tree.c
+        ${m_src_dir}/topic_tok.c
+        ${m_src_dir}/websockets.c
+        ${m_src_dir}/will_delay.c
+        ${m_src_dir}/xtreport.c)
+
+idf_component_register(SRCS ${m_srcs}
+                            port/callbacks.c port/config.c port/signals.c port/ifaddrs.c port/broker.c port/files.c
+                    PRIV_INCLUDE_DIRS port/priv_include port/priv_include/sys ${m_dir} ${m_src_dir}
+                                      ${m_incl_dir} ${m_lib_dir} ${m_deps_dir}
+                    INCLUDE_DIRS ${m_incl_dir} port/include
+                    PRIV_REQUIRES newlib
+        )
+
+target_compile_definitions(${COMPONENT_LIB} PRIVATE "WITH_BROKER")
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
+
+# Some mosquittos source unconditionally define `_GNU_SOURCE` which collides with IDF build system
+# producing warning: "_GNU_SOURCE" redefined
+# This workarounds this issue by undefining the macro for the selected files
+set(sources_that_define_gnu_source ${m_lib_dir}/net_mosq.c ${m_src_dir}/loop.c ${m_src_dir}/mux_poll.c)
+foreach(offending_src ${sources_that_define_gnu_source})
+    set_source_files_properties(${offending_src} PROPERTIES COMPILE_OPTIONS "-U_GNU_SOURCE")
+endforeach()

components/mosquitto/Doxyfile

@@ -0,0 +1,25 @@
+# Set this to the include directory of your component
+INPUT = port/include
+STRIP_FROM_PATH = ./port/include
+
+# Output goes into doxygen directory, which is added to gitignore
+OUTPUT_DIRECTORY = doxygen
+
+# Warning-related settings, it's recommended to keep them enabled
+WARN_AS_ERROR = YES
+
+# Other common settings
+RECURSIVE = YES
+FULL_PATH_NAMES = YES
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = YES
+PREDEFINED             = $(ENV_DOXYGEN_DEFINES)
+HAVE_DOT = NO
+GENERATE_XML    = YES
+XML_OUTPUT      = xml
+GENERATE_HTML   = NO
+HAVE_DOT        = NO
+GENERATE_LATEX  = NO
+QUIET = YES
+MARKDOWN_SUPPORT = YES

components/mosquitto/LICENSE.txt

@@ -0,0 +1,2 @@
+This project is dual licensed under the Eclipse Public License 2.0 and the
+Eclipse Distribution License 1.0 as described in the epl-v20 and edl-v10 files.

components/mosquitto/README.md

@@ -0,0 +1,32 @@
+# ESP32 Mosquitto Port
+
+This is a lightweight port of the Mosquitto broker designed to run on the ESP32. It currently supports a single listener and TCP transport only.
+
+## Supported Options
+
+The Espressif port supports a limited set of options (with plans to add more in future releases). These options can be configured through a structure passed to the `mosq_broker_start()` function. For detailed information on available configuration options, refer to the [API documentation](api.md).
+
+## API
+
+### Starting the Broker
+
+To start the broker, call the `mosq_broker_start()` function with a properly configured settings structure. The broker operates in the context of the calling task and does not create a separate task.
+
+It's recommended to analyze the stack size needed for the task, but in general, the broker requires at least 4 kB of stack size.
+
+```c
+mosq_broker_start(&config);
+```
+
+## Memory Footprint Considerations
+
+The broker primarily uses the heap for internal data, with minimal use of static/BSS memory. It consumes approximately 60 kB of program memory.
+
+- **Initial Memory Usage**: ~2 kB of heap on startup
+- **Per Client Memory Usage**: ~4 kB of heap for each connected client
+
+When using the broker with multiple clients, ensure sufficient heap space is available to handle scenarios where clients disconnect abruptly and reconnect. In cases of clean disconnections, the broker releases the memory immediately. However, when a client loses connection abruptly, the broker retains its connection information for some time before eventually freeing this memory. If the client reconnects during this period, an additional 4 kB of heap is allocated for the new connection while the memory for the previous connection is still being retained. Therefore, extra heap space is necessary to manage these abrupt reconnections smoothly.
+
+## Testing
+
+Extensive long-term tests have been conducted with the broker handling 5 clients simultaneously. Each client publishes messages every second to a different topic, while all clients subscribe to all topics. The test also included abrupt disconnections, where clients were suspended and reconnected after 10 seconds, to simulate real-world network instability.

components/mosquitto/api.md

@@ -0,0 +1,65 @@
+# API Reference
+
+## Header files
+
+- [mosq_broker.h](#file-mosq_brokerh)
+
+## File mosq_broker.h
+
+
+
+
+
+## Structures and Types
+
+| Type | Name |
+| ---: | :--- |
+| struct | [**mosq\_broker\_config**](#struct-mosq_broker_config) <br>_Mosquitto configuration structure._ |
+
+## Functions
+
+| Type | Name |
+| ---: | :--- |
+|  int | [**mosq\_broker\_start**](#function-mosq_broker_start) (struct [**mosq\_broker\_config**](#struct-mosq_broker_config) \*config) <br>_Start mosquitto broker._ |
+
+
+## Structures and Types Documentation
+
+### struct `mosq_broker_config`
+
+_Mosquitto configuration structure._
+
+ESP port of mosquittto supports only the options in this configuration structure.
+
+Variables:
+
+-  char \* host  <br>Address on which the broker is listening for connections
+
+-  int port  <br>Port number of the broker to listen to
+
+
+## Functions Documentation
+
+### function `mosq_broker_start`
+
+_Start mosquitto broker._
+```c
+int mosq_broker_start (
+    struct mosq_broker_config *config
+)
+```
+
+
+This API runs the broker in the calling thread and blocks until the mosquitto exits.
+
+
+
+**Parameters:**
+
+
+* `config` Mosquitto configuration structure
+
+
+**Returns:**
+
+int Exit code (0 on success)

components/mosquitto/edl-v10

@@ -0,0 +1,30 @@
+Eclipse Distribution License - v 1.0
+
+Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+  Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+  Neither the name of the Eclipse Foundation, Inc. nor the names of its
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.