feat(aquadbg): more troubleshooting tooling (#27)

TROUBLESHOOTING.md

@@ -84,7 +84,7 @@ flowchart TD
   time. If this counter is rising, there might be connectivity issues or a different heatpump protocol using a
   different CRC mechanism.
 
-- Both counter values `msgHandled` and `msgSent` shall be rising over time if the connection is good. If these values stay at `0` there are connectivity issues.
+- The value `msgHandled` shall be rising over time if the connection is good. If this value stays at `0` there are most likely connectivity issues or the heat pump protocol is not (yet) supported.
 
 ### Q: AquaMQTT is connected to MQTT, but I don't see any values from the heatpump?
 
@@ -108,7 +108,7 @@ It seems that depending on your heat-pump brand and model there are sometimes fo
 ### Q: Why do I need the HMI controller? Do I have to connect the HMI controller?
 
 In both of AquaMQTT current implemented OperationModes `LISTENER` and `MITM` it is expected to have an HMI controller
-connected, since AquaMQTT is either just forwarding or altering the original messages emitted by the heat-pump
+connected, since AquaMQTT is either just listening or forwarding original or modified messages emitted by the heat-pump
 controllers.
 
 Technically it would be feasible to build another OperationMode without the need for the original HMI controller, but
@@ -146,4 +146,8 @@ There is a [python script](./tools) which subscribes all the debug topics and wr
 
 *The python script has to be modified to connect to your mqtt broker, with your mqtt broker credentials etc.*
 
-Afterwards you can inspect changing values over time and identify attributes by making changes to your HMI controller. If you identifiy something new which is not documented within [PROTOCOL.md](./PROTOCOL.md) yet, please create an issue or even an PR.
+Afterwards you can inspect changing values over time and identify attributes by making changes to your HMI controller. If you identifiy something new which is not documented within [PROTOCOL.md](./PROTOCOL.md) yet, please create an issue or even an PR.
+
+### Q: I think my heat-pump uses an incompatible serial protocol? What can I do?
+
+If you already have the AquaMQTT Board installed and available, you can set it to LISTENER Mode and install [AquaDebug](./tools/AquaDebug/) to collect unmodified serial data traces from the heat-pump controller. If there are no other hardware and or connectivity related issues you should be able to identify the [heat-pump messages](./PROTOCOL.md) within the raw data. If the messages differ from the expected protocol, open up an github issue to discuss further.

tools/AquaDebug/.clang-format

@@ -0,0 +1,137 @@
+# Configured for clang-format-9
+# If You want to test some setting, try https://zed0.co.uk/clang-format-configurator/
+# Integration helpers, see here: https://clang.llvm.org/docs/ClangFormat.html
+
+Language:        Cpp
+BasedOnStyle:  Google
+AccessModifierOffset: -4
+AllowAllConstructorInitializersOnNextLine: false
+AllowAllArgumentsOnNextLine : false
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: true
+AlignConsecutiveDeclarations: true
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AlignConsecutiveMacros: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterClass:      true
+  AfterControlStatement: true
+  AfterEnum:       true
+  AfterFunction:   true
+  AfterNamespace:  true
+  AfterObjCDeclaration: false
+  AfterStruct:     true
+  AfterUnion:      true
+  AfterExternBlock: false
+  BeforeCatch:     true
+  BeforeElse:      true
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+  AfterCaseLabel: true
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeComma
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+BreakInheritanceList: BeforeComma
+ColumnLimit:     119
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 8
+Cpp11BracedListStyle: false
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+  - std::for_each
+IncludeBlocks:   Regroup
+IncludeCategories:
+
+  - Regex:           '^<(gmock|gtest|Poco|rapidjson|eb)\/([A-Z0-9.\/-_])+'
+    Priority:        4
+
+  - Regex:           '^<([A-Z0-9\/-_])+>'
+    Priority:        3
+
+  - Regex:           '^<.*/.*\.hpp>'
+    Priority:        5
+
+  - Regex:           '^<.*.h>'
+    Priority:        1
+
+  - Regex:           '^<.*/.*\.h>'
+    Priority:        2
+
+  - Regex:           '^".*/.*\.hpp"'
+    Priority:        6
+
+  - Regex:           '^"([A-Z0-9.-_])+"'
+    Priority:        7
+
+IncludeIsMainRegex: '(_test)?$'
+IndentCaseLabels: true
+IndentPPDirectives: AfterHash
+IndentWidth:     4
+IndentWrappedFunctionNames: true
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 4
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PointerAlignment: Left
+RawStringFormats:
+  - Language: TextProto
+    BasedOnStyle: google
+    Delimiters: - pb
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: true
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        4
+UseTab:          Never
+
+PenaltyBreakAssignment: 200
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200

tools/AquaDebug/.gitignore

@@ -0,0 +1,2 @@
+.pio
+include/config/CustomConfiguration.h

tools/AquaDebug/README.md

@@ -0,0 +1,39 @@
+# AquaDebug
+
+AquaDebug is a simple serial bus debugging application meant to be flashed on the AquaMQTT Board for troubleshooting the
+heat-pumps serial protocol.
+
+- AquaDebug is sniffing the serial protocol without altering or interpreting any data
+- AquaDebug expects the AquaMQTT Board to be configured as LISTENER
+- AquaDebug reads the heat-pumps serial protocol and provides raw data to MQTT
+- A python script running on any machine collects the data from MQTT and writes the data into files
+
+In case you suspect that your heat-pump is incompatible with the existing AquaMQTT implementation, due to a different
+protocol, CRC or any other reason, you may try AquaDebug and provide a trace from your heat-pump communication as an
+github issue.
+
+## Getting Started
+
+Refer to the [AquaMQTT README](./../../AquaMQTT/README.md) for installation and configuration instructions.
+
+## Debugging / Troubleshooting
+
+As soon as AquaDebug has been flashed, it will publish mqtt messages to `$prefix/aquadbg/debug` in case values have been
+read from the Serial port.
+
+These data is collected by a little python script and stored on your machine.
+
+### Create the python environment
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip install paho-mqtt
+```
+
+### Run the python script
+
+```
+source venv/bin/activate
+python3 collect.py
+```

tools/AquaDebug/collect.py

@@ -0,0 +1,44 @@
+import os
+import time
+import datetime
+import paho.mqtt.client as mqtt
+
+last_payloads = {}
+
+def on_message(client, userdata, message):
+    topic = message.topic
+    payload = message.payload
+
+    hex_file_name = topic.replace('/', '_') + '_hex.csv'
+    dec_file_name = topic.replace('/', '_') + '_dec.csv'
+
+    hex_payload = payload.decode('utf-8')
+
+    if topic in last_payloads and last_payloads[topic] == hex_payload:
+        return
+
+    timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
+
+    with open(hex_file_name, 'a') as hex_file:
+        hex_file.write(timestamp + ',' + hex_payload + '\n')
+
+    dec_payload = ' '.join([str(int(payload[i:i+2], 16)) for i in range(0, len(payload), 2)])
+
+    with open(dec_file_name, 'a') as dec_file:
+        dec_file.write(timestamp + ',' + dec_payload + '\n')
+
+    last_payloads[topic] = hex_payload
+
+client = mqtt.Client()
+client.username_pw_set(username='your-broker-user', password='your-broker-pass')
+client.connect('your-broker-addr', 1883)
+
+client.subscribe('your-prefix/aquadbg/debug')
+
+client.on_message = on_message
+
+client.loop_start()
+
+while True:
+    time.sleep(1)
+

tools/AquaDebug/include/config/Configuration.h

@@ -0,0 +1,44 @@
+#ifndef AQUAMQTT_CONFIGURATION_H
+#define AQUAMQTT_CONFIGURATION_H
+
+/**
+ * Possibility to include your own configuration file (added to .gitignore)
+ */
+//#define CUSTOM_CONFIGURATION
+
+#ifdef CUSTOM_CONFIGURATION
+#    include "CustomConfiguration.h"
+#else
+#    include "ExampleConfiguration.h"
+#endif
+
+namespace aquamqtt
+{
+namespace config
+{
+/**
+ * Defines the network name of your esp32 device in your network
+ */
+constexpr char networkName[] = "aquamqtt";
+
+/**
+ * Self explanatory internal settings: most probably you don't want to change them.
+ */
+constexpr uint8_t  WATCHDOG_TIMEOUT_S    = 60;
+constexpr uint8_t  MQTT_MAX_TOPIC_SIZE   = 80;
+constexpr uint8_t  MQTT_MAX_PAYLOAD_SIZE = 255;
+
+/**
+ * Pin assignments for AquaMQTT Board Rev 1.0
+ */
+constexpr uint8_t GPIO_MAIN_RX = 2;
+constexpr uint8_t GPIO_MAIN_TX = 3;
+constexpr uint8_t GPIO_HMI_RX  = 4;
+constexpr uint8_t GPIO_HMI_TX  = 5;
+constexpr uint8_t GPIO_SDA_RTC = A4;
+constexpr uint8_t GPIO_SCL_RTC = A5;
+
+}  // namespace config
+}  // namespace aquamqtt
+
+#endif  // AQUAMQTT_CONFIGURATION_H

tools/AquaDebug/include/config/ExampleConfiguration.h

@@ -0,0 +1,22 @@
+#ifndef AQUAMQTT_EXAMPLECONFIGURATION_H
+#define AQUAMQTT_EXAMPLECONFIGURATION_H
+
+namespace aquamqtt
+{
+namespace config
+{
+
+constexpr char     ssid[]           = "ExampleSSID";
+constexpr char     psk[]            = "ExamplePSK";
+constexpr char     brokerAddr[]     = "192.168.188.1";
+constexpr uint16_t brokerPort       = 1883;
+constexpr char     brokerClientId[] = "aquamqtt";
+// leave blank if your broker does not require a username/password
+constexpr char brokerUser[]     = "";
+constexpr char brokerPassword[] = "";
+constexpr char mqttPrefix[]     = "";
+
+}  // namespace config
+}  // namespace aquamqtt
+
+#endif  // AQUAMQTT_EXAMPLECONFIGURATION_H

tools/AquaDebug/include/handler/OTA.h

@@ -0,0 +1,19 @@
+#ifndef AQUAMQTT_OTA_H
+#define AQUAMQTT_OTA_H
+
+namespace aquamqtt
+{
+class OTAHandler
+{
+public:
+    OTAHandler() = default;
+
+    virtual ~OTAHandler() = default;
+
+    void setup();
+
+    void loop();
+};
+}  // namespace aquamqtt
+
+#endif  // AQUAMQTT_OTA_H