40 qnn direct

.clang-tidy

@@ -54,12 +54,16 @@ CheckOptions:
     value: 'camelBack'
   - key: 'readability-identifier-naming.ClassMemberCase'
     value: 'camelBack'
+  - key: 'readability-identifier-naming.ClassMemberPrefix'
+    value: 'm_'
   - key: 'readability-identifier-naming.ClassMethodCase'
     value: 'camelBack'
   - key: 'readability-identifier-naming.ConstantCase'
     value: 'camelBack'
   - key: 'readability-identifier-naming.ConstantMemberCase'
     value: 'camelBack'
+  - key: 'readability-identifier-naming.ConstantMemberPrefix'
+    value: 'm_'
   - key: 'readability-identifier-naming.ConstantParameterCase'
     value: 'camelBack'
   - key: 'readability-identifier-naming.ConstantPointerParameterCase'
@@ -69,11 +73,11 @@ CheckOptions:
   - key: 'readability-identifier-naming.ConstexprMethodCase'
     value: 'camelBack'
   - key: 'readability-identifier-naming.ConstexprVariableCase'
-    value: 'camelBack'
+    value: 'CamelCase'
   - key: 'readability-identifier-naming.EnumCase'
     value: 'CamelCase'
   - key: 'readability-identifier-naming.EnumConstantCase'
-    value: 'CamelCase'
+    value: 'UPPER_CASE'
   - key: 'readability-identifier-naming.FunctionCase'
     value: 'camelBack'
   - key: 'readability-identifier-naming.GlobalConstantCase'
@@ -133,15 +137,15 @@ CheckOptions:
   - key: 'readability-identifier-naming.StaticVariableCase'
     value: 'camelBack'
   - key: 'readability-identifier-naming.StructCase'
-    value: 'camelBack'
+    value: 'CamelCase'
   - key: 'readability-identifier-naming.TemplateParameterCase'
     value: 'CamelCase'
   - key: 'readability-identifier-naming.TemplateTemplateParameterCase'
     value: 'CamelCase'
   - key: 'readability-identifier-naming.TypeAliasCase'
-    value: 'camelBack'
+    value: 'CamelCase'
   - key: 'readability-identifier-naming.TypedefCase'
-    value: 'camelBack'
+    value: 'CamelCase'
   - key: 'readability-identifier-naming.TypeTemplateParameterCase'
     value: 'CamelCase'
   - key: 'readability-identifier-naming.UnionCase'

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
       run: pip3 install codespell
 
     - name: Lint
-      run: cmake -D FORMAT_COMMAND=clang-format-14 -P cmake/lint.cmake
+      run: cmake -D FORMAT_COMMAND=clang-format-15 -P cmake/lint.cmake
 
     - name: Spell check
       if: always()

CMakeLists.txt

@@ -73,19 +73,29 @@ target_link_libraries(edgerunner_edgerunner PUBLIC nonstd::span-lite)
 find_package(tensorflowlite REQUIRED)
 target_link_libraries(edgerunner_edgerunner PRIVATE tensorflow::tensorflowlite)
 
-if(ANDROID AND edgerunner_ENABLE_NPU)
-    find_package(qnn REQUIRED COMPONENTS tfliteDelegate)
-    target_link_libraries(edgerunner_edgerunner PUBLIC qnn::tfliteDelegate)
-
-    target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_QNN)
+if(edgerunner_ENABLE_GPU)
+    target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_GPU)
+endif()
 
-    file(COPY "${CMAKE_BINARY_DIR}/../runtimeLibs/"
-         DESTINATION ${CMAKE_BINARY_DIR}
+if(edgerunner_ENABLE_NPU)
+    target_sources(
+        edgerunner_edgerunner
+        PRIVATE source/qnn/model.cpp source/qnn/tensor.cpp
+                source/qnn/backend.cpp
     )
-endif()
 
-if(edgerunner_ENABLE_GPU)
-    target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_GPU)
+    find_package(qnn REQUIRED)
+
+    if(ANDROID)
+        target_link_libraries(edgerunner_edgerunner PRIVATE qnn::tflite)
+        target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_QNN)
+
+        file(COPY "${CMAKE_BINARY_DIR}/../runtimeLibs/"
+             DESTINATION ${CMAKE_BINARY_DIR}
+        )
+    else()
+        target_link_libraries(edgerunner_edgerunner PRIVATE qnn::qnn)
+    endif()
 endif()
 
 # ---- Install rules ----

README.md

@@ -93,7 +93,7 @@ Please request additional features through Github issues or on our [Discord](htt
 
 | TFLite | CoreML | Onnx | QNN | OpenVino | Ryzen AI | NeuroPilot |
 |:------:|:------:|:----:|:---:|:--------:|:--------:|:----------:|
-|   ✅   |   ⏳   |  ⏳  | ⏳  |    ⏳    |    ⏳    |     ⏳     |
+|   ✅   |   ⏳   |  ⏳  | ✅  |    ⏳    |    ⏳    |     ⏳     |
 
 ### Chip Vendor
 

conanfile.py

@@ -59,7 +59,7 @@ def requirements(self):
         if self.options.examples:
             self.requires("opencv/4.9.0")
 
-        if self.settings.os == "Android" and self.options.with_npu:
+        if self.options.with_npu:
             self.requires("qnn/2.23.0.24.06.24")
 
     def build_requirements(self):
@@ -92,12 +92,12 @@ def generate(self):
 
         toolchain.generate()
 
-        if self.settings.os == "Android" and self.options.with_npu:
+        if self.options.with_npu:
             qnn = self.dependencies["qnn"]
             copy(
                 self,
                 "*.so",
-                qnn.cpp_info.components["tfliteDelegate"].libdirs[0],
+                qnn.cpp_info.components["tflite"].libdirs[0],
                 os.path.join(self.source_folder, "build", "runtimeLibs"),
             )
             copy(

example/imageClassifier.hpp

@@ -5,6 +5,7 @@
 #include <fstream>
 #include <limits>
 #include <numeric>
+#include <ratio>
 #include <string>
 
 #include <fmt/core.h>
@@ -19,7 +20,6 @@ class ImageClassifier {
   public:
     ImageClassifier(const std::filesystem::path& modelPath,
                     const std::filesystem::path& labelListPath);
-
     auto loadImage(const std::filesystem::path& imagePath) -> edge::STATUS;
 
     auto setDelegate(edge::DELEGATE delegate) -> edge::STATUS;
@@ -76,7 +76,7 @@ class ImageClassifier {
 };
 
 inline ImageClassifier::ImageClassifier(
-    const std::filesystem::path& modelPath,
+    const std::filesystem::path& modelPath, /* NOLINT */
     const std::filesystem::path& labelListPath)
     : m_model(edge::createModel(modelPath))
     , m_labelList(loadLabelList(labelListPath)) {}

example/mobilenet_v3_small.cpp

@@ -14,13 +14,13 @@ auto main() -> int {
     const std::filesystem::path modelPath {
         "models/tflite/mobilenet_v3_small.tflite"};
     const std::filesystem::path labelListPath {
-        "models/tflite/imagenet_labels.txt"};
+        "models/common/imagenet_labels.txt"};
 
     ImageClassifier imageClassifier(modelPath, labelListPath);
 
-#ifdef EDGERUNNER_QNN
+#if defined(EDGERUNNER_QNN)
     imageClassifier.setDelegate(edge::DELEGATE::NPU);
-#elif EDGERUNNER_GPU
+#elif defined(EDGERUNNER_GPU)
     imageClassifier.setDelegate(edge::DELEGATE::GPU);
 #endif
 

include/edgerunner/model.hpp

@@ -70,16 +70,26 @@ class EDGERUNNER_EXPORT Model {
     /**
      * @brief Pure virtual function to load a model from a file path.
      *
+     * This function is a pure virtual function that must be implemented by any
+     * derived classes. It is used to load a model from a file path.
+     *
      * @param modelPath The path to the model file
+     * @return STATUS The status of the model loading operation
      */
-    virtual void loadModel(const std::filesystem::path& modelPath) = 0;
+    virtual auto loadModel(const std::filesystem::path& modelPath)
+        -> STATUS = 0;
 
     /**
      * @brief Pure virtual function to load a model from a file buffer.
      *
+     * This function is a pure virtual function that must be implemented by any
+     * derived classes. It is used to load a model from a file buffer.
+     *
      * @param modelBuffer The buffer containing the model
+     * @return STATUS The status of the model loading operation
      */
-    virtual void loadModel(const nonstd::span<uint8_t>& modelBuffer) = 0;
+    virtual auto loadModel(const nonstd::span<uint8_t>& modelBuffer)
+        -> STATUS = 0;
 
     /**
      * @brief Get the number of input tensors in the model.
@@ -168,14 +178,40 @@ class EDGERUNNER_EXPORT Model {
      */
     auto name() const -> const std::string& { return m_name; }
 
+    /**
+     * @brief Get the status of model creation.
+     *
+     * Verify that the model was created successfully
+     *
+     * @return The status of model creation
+     */
+    auto getCreationStatus() const -> STATUS { return m_creationStatus; }
+
   protected:
     /**
      * @brief Set the delegate for model execution.
      *
+     * This method is used by derivatives to allow users to query the currently
+     * set delegate
+     *
      * @param delegate The delegate to set
      */
     void setDelegate(const DELEGATE& delegate) { m_delegate = delegate; }
 
+    /**
+     * @brief Set the status of model creation.
+     *
+     * This method is used by derivatives to allow querying of model creation
+     * status
+     *
+     * @param status The status to set
+     */
+    void setCreationStatus(const STATUS& status) {
+        if (m_creationStatus == STATUS::SUCCESS) {
+            m_creationStatus = status;
+        }
+    }
+
   private:
     EDGERUNNER_SUPPRESS_C4251
     std::string m_name; /**< Name of the model */
@@ -191,6 +227,9 @@ class EDGERUNNER_EXPORT Model {
     EDGERUNNER_SUPPRESS_C4251
     DELEGATE m_delegate =
         DELEGATE::CPU; /**< Delegate used for model execution */
+
+    EDGERUNNER_SUPPRESS_C4251
+    STATUS m_creationStatus = STATUS::SUCCESS; /**< Status of model creation */
 };
 
 inline auto Model::getInput(size_t index) const -> std::shared_ptr<Tensor> {

include/edgerunner/qnn/backend.hpp

@@ -0,0 +1,139 @@
+/**
+ * @file Backend.h
+ * @brief Definition of the Backend class for handling QNN backends.
+ *
+ * This class represents a backend for handling interfacing with QNN backend
+ * libraries. It provides functionality for loading the backend, creating a
+ * device, initializing the backend, and managing the context for QNN
+ * operations.
+ *
+ * The Backend class is currently restricted to NPU inference support.
+ */
+
+#pragma once
+
+#include <unordered_map>
+
+#include <HTP/QnnHtpDevice.h>
+#include <QnnCommon.h>
+#include <QnnInterface.h>
+#include <QnnTypes.h>
+#include <System/QnnSystemInterface.h>
+#include <fmt/core.h>
+
+/* TODO: move STATUS to dedicated header */
+#include "edgerunner/model.hpp"
+
+namespace edge::qnn {
+
+/**
+ * @class Backend
+ * @brief Class for handling QNN backends.
+ */
+class Backend {
+  public:
+    /**
+     * @brief Constructor for the Backend class.
+     * @param delegate The delegate type for the backend (CPU, GPU, NPU).
+     */
+    explicit Backend(DELEGATE delegate);
+
+    Backend(const Backend&) = default;
+    Backend(Backend&&) = delete;
+    auto operator=(const Backend&) -> Backend& = delete;
+    auto operator=(Backend&&) -> Backend& = delete;
+
+    /**
+     * @brief Destructor for the Backend class.
+     */
+    ~Backend();
+
+    /**
+     * @brief Get the backend handle.
+     * @return Reference to the backend handle.
+     */
+    auto getHandle() -> auto& { return m_backendHandle; }
+
+    /**
+     * @brief Get the context for the backend.
+     * @return Reference to the backend context.
+     */
+    auto getContext() -> auto& { return m_context; }
+
+    /**
+     * @brief Get the QNN interface.
+     * @return Reference to the QNN interface.
+     */
+    auto getInterface() -> auto& { return m_qnnInterface; }
+
+    /**
+     * @brief Get the delegate type for the backend.
+     * @return The delegate type.
+     */
+    auto getDelegate() { return m_delegate; }
+
+    /**
+     * @brief Static callback function for logging.
+     * @param fmtStr The format string for the log message.
+     * @param level The log level.
+     * @param timestamp The timestamp of the log message.
+     * @param argp Additional arguments for the log message.
+     */
+    static void logCallback(const char* fmtStr,
+                            QnnLog_Level_t level,
+                            uint64_t timestamp,
+                            va_list argp);
+
+  private:
+    auto loadBackend() -> STATUS;
+
+    auto createLogger() -> STATUS;
+
+    auto initializeBackend() -> STATUS;
+
+    auto loadSystemLibrary() -> STATUS;
+
+    auto createDevice() -> STATUS;
+
+    auto createContext() -> STATUS;
+
+    auto setPowerConfig() -> STATUS;
+
+    auto destroyPowerConfig() const -> STATUS;
+
+    auto loadContextFromBinary() -> STATUS;
+
+    auto validateBackendId(uint32_t backendId) const -> STATUS;
+
+    void* m_backendLibHandle {};
+    void* m_systemLibHandle {};
+
+    Qnn_BackendHandle_t m_backendHandle {};
+    QnnBackend_Config_t** m_backendConfig {};
+
+    Qnn_DeviceHandle_t m_deviceHandle {};
+
+    Qnn_ContextHandle_t m_context {};
+
+    Qnn_LogHandle_t m_logHandle {};
+
+    uint32_t m_powerConfigId {};
+
+    QnnHtpDevice_PerfInfrastructure_t m_devicePerfInfrastructure {};
+
+    QNN_INTERFACE_VER_TYPE m_qnnInterface = QNN_INTERFACE_VER_TYPE_INIT;
+    QNN_SYSTEM_INTERFACE_VER_TYPE m_qnnSystemInterface =
+        QNN_SYSTEM_INTERFACE_VER_TYPE_INIT;
+
+    DELEGATE m_delegate;
+
+    std::unordered_map<DELEGATE, std::string> m_backendLibrariesByDelegate {
+        {DELEGATE::CPU, "libQnnCpu.so"},
+        {DELEGATE::GPU, "libQnnGpu.so"},
+        {DELEGATE::NPU, "libQnnHtp.so"}};
+
+    uint32_t m_deviceId {};
+    QnnHtpDevice_Arch_t m_htpArch {};
+};
+
+}  // namespace edge::qnn