MINIFICPP-2715 - Use symbols to check api compatibility

Extensions.md

@@ -17,8 +17,41 @@
 To enable all extensions for your platform, you may use -DENABLE_ALL=TRUE OR select the option to "Enable all Extensions" in the bootstrap script. [ReadMe](https://github.com/apache/nifi-minifi-cpp/#bootstrapping)
 
 # Extension internals
-Extensions are dynamic libraries loaded at runtime by the agent. An extension makes its
-capabilities (classes) available to the system through registrars. Registration must happen in source files, not headers.
+Extensions are dynamic libraries loaded at runtime by the agent.
+
+## C extensions
+You can build a shared library depending on the C capabilities of the agent as given in the `minifi-c.h` file.
+For the shared library to be considered a valid extension, it has to have a global symbol with the name `MinifiCApiVersion`
+with its value as a null terminated string (`const char*`) of the macro `MINIFI_API_VERSION` from `minifi-c.h`.
+
+Moreover the actual resource registration (processors/controller services) has to happen during the `MinifiInitExtension` call.
+One possible example of this is:
+
+```C++
+extern "C" const char* const MinifiApiVersion = MINIFI_API_VERSION;
+
+extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* /*config*/) {
+  MinifiExtension* extension = nullptr;
+  minifi::api::core::useProcessorClassDescription<minifi::extensions::llamacpp::processors::RunLlamaCppInference>([&] (const MinifiProcessorClassDefinition& description) {
+    MinifiExtensionCreateInfo ext_create_info{
+      .name = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_NAME)),
+      .version = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_VERSION)),
+      .deinit = nullptr,
+      .user_data = nullptr,
+      .processors_count = 1,
+      .processors_ptr = &description,
+    };
+    extension = MinifiCreateExtension(&ext_create_info);
+  });
+  return extension;
+}
+```
+
+## C++ extensions
+You can utilize the C++ api, linking to `minifi-api` and possibly using the helpers in `extension-framework`.
+No compatibilities are guaranteed beyond what extensions are built together with the agent at the same time.
+
+An extension makes its capabilities (classes) available to the system through registrars. Registration must happen in source files, not headers.
 
 ```C++
 // register user-facing classes as
@@ -33,10 +66,10 @@ REGISTER_RESOURCE(RESTSender, DescriptionOnly);
 ```
 
 Some extensions (e.g. `OpenCVExtension`) require initialization before use.
-You need to define an `InitExtension` function of type `MinifiExtension*(MinifiConfig*)` to be called.
+You need to define an `MinifiInitCppExtension` function of type `MinifiExtension*(MinifiConfig*)` to be called.
 
 ```C++
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" MinifiExtension* MinifiInitCppExtension(MinifiConfig* /*config*/) {
   const auto success = org::apache::nifi::minifi::utils::Environment::setEnvironmentVariable("OPENCV_FFMPEG_CAPTURE_OPTIONS", "rtsp_transport;udp", false /*overwrite*/);
   if (!success) {
     return nullptr;
@@ -49,7 +82,7 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  return MinifiCreateCppExtension(&ext_create_info);
 }
 ```
 

cmake/Extensions.cmake

@@ -22,29 +22,14 @@ define_property(GLOBAL PROPERTY EXTENSION-OPTIONS
 
 set_property(GLOBAL PROPERTY EXTENSION-OPTIONS "")
 
-set(extension-build-info-file "${CMAKE_CURRENT_BINARY_DIR}/ExtensionBuildInfo.cpp")
-file(GENERATE OUTPUT ${extension-build-info-file}
-    CONTENT "\
-    #include \"minifi-cpp/utils/Export.h\"\n\
-    #ifdef BUILD_ID_VARIABLE_NAME\n\
-    EXTENSIONAPI extern const char* const BUILD_ID_VARIABLE_NAME = \"__EXTENSION_BUILD_IDENTIFIER_BEGIN__${BUILD_IDENTIFIER}__EXTENSION_BUILD_IDENTIFIER_END__\";\n\
-    #else\n\
-    static_assert(false, \"BUILD_ID_VARIABLE_NAME is not defined\");\n\
-    #endif\n")
-
-function(get_build_id_variable_name extension-name output)
-    string(REPLACE "-" "_" result ${extension-name})
-    string(APPEND result "_build_identifier")
-    set("${output}" "${result}" PARENT_SCOPE)
-endfunction()
-
 macro(register_extension extension-name extension-display-name extension-guard description)
     set(${extension-guard} ${extension-name} PARENT_SCOPE)
     get_property(extensions GLOBAL PROPERTY EXTENSION-OPTIONS)
     set_property(GLOBAL APPEND PROPERTY EXTENSION-OPTIONS ${extension-name})
-    get_build_id_variable_name(${extension-name} build-id-variable-name)
-    set_source_files_properties(${extension-build-info-file} PROPERTIES GENERATED TRUE)
-    target_sources(${extension-name} PRIVATE ${extension-build-info-file})
+    get_target_property(has_custom_initializer ${extension-name} HAS_CUSTOM_INITIALIZER)
+    if (NOT has_custom_initializer)
+        target_sources(${extension-name} PRIVATE ${CMAKE_SOURCE_DIR}/extensions/ExtensionInitializer.cpp)
+    endif()
     target_compile_definitions(${extension-name}
         PRIVATE "MODULE_NAME=${extension-name}"
         PRIVATE "BUILD_ID_VARIABLE_NAME=${build-id-variable-name}")

extension-framework/include/utils/ExtensionInitUtils.h

@@ -30,4 +30,8 @@ inline MinifiStringView toStringView(std::string_view str) {
 
 using ConfigReader = std::function<std::optional<std::string>(std::string_view key)>;
 
+static inline MinifiExtension* MinifiCreateCppExtension(const MinifiExtensionCreateInfo* create_info) {
+  return MINIFI_CREATE_EXTENSION_FN(create_info);
+}
+
 }  // namespace org::apache::nifi::minifi::utils

extensions/ExtensionInitializer.cpp

@@ -0,0 +1,35 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "minifi-c/minifi-c.h"
+#include "utils/ExtensionInitUtils.h"
+#include "minifi-cpp/agent/agent_version.h"
+#include "core/Resource.h"
+
+namespace minifi = org::apache::nifi::minifi;
+
+extern "C" MinifiExtension* MinifiInitCppExtension(MinifiConfig* /*config*/) {
+  MinifiExtensionCreateInfo ext_create_info{
+    .name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
+    .version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
+    .deinit = nullptr,
+    .user_data = nullptr,
+    .processors_count = 0,
+    .processors_ptr = nullptr
+  };
+  return minifi::utils::MinifiCreateCppExtension(&ext_create_info);
+}

extensions/llamacpp/processors/ExtensionInitializer.cpp

@@ -24,7 +24,9 @@
 
 namespace minifi = org::apache::nifi::minifi;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" const char* const MinifiApiVersion = MINIFI_API_VERSION;
+
+extern "C" MinifiExtension* MinifiInitExtension(MinifiConfig* /*config*/) {
   MinifiExtension* extension = nullptr;
   minifi::api::core::useProcessorClassDescription<minifi::extensions::llamacpp::processors::RunLlamaCppInference>([&] (const MinifiProcessorClassDefinition& description) {
     MinifiExtensionCreateInfo ext_create_info{
@@ -35,9 +37,7 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
       .processors_count = 1,
       .processors_ptr = &description,
     };
-    extension = MinifiCreateExtension(minifi::api::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+    extension = MinifiCreateExtension(&ext_create_info);
   });
   return extension;
 }
-
-extern const char* const MINIFI_API_VERSION_TAG_var = MINIFI_API_VERSION_TAG;

extensions/opencv/CMakeLists.txt

@@ -29,6 +29,7 @@ include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
 file(GLOB SOURCES  "*.cpp")
 
 add_minifi_library(minifi-opencv SHARED ${SOURCES})
+set_target_properties(minifi-opencv PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
 
 target_link_libraries(minifi-opencv ${LIBMINIFI})
 target_link_libraries(minifi-opencv OPENCV::libopencv)

extensions/opencv/OpenCVLoader.cpp

@@ -24,7 +24,7 @@
 
 namespace minifi = org::apache::nifi::minifi;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" MinifiExtension* MinifiInitCppExtension(MinifiConfig* /*config*/) {
   // By default in OpenCV, ffmpeg capture is hardcoded to use TCP and this is a workaround
   // also if UDP timeout, ffmpeg will retry with TCP
   // Note:
@@ -42,5 +42,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  return minifi::utils::MinifiCreateCppExtension(&ext_create_info);
 }

extensions/python/CMakeLists.txt

@@ -25,12 +25,14 @@ include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
 
 if (NOT WIN32)
     add_minifi_library(minifi-python-lib-loader-extension SHARED pythonlibloader/PythonLibLoader.cpp)
+    set_target_properties(minifi-python-lib-loader-extension PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
     target_link_libraries(minifi-python-lib-loader-extension PRIVATE ${LIBMINIFI})
 endif()
 
 file(GLOB SOURCES "*.cpp" "types/*.cpp" "pythonloader/PyProcLoader.cpp")
 
 add_minifi_library(minifi-python-script-extension SHARED ${SOURCES})
+set_target_properties(minifi-python-script-extension PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
 
 target_link_libraries(minifi-python-script-extension PRIVATE ${LIBMINIFI} Threads::Threads)
 

extensions/python/pythonlibloader/PythonLibLoader.cpp

@@ -98,7 +98,7 @@ class PythonLibLoader {
   std::shared_ptr<minifi::core::logging::Logger> logger_ = minifi::core::logging::LoggerFactory<PythonLibLoader>::getLogger();
 };
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
+extern "C" MinifiExtension* MinifiInitCppExtension(MinifiConfig* config) {
   static PythonLibLoader python_lib_loader([&] (std::string_view key) -> std::optional<std::string> {
     std::optional<std::string> result;
     MinifiConfigGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
@@ -114,5 +114,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  return minifi::utils::MinifiCreateCppExtension(&ext_create_info);
 }

extensions/python/pythonloader/PyProcLoader.cpp

@@ -33,7 +33,7 @@ static minifi::extensions::python::PythonCreator& getPythonCreator() {
 // the symbols of the python library
 extern "C" const int LOAD_MODULE_AS_GLOBAL = 1;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
+extern "C" MinifiExtension* MinifiInitCppExtension(MinifiConfig* config) {
   getPythonCreator().configure([&] (std::string_view key) -> std::optional<std::string> {
     std::optional<std::string> result;
     MinifiConfigGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
@@ -49,5 +49,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  return minifi::utils::MinifiCreateCppExtension( &ext_create_info);
 }

extensions/sftp/CMakeLists.txt

@@ -31,6 +31,7 @@ include_directories(client processors)
 file(GLOB SOURCES  "*.cpp" "client/*.cpp" "processors/*.cpp")
 
 add_minifi_library(minifi-sftp SHARED ${SOURCES})
+set_target_properties(minifi-sftp PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
 
 target_link_libraries(minifi-sftp ${LIBMINIFI} Threads::Threads)
 target_link_libraries(minifi-sftp libssh2 RapidJSON)

extensions/sftp/SFTPLoader.cpp

@@ -25,7 +25,7 @@
 
 namespace minifi = org::apache::nifi::minifi;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" MinifiExtension* MinifiInitCppExtension(MinifiConfig* /*config*/) {
   if (libssh2_init(0) != 0) {
     return nullptr;
   }
@@ -44,5 +44,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  return minifi::utils::MinifiCreateCppExtension(&ext_create_info);
 }

libminifi/include/core/extension/Extension.h

@@ -17,11 +17,12 @@
 
 #pragma once
 
-#include <memory>
+#include <filesystem>
 #include <map>
+#include <memory>
 #include <string>
-#include <filesystem>
 
+#include "minifi-c/minifi-c.h"
 #include "minifi-cpp/core/logging/Logger.h"
 #include "minifi-cpp/properties/Configure.h"
 
@@ -38,6 +39,12 @@ class Extension {
     void* user_data;
   };
 
+  struct Version {
+    int major{MINIFI_API_MAJOR_VERSION};
+    int minor{MINIFI_API_MINOR_VERSION};
+    int patch{MINIFI_API_PATCH_VERSION};
+  };
+
   Extension(std::string name, std::filesystem::path library_path);
 
   Extension(const Extension&) = delete;
@@ -49,6 +56,8 @@ class Extension {
 
   bool initialize(const std::shared_ptr<minifi::Configure>& configure);
 
+  Version version() const {return version_;}
+
  private:
 #ifdef WIN32
   std::map<void*, std::string> resource_mapping_;
@@ -71,6 +80,7 @@ class Extension {
   std::filesystem::path library_path_;
   gsl::owner<void*> handle_ = nullptr;
 
+  Version version_{};
   std::unique_ptr<Info> info_;
 
   const std::shared_ptr<logging::Logger> logger_;

libminifi/include/core/extension/ExtensionManager.h

@@ -31,6 +31,8 @@ class ExtensionManager {
  public:
   explicit ExtensionManager(const std::shared_ptr<Configure>& config);
 
+  static Extension* getExtensionBeingInitialized();
+
  private:
   std::vector<std::unique_ptr<Extension>> extensions_;
 

libminifi/include/core/extension/Utils.h

@@ -44,37 +44,11 @@ class Timer {
   Callback cb_;
 };
 
-enum LibraryType {
-  Cpp,
-  CApi,
-  Invalid
-};
-
 struct LibraryDescriptor {
   std::string name;
   std::filesystem::path dir;
   std::string filename;
 
-
-  [[nodiscard]] bool verify_as_cpp_extension() const;
-
-  [[nodiscard]] bool verify_as_c_extension(const std::shared_ptr<logging::Logger>& logger) const;
-
-  [[nodiscard]]
-  LibraryType verify(const std::shared_ptr<logging::Logger>& logger) const {
-    const auto path = getFullPath();
-    const Timer timer{[&](const std::chrono::milliseconds elapsed) {
-      logger->log_debug("Verification for '{}' took {}", path, elapsed);
-    }};
-    if (verify_as_cpp_extension()) {
-      return Cpp;
-    }
-    if (verify_as_c_extension(logger)) {
-      return CApi;
-    }
-    return Invalid;
-  }
-
   [[nodiscard]]
   std::filesystem::path getFullPath() const {
     return dir / filename;