DRAFT: Adiak Integration

docs/AdiakIntegration.rst

@@ -0,0 +1,90 @@
+..
+   # Copyright 2021 Lawrence Livermore National Security, LLC and other
+   # PerfFlowAspect Project Developers. See the top-level LICENSE file for
+   # details.
+   #
+   # SPDX-License-Identifier: LGPL-3.0
+
+###################
+ Adiak Integration
+###################
+
+PerfFlowAspect can be built with Adiak, a tool that collects metadata on HPC
+runs. Learn more about Adiak `here <https://github.com/LLNL/Adiak>`_.
+
+PerfFlowAspect integration with Adiak is currently only supported in C/C++. When
+integrated, the generated PerfFlowAspect `.pfw` trace file contains Adiak
+metadata. `Note: the trace file must be generated in object format.`
+
+Adiak must be installed prior to PerfFlowAspect integration.
+
+*************
+ C/C++ Build
+*************
+
+Adiak is enabled by specifying ``PERFFLOWASPECT_WITH_ADIAK=On`` along with the
+path to Adiak's package configuration file, i.e.
+``<installpath>/lib/cmake/adiak>``.
+
+.. code:: bash
+
+   cmake -DCMAKE_C_COMPILER=<path-to-clang20-compiler> \
+         -DCMAKE_CXX_COMPILER=<path-to-clang20++-compiler> \
+         -DPERFFLOWASPECT_WITH_ADIAK=On \
+         -Dadiak_DIR=<installpath>/lib/cmake/adiak> ../
+
+Adiak can gather additional metadata related to MPI. The flag
+``PERFFLOWASPECT_WITH_MPI=On`` must be specified.
+
+.. code:: bash
+
+   cmake -DCMAKE_C_COMPILER=<path-to-clang20-compiler> \
+         -DCMAKE_CXX_COMPILER=<path-to-clang20++-compiler> \
+         -DPERFFLOWASPECT_WITH_ADIAK=On \
+         -Dadiak_DIR=<installpath>/lib/cmake/adiak> \
+         -DPERFFLOWASPECT_WITH_MPI=On ../
+
+***************
+ C/C++ Example
+***************
+
+Adiak's metadata can only be displayed in the object format of a PerfFlowAspect
+`.pfw` trace. Thus, ``PERFFLOW_OPTIONS="log-format=Object"`` must be specified
+
+.. code:: bash
+
+   PERFFLOW_OPTIONS="log-format=Object" ./smoketest
+
+**********************
+ Output Demonstration
+**********************
+
+The following is a snippet of running the CUDA smoketest on LLNL's Matrix
+Cluster. The Adiak metadata appears in the Chrome Trace Format otherData
+section.
+
+.. code:: json
+
+   {
+   "displayTimeUnit": "us",
+   "otherData": {
+      "walltime": {"tv_sec": 0, "tv_usec": 903734},
+      "systime": {"tv_sec": 0, "tv_usec": 370000},
+      "cputime": {"tv_sec": 0, "tv_usec": 40000},
+      "numhosts": 1,
+      "jobsize": 1,
+      "cluster": "matrix",
+      "hostname": "matrix10",
+      "working_directory": "/g/g14/greene36/PerfFlowAspect/src/c/build/test",
+      "executablepath": "/g/g14/greene36/PerfFlowAspect/src/c/build/test/smoketest_cuda",
+      "executable": "smoketest_cuda",
+      "launchday": 1761091200,
+      "launchdate": 1761148496,
+      "uid": "Spencer Greene",
+      "user": "greene36",
+      "adiakversion": "0.4.1"
+
+   },
+   "traceEvents": [
+   {"name": "main", "cat": "/g/g14/greene36/PerfFlowAspect/src/c/test/smoketest_cuda_wrapper.cpp", "pid": 3220242, "tid": 3220242, "ts": 1761148496078466.0, "ph": "B"},
+   {"name": "main", "cat": "/g/g14/greene36/PerfFlowAspect/src/c/test/smoketest_cuda_wrapper.cpp", "pid": 3220242, "tid": 3220242, "ts": 1761148496968874.0, "ph": "E"}, ...

docs/index.rst

@@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled.
 
    BuildingPerfFlowAspect
    Annotations
+   AdiakIntegration
    CaliperIntegration
    UpcomingFeatures
 

src/c/CMakeLists.txt

@@ -14,6 +14,8 @@ else()
     message(FATAL_ERROR "Unsupported CXX compiler: please use Clang == 20.0")
 endif()
 
+option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" OFF)
+
 include(cmake/CMakeBasics.cmake)
 
 include(cmake/Setup3rdParty.cmake)

src/c/cmake/Setup3rdParty.cmake

@@ -20,6 +20,21 @@ if(PERFFLOWASPECT_WITH_MULTITHREADS)
     include(cmake/thirdparty/FindThreads.cmake)
 endif()
 
+if(PERFFLOWASPECT_WITH_ADIAK)
+    if(NOT adiak_DIR)
+        message(FATAL_ERROR "PFA + Adiak needs explicit adiak_DIR")
+    endif()
+
+    if (adiak_DIR)
+        message(STATUS "${adiak_DIR}")
+        include(cmake/thirdparty/FindAdiak.cmake)
+    endif()
+
+    if (ADIAK_FOUND)
+        add_definitions(-DPERFFLOWASPECT_WITH_ADIAK)
+    endif()
+endif()
+
 if(PERFFLOWASPECT_WITH_CALIPER)
     # first Check for CALIPER_DIR
     if(NOT caliper_DIR)

src/c/cmake/thirdparty/FindAdiak.cmake

@@ -0,0 +1,12 @@
+message(STATUS "Looking for Adiak in ${adiak_DIR}")
+find_package(adiak REQUIRED 
+			PATHS ${adiak_DIR}/lib/cmake/adiak
+			NO_DEFAULT_PATH
+			NO_CMAKE_ENVIRONMENT_PATH
+			NO_CMAKE_PATH
+			NO_SYSTEM_ENVIRONMENT_PATH
+			NO_CMAKE_SYSTEM_PATH)
+
+message(STATUS "FOUND Adiak: ${adiak_DIR}")
+set(ADIAK_FOUND TRUE)
+set(PERFFLOWASPECT_ADIAK_ENABLED TRUE)

src/c/host-configs/matrix-4.18.0-x86_64-intel-classic-2021.6.0@clang@19.1.7.cmake

@@ -0,0 +1,12 @@
+##############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+set(CMAKE_C_COMPILER "/usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/icc" CACHE PATH "")
+set(CMAKE_CXX_COMPILER "/usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/icpc" CACHE PATH "")

src/c/runtime/CMakeLists.txt

@@ -14,6 +14,9 @@ set(perfflow_runtime_sources
 
 include_directories(${JANSSON_INCLUDE_DIRS})
 
+if(PERFFLOWASPECT_WITH_ADIAK)
+    include_directories(${adiak_INCLUDE_DIR})
+endif()
 if(PERFFLOWASPECT_WITH_CALIPER)
     include_directories(${caliper_INCLUDE_DIR})
 endif()
@@ -23,6 +26,13 @@ add_library(perfflow_runtime SHARED
             ${perfflow_runtime_headers}
 )
 
+if (PERFFLOWASPECT_WITH_ADIAK)
+    target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY}
+                         OpenSSL::SSL OpenSSL::Crypto adiak::adiak)
+else()
+    target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY}
+                         OpenSSL::SSL OpenSSL::Crypto )
+endif()
 if(PERFFLOWASPECT_WITH_CALIPER)
     target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto caliper)
 else()

src/c/runtime/advice_chrome_tracing.cpp

@@ -228,6 +228,27 @@ int advice_chrome_tracing_t::flush_if(size_t size)
                     m_ofs << "{" << std::endl;
                     m_ofs << "  \"displayTimeUnit\": \"us\"," << std::endl;
                     m_ofs << "  \"otherData\": {" << std::endl;
+#ifdef PERFFLOWASPECT_WITH_ADIAK
+                    {
+                        const char *key;
+                        json_t *val;
+                        json_t *metadata = get_adiak_statistics();
+                        size_t total = json_object_size(metadata);
+                        size_t idx = 0;
+                        json_object_foreach(metadata, key, val)
+                        {
+                            char *v = json_dumps(val, JSON_ENCODE_ANY);
+                            m_ofs << "    \"" << key << "\": " << v;
+                            free(v);
+                            if (++idx < total)
+                            {
+                                m_ofs << ",";
+                            }
+                            m_ofs << "\n";
+                        }
+                        json_decref(metadata);
+                    }
+#endif
                     m_ofs << "" << std::endl;
                     m_ofs << "  }," << std::endl;
                     m_ofs << "  \"traceEvents\": [" << std::endl;
@@ -663,6 +684,60 @@ long advice_chrome_tracing_t::get_memory_usage()
     return max_ram_usage;
 }
 
+#ifdef PERFFLOWASPECT_WITH_ADIAK
+void advice_chrome_tracing_t::adiak_cb(const char *name, int cat,
+                                       const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque)
+{
+    json_t *obj = static_cast<json_t *>(opaque);
+    json_t *metadata = nullptr;
+
+    switch (t->dtype)
+    {
+        case adiak_version:
+        case adiak_string:
+        case adiak_catstring:
+        case adiak_path:
+            metadata = json_string(reinterpret_cast<const char*>(val->v_ptr));
+            break;
+        case adiak_long:
+        case adiak_date:
+            metadata = json_integer(val->v_long);
+            break;
+        case adiak_ulong:
+            metadata = json_integer(unsigned(val->v_long));
+            break;
+        case adiak_double:
+            metadata = json_real(val->v_double);
+            break;
+        case adiak_int:
+            metadata = json_integer(val->v_int);
+            break;
+        case adiak_uint:
+            metadata = json_integer(unsigned(val->v_int));
+            break;
+        case adiak_timeval:
+        {
+            struct timeval *tv = (struct timeval *)(val->v_ptr);
+            json_t *tv_obj = json_object();
+            json_object_set_new(tv_obj, "tv_sec",  json_integer(tv->tv_sec));
+            json_object_set_new(tv_obj, "tv_usec", json_integer(tv->tv_usec));
+            metadata = tv_obj;
+            break;
+        }
+        default:
+            return;
+    }
+    json_object_set_new(obj, name, metadata);
+}
+
+json_t *advice_chrome_tracing_t::get_adiak_statistics()
+{
+    json_t *adiak_metadata = json_object();
+    adiak_list_namevals(1, adiak_category_all, adiak_cb, adiak_metadata);
+    return adiak_metadata;
+}
+#endif
+
 int advice_chrome_tracing_t::with_flow(const char *module,
                                        const char *function,
                                        const char *flow,
@@ -747,6 +822,7 @@ int advice_chrome_tracing_t::before(const char *module,
     std::string fname;
     long mem_start;
 
+
     if (m_enable_logging)
     {
         if ((rc = create_event(&event, module, function, my_ts)) < 0)
@@ -1022,6 +1098,7 @@ int advice_chrome_tracing_t::after(const char *module,
 
         free(json_str);
         json_decref(event);
+
         return flush_if(FLUSH_SIZE);
     }
     else

src/c/runtime/advice_chrome_tracing.hpp

@@ -16,6 +16,10 @@
 #include <string>
 #include "advice_base.hpp"
 
+#ifdef PERFFLOWASPECT_WITH_ADIAK
+#include <adiak_tool.h>
+#endif
+
 class advice_chrome_tracing_t : public advice_base_t
 {
 public:
@@ -43,6 +47,11 @@ class advice_chrome_tracing_t : public advice_base_t
     double get_wall_time();
     double get_cpu_time();
     long get_memory_usage();
+#ifdef PERFFLOWASPECT_WITH_ADIAK
+    json_t *get_adiak_statistics();
+    static void adiak_cb(const char *name, int cat, const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque);
+
+#endif
 
     int cannonicalize_perfflow_options ();
     int parse_perfflow_options ();

src/c/test/CMakeLists.txt

@@ -8,10 +8,18 @@ set(SMOKETESTS
 set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto)
 
 message(STATUS "Adding CXX unit tests")
+
+if(PERFFLOWASPECT_WITH_ADIAK)
+    message(STATUS "Including Adiak in test dependencies")
+    set(perfflow_deps "${perfflow_deps}" adiak::adiak)
+endif()
+
 foreach(TEST ${SMOKETESTS})
     message(STATUS " [*] Adding test: ${TEST}")
     add_executable(${TEST} ${TEST}.cpp)
 	set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC")
+    target_link_libraries(${TEST} ${perfflow_deps})
+    message(STATUS "${perfflow_deps}")
     if(PERFFLOWASPECT_WITH_CALIPER)
         target_link_libraries(${TEST} ${perfflow_deps} caliper)
     else()
@@ -34,6 +42,8 @@ else()
     option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" OFF)
 endif()
 
+
+
 if(PERFFLOWASPECT_WITH_MULTITHREADS)
     message(STATUS " [*] Adding test: smoketest_MT")
     add_executable(smoketest_MT smoketest_MT.cpp)
@@ -60,8 +70,12 @@ endif()
 
 if(PERFFLOWASPECT_WITH_CUDA)
     message(STATUS " [*] Adding test: smoketest_cuda")
-	set(CUDA_NVCC_FLAGS  "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so -Xcompiler=-fpass-plugin=../../../weaver/weave/libWeavePass.so")
-    cuda_add_executable(smoketest_cuda smoketest_cuda_wrapper.cpp smoketest_cuda_kernel.cu)
+    set(CUDA_NVCC_FLAGS  "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so -Xcompiler=-fpass-plugin=../../../weaver/weave/libWeavePass.so")
+    cuda_add_executable(smoketest_cuda
+        smoketest_cuda_wrapper.cpp
+        smoketest_cuda_kernel.cu
+    )
+    set_source_files_properties(smoketest_cuda_wrapper.cpp COMPILE_FLAGS "-Xclang -load -Xclang ${CMAKE_BINARY_DIR}/weaver/weave/libWeavePass.so -fpass-plugin=${CMAKE_BINARY_DIR}/weaver/weave/libWeavePass.so  -fPIC")
     if(PERFFLOWASPECT_WITH_CALIPER)
         target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES} caliper)
     else()

src/c/test/smoketest_cuda_wrapper.cpp

@@ -10,6 +10,7 @@
 
 #include "smoketest_cuda_kernel.cuh"
 
+__attribute__((annotate("@critical_path(pointcut='around')")))
 int main(int argc, char *argv[])
 {
     Wrapper::wrapper();

src/c/weaver/weave/CMakeLists.txt

@@ -16,6 +16,13 @@ set_target_properties(WeavePass PROPERTIES
 
 target_link_libraries(WeavePass perfflow_parser ${JANSSON_LIB})
 
+if(PERFFLOWASPECT_WITH_MPI)
+    include_directories(${MPI_C_INCLUDE_PATH})
+    target_link_libraries(WeavePass ${MPI_C_LIBRARIES})
+    target_compile_definitions(WeavePass PRIVATE PERFFLOWASPECT_WITH_MPI)
+endif()
+
+
 add_library(WeavePassPlugin INTERFACE)
 target_compile_options(WeavePassPlugin INTERFACE
     "SHELL:$<BUILD_INTERFACE:-Xclang -load -Xclang $<TARGET_FILE:WeavePass>>$<INSTALL_INTERFACE:-Xclang -load -Xclang $<TARGET_FILE:perfflowaspect::WeavePass>>"