diff --git a/docs/AdiakIntegration.rst b/docs/AdiakIntegration.rst new file mode 100644 index 00000000..2fa7dfd2 --- /dev/null +++ b/docs/AdiakIntegration.rst @@ -0,0 +1,90 @@ +.. + # Copyright 2021 Lawrence Livermore National Security, LLC and other + # PerfFlowAspect Project Developers. See the top-level LICENSE file for + # details. + # + # SPDX-License-Identifier: LGPL-3.0 + +################### + Adiak Integration +################### + +PerfFlowAspect can be built with Adiak, a tool that collects metadata on HPC +runs. Learn more about Adiak `here `_. + +PerfFlowAspect integration with Adiak is currently only supported in C/C++. When +integrated, the generated PerfFlowAspect `.pfw` trace file contains Adiak +metadata. `Note: the trace file must be generated in object format.` + +Adiak must be installed prior to PerfFlowAspect integration. + +************* + C/C++ Build +************* + +Adiak is enabled by specifying ``PERFFLOWASPECT_WITH_ADIAK=On`` along with the +path to Adiak's package configuration file, i.e. +``/lib/cmake/adiak>``. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_CXX_COMPILER= \ + -DPERFFLOWASPECT_WITH_ADIAK=On \ + -Dadiak_DIR=/lib/cmake/adiak> ../ + +Adiak can gather additional metadata related to MPI. The flag +``PERFFLOWASPECT_WITH_MPI=On`` must be specified. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_CXX_COMPILER= \ + -DPERFFLOWASPECT_WITH_ADIAK=On \ + -Dadiak_DIR=/lib/cmake/adiak> \ + -DPERFFLOWASPECT_WITH_MPI=On ../ + +*************** + C/C++ Example +*************** + +Adiak's metadata can only be displayed in the object format of a PerfFlowAspect +`.pfw` trace. Thus, ``PERFFLOW_OPTIONS="log-format=Object"`` must be specified + +.. code:: bash + + PERFFLOW_OPTIONS="log-format=Object" ./smoketest + +********************** + Output Demonstration +********************** + +The following is a snippet of running the CUDA smoketest on LLNL's Matrix +Cluster. The Adiak metadata appears in the Chrome Trace Format otherData +section. + +.. code:: json + + { + "displayTimeUnit": "us", + "otherData": { + "walltime": {"tv_sec": 0, "tv_usec": 903734}, + "systime": {"tv_sec": 0, "tv_usec": 370000}, + "cputime": {"tv_sec": 0, "tv_usec": 40000}, + "numhosts": 1, + "jobsize": 1, + "cluster": "matrix", + "hostname": "matrix10", + "working_directory": "/g/g14/greene36/PerfFlowAspect/src/c/build/test", + "executablepath": "/g/g14/greene36/PerfFlowAspect/src/c/build/test/smoketest_cuda", + "executable": "smoketest_cuda", + "launchday": 1761091200, + "launchdate": 1761148496, + "uid": "Spencer Greene", + "user": "greene36", + "adiakversion": "0.4.1" + + }, + "traceEvents": [ + {"name": "main", "cat": "/g/g14/greene36/PerfFlowAspect/src/c/test/smoketest_cuda_wrapper.cpp", "pid": 3220242, "tid": 3220242, "ts": 1761148496078466.0, "ph": "B"}, + {"name": "main", "cat": "/g/g14/greene36/PerfFlowAspect/src/c/test/smoketest_cuda_wrapper.cpp", "pid": 3220242, "tid": 3220242, "ts": 1761148496968874.0, "ph": "E"}, ... diff --git a/docs/index.rst b/docs/index.rst index 1d0fa902..1853f376 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled. BuildingPerfFlowAspect Annotations + AdiakIntegration CaliperIntegration UpcomingFeatures diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt index 57225b41..041a947f 100644 --- a/src/c/CMakeLists.txt +++ b/src/c/CMakeLists.txt @@ -14,6 +14,8 @@ else() message(FATAL_ERROR "Unsupported CXX compiler: please use Clang == 20.0") endif() +option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" OFF) + include(cmake/CMakeBasics.cmake) include(cmake/Setup3rdParty.cmake) diff --git a/src/c/cmake/Setup3rdParty.cmake b/src/c/cmake/Setup3rdParty.cmake index d7744889..48617da6 100644 --- a/src/c/cmake/Setup3rdParty.cmake +++ b/src/c/cmake/Setup3rdParty.cmake @@ -20,6 +20,21 @@ if(PERFFLOWASPECT_WITH_MULTITHREADS) include(cmake/thirdparty/FindThreads.cmake) endif() +if(PERFFLOWASPECT_WITH_ADIAK) + if(NOT adiak_DIR) + message(FATAL_ERROR "PFA + Adiak needs explicit adiak_DIR") + endif() + + if (adiak_DIR) + message(STATUS "${adiak_DIR}") + include(cmake/thirdparty/FindAdiak.cmake) + endif() + + if (ADIAK_FOUND) + add_definitions(-DPERFFLOWASPECT_WITH_ADIAK) + endif() +endif() + if(PERFFLOWASPECT_WITH_CALIPER) # first Check for CALIPER_DIR if(NOT caliper_DIR) diff --git a/src/c/cmake/thirdparty/FindAdiak.cmake b/src/c/cmake/thirdparty/FindAdiak.cmake new file mode 100644 index 00000000..85e56c5f --- /dev/null +++ b/src/c/cmake/thirdparty/FindAdiak.cmake @@ -0,0 +1,12 @@ +message(STATUS "Looking for Adiak in ${adiak_DIR}") +find_package(adiak REQUIRED + PATHS ${adiak_DIR}/lib/cmake/adiak + NO_DEFAULT_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_CMAKE_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + +message(STATUS "FOUND Adiak: ${adiak_DIR}") +set(ADIAK_FOUND TRUE) +set(PERFFLOWASPECT_ADIAK_ENABLED TRUE) diff --git a/src/c/host-configs/matrix-4.18.0-x86_64-intel-classic-2021.6.0@clang@19.1.7.cmake b/src/c/host-configs/matrix-4.18.0-x86_64-intel-classic-2021.6.0@clang@19.1.7.cmake new file mode 100644 index 00000000..500b00ec --- /dev/null +++ b/src/c/host-configs/matrix-4.18.0-x86_64-intel-classic-2021.6.0@clang@19.1.7.cmake @@ -0,0 +1,12 @@ +############################################################## +# Copyright 2021 Lawrence Livermore National Security, LLC +# (c.f. AUTHORS, NOTICE.LLNS, COPYING) +# +# This file is part of the Flux resource manager framework. +# For details, see https://github.com/flux-framework. +# +# SPDX-License-Identifier: LGPL-3.0 +############################################################## + +set(CMAKE_C_COMPILER "/usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/icc" CACHE PATH "") +set(CMAKE_CXX_COMPILER "/usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/icpc" CACHE PATH "") diff --git a/src/c/runtime/CMakeLists.txt b/src/c/runtime/CMakeLists.txt index 05130853..10efade2 100644 --- a/src/c/runtime/CMakeLists.txt +++ b/src/c/runtime/CMakeLists.txt @@ -14,6 +14,9 @@ set(perfflow_runtime_sources include_directories(${JANSSON_INCLUDE_DIRS}) +if(PERFFLOWASPECT_WITH_ADIAK) + include_directories(${adiak_INCLUDE_DIR}) +endif() if(PERFFLOWASPECT_WITH_CALIPER) include_directories(${caliper_INCLUDE_DIR}) endif() @@ -23,6 +26,13 @@ add_library(perfflow_runtime SHARED ${perfflow_runtime_headers} ) +if (PERFFLOWASPECT_WITH_ADIAK) + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto adiak::adiak) +else() + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto ) +endif() if(PERFFLOWASPECT_WITH_CALIPER) target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto caliper) else() diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 4f3ddb0e..4475f1ac 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -228,6 +228,27 @@ int advice_chrome_tracing_t::flush_if(size_t size) m_ofs << "{" << std::endl; m_ofs << " \"displayTimeUnit\": \"us\"," << std::endl; m_ofs << " \"otherData\": {" << std::endl; +#ifdef PERFFLOWASPECT_WITH_ADIAK + { + const char *key; + json_t *val; + json_t *metadata = get_adiak_statistics(); + size_t total = json_object_size(metadata); + size_t idx = 0; + json_object_foreach(metadata, key, val) + { + char *v = json_dumps(val, JSON_ENCODE_ANY); + m_ofs << " \"" << key << "\": " << v; + free(v); + if (++idx < total) + { + m_ofs << ","; + } + m_ofs << "\n"; + } + json_decref(metadata); + } +#endif m_ofs << "" << std::endl; m_ofs << " }," << std::endl; m_ofs << " \"traceEvents\": [" << std::endl; @@ -663,6 +684,60 @@ long advice_chrome_tracing_t::get_memory_usage() return max_ram_usage; } +#ifdef PERFFLOWASPECT_WITH_ADIAK +void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, + const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque) +{ + json_t *obj = static_cast(opaque); + json_t *metadata = nullptr; + + switch (t->dtype) + { + case adiak_version: + case adiak_string: + case adiak_catstring: + case adiak_path: + metadata = json_string(reinterpret_cast(val->v_ptr)); + break; + case adiak_long: + case adiak_date: + metadata = json_integer(val->v_long); + break; + case adiak_ulong: + metadata = json_integer(unsigned(val->v_long)); + break; + case adiak_double: + metadata = json_real(val->v_double); + break; + case adiak_int: + metadata = json_integer(val->v_int); + break; + case adiak_uint: + metadata = json_integer(unsigned(val->v_int)); + break; + case adiak_timeval: + { + struct timeval *tv = (struct timeval *)(val->v_ptr); + json_t *tv_obj = json_object(); + json_object_set_new(tv_obj, "tv_sec", json_integer(tv->tv_sec)); + json_object_set_new(tv_obj, "tv_usec", json_integer(tv->tv_usec)); + metadata = tv_obj; + break; + } + default: + return; + } + json_object_set_new(obj, name, metadata); +} + +json_t *advice_chrome_tracing_t::get_adiak_statistics() +{ + json_t *adiak_metadata = json_object(); + adiak_list_namevals(1, adiak_category_all, adiak_cb, adiak_metadata); + return adiak_metadata; +} +#endif + int advice_chrome_tracing_t::with_flow(const char *module, const char *function, const char *flow, @@ -747,6 +822,7 @@ int advice_chrome_tracing_t::before(const char *module, std::string fname; long mem_start; + if (m_enable_logging) { if ((rc = create_event(&event, module, function, my_ts)) < 0) @@ -1022,6 +1098,7 @@ int advice_chrome_tracing_t::after(const char *module, free(json_str); json_decref(event); + return flush_if(FLUSH_SIZE); } else diff --git a/src/c/runtime/advice_chrome_tracing.hpp b/src/c/runtime/advice_chrome_tracing.hpp index 04c13a60..0c6d84d4 100644 --- a/src/c/runtime/advice_chrome_tracing.hpp +++ b/src/c/runtime/advice_chrome_tracing.hpp @@ -16,6 +16,10 @@ #include #include "advice_base.hpp" +#ifdef PERFFLOWASPECT_WITH_ADIAK +#include +#endif + class advice_chrome_tracing_t : public advice_base_t { public: @@ -43,6 +47,11 @@ class advice_chrome_tracing_t : public advice_base_t double get_wall_time(); double get_cpu_time(); long get_memory_usage(); +#ifdef PERFFLOWASPECT_WITH_ADIAK + json_t *get_adiak_statistics(); + static void adiak_cb(const char *name, int cat, const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque); + +#endif int cannonicalize_perfflow_options (); int parse_perfflow_options (); diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index 98df50e7..260f113e 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -8,10 +8,18 @@ set(SMOKETESTS set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto) message(STATUS "Adding CXX unit tests") + +if(PERFFLOWASPECT_WITH_ADIAK) + message(STATUS "Including Adiak in test dependencies") + set(perfflow_deps "${perfflow_deps}" adiak::adiak) +endif() + foreach(TEST ${SMOKETESTS}) message(STATUS " [*] Adding test: ${TEST}") add_executable(${TEST} ${TEST}.cpp) set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC") + target_link_libraries(${TEST} ${perfflow_deps}) + message(STATUS "${perfflow_deps}") if(PERFFLOWASPECT_WITH_CALIPER) target_link_libraries(${TEST} ${perfflow_deps} caliper) else() @@ -34,6 +42,8 @@ else() option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" OFF) endif() + + if(PERFFLOWASPECT_WITH_MULTITHREADS) message(STATUS " [*] Adding test: smoketest_MT") add_executable(smoketest_MT smoketest_MT.cpp) @@ -60,8 +70,12 @@ endif() if(PERFFLOWASPECT_WITH_CUDA) message(STATUS " [*] Adding test: smoketest_cuda") - set(CUDA_NVCC_FLAGS "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so -Xcompiler=-fpass-plugin=../../../weaver/weave/libWeavePass.so") - cuda_add_executable(smoketest_cuda smoketest_cuda_wrapper.cpp smoketest_cuda_kernel.cu) + set(CUDA_NVCC_FLAGS "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so -Xcompiler=-fpass-plugin=../../../weaver/weave/libWeavePass.so") + cuda_add_executable(smoketest_cuda + smoketest_cuda_wrapper.cpp + smoketest_cuda_kernel.cu + ) + set_source_files_properties(smoketest_cuda_wrapper.cpp COMPILE_FLAGS "-Xclang -load -Xclang ${CMAKE_BINARY_DIR}/weaver/weave/libWeavePass.so -fpass-plugin=${CMAKE_BINARY_DIR}/weaver/weave/libWeavePass.so -fPIC") if(PERFFLOWASPECT_WITH_CALIPER) target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES} caliper) else() diff --git a/src/c/test/smoketest_cuda_wrapper.cpp b/src/c/test/smoketest_cuda_wrapper.cpp index ed325429..0bca947c 100644 --- a/src/c/test/smoketest_cuda_wrapper.cpp +++ b/src/c/test/smoketest_cuda_wrapper.cpp @@ -10,6 +10,7 @@ #include "smoketest_cuda_kernel.cuh" +__attribute__((annotate("@critical_path(pointcut='around')"))) int main(int argc, char *argv[]) { Wrapper::wrapper(); diff --git a/src/c/weaver/weave/CMakeLists.txt b/src/c/weaver/weave/CMakeLists.txt index 7945e63f..96ddb95c 100644 --- a/src/c/weaver/weave/CMakeLists.txt +++ b/src/c/weaver/weave/CMakeLists.txt @@ -16,6 +16,13 @@ set_target_properties(WeavePass PROPERTIES target_link_libraries(WeavePass perfflow_parser ${JANSSON_LIB}) +if(PERFFLOWASPECT_WITH_MPI) + include_directories(${MPI_C_INCLUDE_PATH}) + target_link_libraries(WeavePass ${MPI_C_LIBRARIES}) + target_compile_definitions(WeavePass PRIVATE PERFFLOWASPECT_WITH_MPI) +endif() + + add_library(WeavePassPlugin INTERFACE) target_compile_options(WeavePassPlugin INTERFACE "SHELL:$>$>" diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index de4116aa..5c0212fe 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -26,6 +26,20 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) return false; } +#ifdef PERFFLOWASPECT_WITH_ADIAK + Function *main = m.getFunction("main"); + + if (main != NULL) + { + outs() << "WeavePass[INFO]: Found main to inject Adiak\n"; + insertAdiak(m, *main); + } + else + { + outs() << "WeavePass[WARN]: Could not find main to inject Adiak\n"; + } +#endif + // Support Caliper annotations by greating the cali_begin_region // and cali_end_region functions. #ifdef PERFFLOWASPECT_WITH_CALIPER @@ -157,17 +171,27 @@ bool weave_ns::WeaveCommon::insertAfter(Module &m, Function &f, StringRef &a, if (isa(inst) || isa(inst)) { valid = true; - } else if (isa(inst)) { - if (auto *call = dyn_cast(inst->getPrevNode())) { - Function *callee = call->getCalledFunction(); - if (callee && callee->getName() == "pthread_exit" || callee->getName() == "exit" || callee->getName() == "abort") { - inst = inst->getPrevNode(); - valid = true; + } + else if (isa(inst)) + { + Instruction *prev = inst->getPrevNode(); + if (prev && isa_and_present(prev)) + { + auto *call = cast(prev); + if (Function *callee = call->getCalledFunction()) + { + StringRef name = callee->getName(); + if (name == "pthread_exit" || name == "exit" || name == "abort") + { + inst = prev; + valid = true; + } } } } - if (valid) { + if (valid) + { IRBuilder<> builder(inst); Value *v1 = builder.CreateGlobalString(m.getName(), "str"); Value *v2 = builder.CreateGlobalString(f.getName(), "str"); @@ -234,6 +258,122 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, return true; } +#ifdef PERFFLOWASPECT_WITH_ADIAK +bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) +{ + LLVMContext &context = m.getContext(); + + // the adiak_init() call + // return and argument types + Type *voidTy = Type::getVoidTy(context); + Type *int8Ty = Type::getInt8Ty(context); + Type *int32Ty = Type::getInt32Ty(context); + PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); + + // adiak_init function signature + std::vector initArgs = { voidPtrTy }; + FunctionType *adiakInitType = FunctionType::get(voidTy, initArgs, false); + FunctionCallee adiakInit = m.getOrInsertFunction("adiak_init", adiakInitType); + + IRBuilder<> builder(context); + Value *arg; + BasicBlock &entry = f.getEntryBlock(); + builder.SetInsertPoint(&entry, entry.begin()); + +#ifdef PERFFLOWASPECT_WITH_MPI + // find the MPI communicator, MPI_COMM_WORLD + // OpenMPI exposes this as ompi_comm_world (untested) + // MPICH exposes this as a constant 0x44000000 + uint64_t mpiValue = 0x44000000; + Value *commVal = ConstantInt::get(int32Ty, mpiValue); + AllocaInst *alloc = builder.CreateAlloca(int32Ty, nullptr, "weave_mpi_comm"); + builder.CreateStore(commVal, alloc); + arg = builder.CreateBitCast( + alloc, + voidPtrTy, + "mpi_comm_world_void" + ); + + CallInst *mpi = nullptr; + // find each instruction to see if there is an MPI_Init call instruction + for (BasicBlock &bb : f) + { + for (Instruction &i : bb) + { + if (auto *call = dyn_cast(&i)) + { + if (Function *callee = call->getCalledFunction()) + { + if (callee->getName() == "MPI_Init") + { + mpi = call; + } + } + } + } + if (mpi) { break; } + } + if (mpi) + { + BasicBlock *mpiBlock = mpi->getParent(); + auto insertPos = BasicBlock::iterator(mpi); + ++insertPos; + builder.SetInsertPoint(mpiBlock, insertPos); + } + else + { + arg = Constant::getNullValue(voidPtrTy); + } + builder.CreateCall(adiakInit, {arg}); + +#else + arg = Constant::getNullValue(voidPtrTy); + builder.CreateCall(adiakInit, {arg}); +#endif + + // call adiak_collect_all() + FunctionType *collectType = FunctionType::get(int32Ty, {}, false); + FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", + collectType); + builder.CreateCall(collectAll, {}); + + // call adiak_walltime() + FunctionCallee walltime = m.getOrInsertFunction("adiak_walltime", collectType); + builder.CreateCall(walltime, {}); + + FunctionCallee systime = m.getOrInsertFunction("adiak_systime", collectType); + builder.CreateCall(systime, {}); + + FunctionCallee cputime = m.getOrInsertFunction("adiak_cputime", collectType); + builder.CreateCall(cputime, {}); + + // adiak_fini signature + FunctionType *adiakFinishType = FunctionType::get(voidTy, {}, false); + FunctionCallee adiakFinish = m.getOrInsertFunction("adiak_fini", + adiakFinishType); + + // find all places where the function terminates from a ReturnInst + std::vector returns; + for (BasicBlock &bb : f) + { + if (auto *ret = dyn_cast(bb.getTerminator())) + { + returns.push_back(ret); + } + } + + // insert adiak_fini at those return instructions + for (ReturnInst *ret : returns) + { + builder.SetInsertPoint(ret); + builder.CreateCall(adiakFinish, {}); + } + + return true; + +} +#endif + #ifdef PERFFLOWASPECT_WITH_CALIPER bool weave_ns::WeaveCommon::instrumentCaliper(Module &M, Function &F) { diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index 164b8e4c..b8362361 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -25,6 +25,10 @@ #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "../../parser/perfflow_parser.hpp" +#ifdef PERFFLOWASPECT_WITH_MPI +#include "mpi.h" +#endif + using namespace llvm; namespace weave_ns { @@ -38,6 +42,9 @@ class WeaveCommon bool insertBefore(Module &m, Function &f, StringRef &a, int async, std::string &scope, std::string &flow, std::string pcut); +#ifdef PERFFLOWASPECT_WITH_ADIAK + bool insertAdiak(Module &m, Function &f); +#endif #ifdef PERFFLOWASPECT_WITH_CALIPER bool instrumentCaliper(Module &m, Function &f);