diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 38d15ed6..79c7d415 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-24.04 strategy: matrix: - config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only] + config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only, boilerplate_caliper] include: - config: boilerplate @@ -17,31 +17,43 @@ jobs: PERFFLOWASPECT_WITH_MPI: ON PERFFLOWASPECT_WITH_MULTITHREADS: ON CMAKE_BUILD_TYPE: Debug + PERFFLOWASPECT_WITH_CALIPER: OFF - config: release PERFFLOWASPECT_WITH_CUDA: OFF PERFFLOWASPECT_WITH_MPI: ON PERFFLOWASPECT_WITH_MULTITHREADS: ON CMAKE_BUILD_TYPE: Release + PERFFLOWASPECT_WITH_CALIPER: OFF - config: debug_cuda_only PERFFLOWASPECT_WITH_CUDA: OFF PERFFLOWASPECT_WITH_MPI: OFF PERFFLOWASPECT_WITH_MULTITHREADS: OFF CMAKE_BUILD_TYPE: Debug + PERFFLOWASPECT_WITH_CALIPER: OFF - config: debug_mpi_only PERFFLOWASPECT_WITH_CUDA: OFF PERFFLOWASPECT_WITH_MPI: ON PERFFLOWASPECT_WITH_MULTITHREADS: OFF CMAKE_BUILD_TYPE: Debug + PERFFLOWASPECT_WITH_CALIPER: OFF - config: debug_threads_only PERFFLOWASPECT_WITH_CUDA: OFF PERFFLOWASPECT_WITH_MPI: OFF PERFFLOWASPECT_WITH_MULTITHREADS: ON CMAKE_BUILD_TYPE: Debug + PERFFLOWASPECT_WITH_CALIPER: OFF + - config: boilerplate_caliper + PERFFLOWASPECT_WITH_CUDA: OFF + PERFFLOWASPECT_WITH_MPI: OFF + PERFFLOWASPECT_WITH_MULTITHREADS: OFF + CMAKE_BUILD_TYPE: Debug + PERFFLOWASPECT_WITH_CALIPER: ON + steps: # Checkout PerfFlowAspect repository under $GITHUB_WORKSPACE - uses: actions/checkout@v2 @@ -49,9 +61,24 @@ jobs: - name: Setup Build Env run: | sudo apt-get update - sudo apt install clang-18 llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich + sudo apt install clang-18 llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich pybind11-dev clang++ --version + - name: Clone Caliper + uses: actions/checkout@v2 + with: + repository: LLNL/Caliper + path: Caliper + + - name: Build Caliper + working-directory: Caliper + run: | + mkdir build && mkdir install + cd build + cmake -DWITH_PYTHON_BINDINGS=On -DCMAKE_INSTALL_PREFIX=../install ../ + make VERBOSE=1 + make install + - name: Compile check run: | cd src/c @@ -62,6 +89,7 @@ jobs: export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CUDA=${{matrix.PERFFLOWASPECT_WITH_CUDA}}" export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MPI=${{matrix.PERFFLOWASPECT_WITH_MPI}}" export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MULTITHREADS=${{matrix.PERFFLOWASPECT_WITH_MULTITHREADS}}" + export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CALIPER=${{matrix.PERFFLOWASPECT_WITH_CALIPER}} -Dcaliper_DIR=/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install" echo -e ${CMAKE_OPTS} cmake ${CMAKE_OPTS} .. # build @@ -97,6 +125,8 @@ jobs: cd src/python python -m pip install -r requirements.txt python -m pip list + # Export path for caliper + export PYTHONPATH="/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install/lib/${{ matrix.python-version }}/site-packages:$PYTHONPATH" - name: Run Python Smoke Tests run: | diff --git a/docs/BasicTutorial.rst b/docs/BasicTutorial.rst index 8f4d74e8..6d9f1c87 100644 --- a/docs/BasicTutorial.rst +++ b/docs/BasicTutorial.rst @@ -103,6 +103,8 @@ variable. Separate multiple variables with a colon as follows: +----------------------+--------------------------------------------------------------+---------------+---------------------------------+ | log-format | Dump JSON events in array or object format | Array | Array, Object | +----------------------+--------------------------------------------------------------+---------------+---------------------------------+ +| caliper-enable | Toggle Caliper data collection on/off (for Python only). | False | True, False | ++----------------------+--------------------------------------------------------------+---------------+---------------------------------+ ********************************************** Visualization of PerfFlowAspect Output Files diff --git a/docs/CaliperIntegration.rst b/docs/CaliperIntegration.rst new file mode 100644 index 00000000..52607ce2 --- /dev/null +++ b/docs/CaliperIntegration.rst @@ -0,0 +1,128 @@ +.. + # Copyright 2021 Lawrence Livermore National Security, LLC and other + # PerfFlowAspect Project Developers. See the top-level LICENSE file for + # details. + # + # SPDX-License-Identifier: LGPL-3.0 + +##################### + Caliper Integration +##################### + +PerfFlowAspect can be built with Caliper to leverage collection of additional +performance data, such as hardware performance counters on CPUs and GPU +measurements on NVIDIA GPUs. `Caliper `_ is an +instrumentation and performance annotation library. + +PerfFlowAspect supports the Caliper integration for both C/C++ and Python +codebases. This enables a single PerfFlowAspect annotation to generate both +`.pfw` trace files and `.cali` files, so users do not need any additional +annotations to obtain Caliper data. + +A Caliper install is required before building PerfFlowAspect in this case. +Caliper can be configured at runtime, as shown in examples below. + +************* + C/C++ Build +************* + +Caliper needs to be is built with the same `clang` compiler chain as +PerfFlowAspect. These can be specified with the `-DCMAKE_C_COMPILER` and +`-DCMAKE_CXX_COMPILER`. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_C_COMPILER= \ + -DWITH_PYTHON_BINDINGS=On \ + -Dcaliper_DIR=/share/lib/caliper ../ + +*************** + C/C++ Example +*************** + +.. code:: bash + + CALI_CONFIG=runtime-report ./smoketest + CALI_CONFIG=runtime-report,output=test.cali ./smoketest + +.. code:: bash + + Path Min time/rank Max time/rank Avg time/rank Time % + _Z3fooRKSs 0.004527 0.004527 0.004527 45.778137 + _Z3barv 0.004511 0.004511 0.004511 45.616341 + _Z3basv 0.000079 0.000079 0.000079 0.798867 + +.. code:: bash + + CALI_PAPI_COUNTERS=PAPI_TOT_CYC,PAPI_L2_DCM CALI_SERVICES_ENABLE=event,trace,papi,report ./smoketest + +.. code:: bash + + event.begin#annotation papi.PAPI_TOT_CYC papi.PAPI_L2_DCM annotation region.count event.end#annotation + _Z3fooRKSs 118289 679 + _Z3barv 200050 765 _Z3fooRKSs + _Z3basv 115098 352 _Z3fooRKSs/_Z3barv + 66564 242 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv + 117061 385 _Z3fooRKSs/_Z3barv 1 _Z3barv + 93592 206 _Z3fooRKSs 1 _Z3fooRKSs + _Z3fooRKSs 146308 332 + _Z3barv 87811 255 _Z3fooRKSs + _Z3basv 84904 244 _Z3fooRKSs/_Z3barv + 34547 66 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv + 82540 168 _Z3fooRKSs/_Z3barv 1 _Z3barv + 80711 144 _Z3fooRKSs 1 _Z3fooRKSs + _Z3fooRKSs 127765 183 + _Z3barv 85440 241 _Z3fooRKSs + _Z3basv 82100 250 _Z3fooRKSs/_Z3barv + 33969 67 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv + 81511 161 _Z3fooRKSs/_Z3barv 1 _Z3barv + 77498 128 _Z3fooRKSs 1 _Z3fooRKSs + _Z3fooRKSs 119853 164 + _Z3barv 83285 227 _Z3fooRKSs + _Z3basv 82702 297 _Z3fooRKSs/_Z3barv + 34170 78 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv + 81589 149 _Z3fooRKSs/_Z3barv 1 _Z3barv + 78920 119 _Z3fooRKSs 1 _Z3fooRKSs + +.. code:: bash + + PERFFLOW_OPTIONS='cpu-mem-usage=True:log-event=compact' CALI_CONFIG="load(time_exclusive.json),spot" ./smoketest + +.. code:: bash + + Path Min time/rank Max time/rank Avg time/rank Total time spot.channel + _Z3fooRKSs 0.018068 0.018068 0.018068 0.018068 regionprofile + _Z3barv 0.009124 0.009124 0.009124 0.009124 regionprofile + _Z3basv 0.000074 0.000074 0.000074 0.000074 regionprofile + +**************** + Python Example +**************** + +For Python applications, Caliper can be enabled by setting the `caliper-enable` +option. In this case, `PYTHONPATH` needs to be set to include the `pycaliper` +package from Caliper installation. + +.. code:: bash + + export PYTHONPATH=/lib/python/site-packages + PERFFLOW_OPTIONS="caliper-enable=True" CALI_CONFIG=runtime-report ./smoketest.py + + Inside main + foo + bar + bas + foo + bar + bas + foo + bar + bas + foo + bar + bas + Path Time (E) Time (I) Time % (E) Time % (I) + 0.004692 0.009423 47.547246 95.486397 + 0.004659 0.004731 47.213609 47.939151 + 0.000072 0.000072 0.725542 0.725542 diff --git a/docs/index.rst b/docs/index.rst index a8de2d9b..1d0fa902 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled. BuildingPerfFlowAspect Annotations + CaliperIntegration UpcomingFeatures .. toctree:: diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt index 8d12a5a8..8fe5976b 100644 --- a/src/c/CMakeLists.txt +++ b/src/c/CMakeLists.txt @@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.12) project(PerfFlowAspect VERSION "0.1.0") +# Higher-level build options. +option(PERFFLOWASPECT_WITH_CALIPER "Build with Caliper support" OFF) + # Fail if using Clang < 18.0 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0) diff --git a/src/c/cmake/Setup3rdParty.cmake b/src/c/cmake/Setup3rdParty.cmake index d7f203a6..d7744889 100644 --- a/src/c/cmake/Setup3rdParty.cmake +++ b/src/c/cmake/Setup3rdParty.cmake @@ -19,3 +19,19 @@ include(cmake/thirdparty/FindOpenSSL.cmake) if(PERFFLOWASPECT_WITH_MULTITHREADS) include(cmake/thirdparty/FindThreads.cmake) endif() + +if(PERFFLOWASPECT_WITH_CALIPER) + # first Check for CALIPER_DIR + if(NOT caliper_DIR) + MESSAGE(FATAL_ERROR "Caliper support needs explicit caliper_DIR") + endif() + + if(caliper_DIR) + message(STATUS "${caliper_DIR}") + include(cmake/thirdparty/FindCaliper.cmake) + endif() + + if(CALIPER_FOUND) + add_definitions(-DPERFFLOWASPECT_WITH_CALIPER) + endif() +endif() diff --git a/src/c/cmake/thirdparty/FindCaliper.cmake b/src/c/cmake/thirdparty/FindCaliper.cmake new file mode 100644 index 00000000..d4423736 --- /dev/null +++ b/src/c/cmake/thirdparty/FindCaliper.cmake @@ -0,0 +1,15 @@ +message(STATUS "Looking for Caliper in: ${caliper_DIR}") + +find_package(caliper REQUIRED + PATHS ${caliper_DIR}/share/cmake/caliper + NO_DEFAULT_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_CMAKE_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + +message(STATUS "FOUND Caliper: ${caliper_DIR}") + +#set(ADIAK_FOUND TRUE) +set(CALIPER_FOUND TRUE) +set(PERFFLOWASPECT_CALIPER_ENABLED TRUE) diff --git a/src/c/config/CMakeLists.txt b/src/c/config/CMakeLists.txt index 455c4089..959f6897 100644 --- a/src/c/config/CMakeLists.txt +++ b/src/c/config/CMakeLists.txt @@ -42,6 +42,8 @@ configure_package_config_file( install(FILES ${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config.cmake ${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config-version.cmake + perfflowaspect_setup_deps.cmake + perfflowaspect_setup_targets.cmake DESTINATION ${PERFFLOWASPECT_INSTALL_CMAKE_MODULE_DIR}) # Create pkg-config .pc file diff --git a/src/c/config/perfflowaspect-config.cmake.in b/src/c/config/perfflowaspect-config.cmake.in index 60a9d4a7..7514b7da 100644 --- a/src/c/config/perfflowaspect-config.cmake.in +++ b/src/c/config/perfflowaspect-config.cmake.in @@ -3,6 +3,9 @@ if (NOT PERFFLOWASPECT_CONFIG_LOADED) set(PERFFLOWASPECT_DIR "@CMAKE_INSTALL_PREFIX@") set(PERFFLOWASPECT_LIB_DIR "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@") + set(PERFFLOWASPECT_CALIPER_ENABLED "@PERFFLOWASPECT_CALIPER_ENABLED@") + set(PERFFLOWASPECT_CALIPER_DIR "@caliper_DIR@") + include(CMakeFindDependencyMacro) find_dependency(OpenSSL REQUIRED) diff --git a/src/c/config/perfflowaspect_setup_deps.cmake b/src/c/config/perfflowaspect_setup_deps.cmake new file mode 100644 index 00000000..98f6ba02 --- /dev/null +++ b/src/c/config/perfflowaspect_setup_deps.cmake @@ -0,0 +1,17 @@ +include(CMakeFindDependencyMacro) + +if (NOT caliper_DIR) + set(caliper_DIR ${PERFFLOWASPECT_CALIPER_DIR}) +endif() + +if(caliper_DIR) + if(NOT PerfFlowAspect_FIND_QUIETLY) + message(STATUS "PerfFlowAspect was built with Caliper Support") + message(STATUS "Looking for Caliper at: ${caliper_DIR}/share/cmake/caliper") + endif() + + # find caliper + find_package(caliper REQUIRED + NO_DEFAULT_PATH + PATHS ${caliper_DIR}/share/cmake/caliper) +endif() diff --git a/src/c/config/perfflowaspect_setup_targets.cmake b/src/c/config/perfflowaspect_setup_targets.cmake new file mode 100644 index 00000000..ed5d28c7 --- /dev/null +++ b/src/c/config/perfflowaspect_setup_targets.cmake @@ -0,0 +1,6 @@ +# create convenience target that bundles all reg perfflowaspect deps +add_library(perfflowaspect::perfflowaspect INTERFACE IMPORTED) + +set_property(TARGET perfflowaspect::perfflowaspect + PROPERTY INTERFACE_LINK_LIBRARIES + perfflowaspect) diff --git a/src/c/host-configs/tioga-4.18.0-x86_64-rocm@6.3.1-amdclang@18.0.0.cmake b/src/c/host-configs/tioga-4.18.0-x86_64-rocm@6.3.1-amdclang@18.0.0.cmake index e23b4189..33fca384 100644 --- a/src/c/host-configs/tioga-4.18.0-x86_64-rocm@6.3.1-amdclang@18.0.0.cmake +++ b/src/c/host-configs/tioga-4.18.0-x86_64-rocm@6.3.1-amdclang@18.0.0.cmake @@ -10,3 +10,8 @@ set(CMAKE_C_COMPILER "/opt/rocm-6.3.1/lib/llvm/bin/amdclang" CACHE PATH "") set(CMAKE_CXX_COMPILER "/opt/rocm-6.3.1/lib/llvm/bin/amdclang++" CACHE PATH "") + +# To enable fine-grained profiling with Caliper, utilize the below options. +# Caliper should be built with the same compilers as the PerfFlowAspect. +# set(PERFFLOWASPECT_WITH_CALIPER ON CACHE BOOL "") +# set(caliper_DIR "path-to-caliper-install-directory" CACHE PATH "") diff --git a/src/c/host-configs/tuolumne-4.18.0-x86_64-rocm@6.2.1-amdclang@18.0.0.cmake b/src/c/host-configs/tuolumne-4.18.0-x86_64-rocm@6.2.1-amdclang@18.0.0.cmake index 53c825a2..39eda4fd 100644 --- a/src/c/host-configs/tuolumne-4.18.0-x86_64-rocm@6.2.1-amdclang@18.0.0.cmake +++ b/src/c/host-configs/tuolumne-4.18.0-x86_64-rocm@6.2.1-amdclang@18.0.0.cmake @@ -10,3 +10,8 @@ set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/lib/llvm/bin/amdclang" CACHE PATH "") set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/lib/llvm/bin/amdclang++" CACHE PATH "") + +# To enable fine-grained profiling with Caliper, utilize the below options. +# Caliper should be built with the same compilers as the PerfFlowAspect. +# set(PERFFLOWASPECT_WITH_CALIPER ON CACHE BOOL "") +# set(caliper_DIR "path-to-caliper-install-directory" CACHE PATH "") diff --git a/src/c/runtime/CMakeLists.txt b/src/c/runtime/CMakeLists.txt index 12e4fba5..05130853 100644 --- a/src/c/runtime/CMakeLists.txt +++ b/src/c/runtime/CMakeLists.txt @@ -14,12 +14,20 @@ set(perfflow_runtime_sources include_directories(${JANSSON_INCLUDE_DIRS}) +if(PERFFLOWASPECT_WITH_CALIPER) + include_directories(${caliper_INCLUDE_DIR}) +endif() + add_library(perfflow_runtime SHARED ${perfflow_runtime_sources} ${perfflow_runtime_headers} ) -target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +if(PERFFLOWASPECT_WITH_CALIPER) + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto caliper) +else() + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +endif() install(TARGETS perfflow_runtime EXPORT perfflow_export diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index cb1fa0ee..98df50e7 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -12,7 +12,11 @@ foreach(TEST ${SMOKETESTS}) message(STATUS " [*] Adding test: ${TEST}") add_executable(${TEST} ${TEST}.cpp) set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC") - target_link_libraries(${TEST} ${perfflow_deps}) + if(PERFFLOWASPECT_WITH_CALIPER) + target_link_libraries(${TEST} ${perfflow_deps} caliper) + else() + target_link_libraries(${TEST} ${perfflow_deps}) + endif() endforeach() # Build Options @@ -35,7 +39,11 @@ if(PERFFLOWASPECT_WITH_MULTITHREADS) add_executable(smoketest_MT smoketest_MT.cpp) set_source_files_properties(smoketest_MT.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC") set(THREADS_PREFER_PTHREAD_FLAG ON) - target_link_libraries(smoketest_MT ${perfflow_deps} pthread) + if(PERFFLOWASPECT_WITH_CALIPER) + target_link_libraries(smoketest_MT ${perfflow_deps} pthread caliper) + else() + target_link_libraries(smoketest_MT ${perfflow_deps} pthread) + endif() endif() if(PERFFLOWASPECT_WITH_MPI) @@ -43,14 +51,22 @@ if(PERFFLOWASPECT_WITH_MPI) add_executable(smoketest_MPI smoketest_MPI.cpp) set_source_files_properties(smoketest_MPI.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC") include_directories(${MPI_INCLUDE_PATH}) - target_link_libraries(smoketest_MPI ${perfflow_deps} ${MPI_LIBRARIES}) + if(PERFFLOWASPECT_WITH_CALIPER) + target_link_libraries(smoketest_MPI ${perfflow_deps} ${MPI_LIBRARIES} caliper) + else() + target_link_libraries(smoketest_MPI ${perfflow_deps} ${MPI_LIBRARIES}) + endif() endif() if(PERFFLOWASPECT_WITH_CUDA) message(STATUS " [*] Adding test: smoketest_cuda") set(CUDA_NVCC_FLAGS "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so -Xcompiler=-fpass-plugin=../../../weaver/weave/libWeavePass.so") cuda_add_executable(smoketest_cuda smoketest_cuda_wrapper.cpp smoketest_cuda_kernel.cu) - target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES}) + if(PERFFLOWASPECT_WITH_CALIPER) + target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES} caliper) + else() + target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES}) + endif() endif() configure_file(t0001-cbinding-basic.t.in diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 8470388f..398aa648 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -26,6 +26,24 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) return false; } + // Support Caliper annotations by greating the cali_begin_region + // and cali_end_region functions. +#ifdef PERFFLOWASPECT_WITH_CALIPER + IRBuilder<> IRB(m.getContext()); + AttrBuilder AB(m.getContext()); + AB.addAttribute(Attribute::AlwaysInline); + //It was not added in LLVM@10. + //AB.addAttribute(Attribute::ArgMemOnly); + AttributeList Attrs = AttributeList::get(m.getContext(), + AttributeList::FunctionIndex, AB); + // Insert Functions on the module + CaliBeginRegion = m.getOrInsertFunction("cali_begin_region", Attrs, + IRB.getVoidTy(), IRB.getPtrTy()); + CaliEndRegion = m.getOrInsertFunction("cali_end_region", Attrs, IRB.getVoidTy(), + IRB.getPtrTy()); +#endif + + bool changed = false; if (annotations->getNumOperands() <= 0) @@ -40,6 +58,12 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) auto *fn = dyn_cast (e->getOperand(0)); if (fn != NULL) { +#ifdef PERFFLOWASPECT_WITH_CALIPER + // We insert Caliper Instrumentation before weaver. + // Thus weaver will include Caliper overheads + changed |= instrumentCaliper(m, *fn); +#endif + auto anno = cast( cast(e->getOperand(1)) ->getOperand(0)) @@ -197,6 +221,55 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, return true; } +#ifdef PERFFLOWASPECT_WITH_CALIPER +bool weave_ns::WeaveCommon::instrumentCaliper(Module &M, Function &F) +{ + IRBuilder<> IRB(M.getContext()); + BasicBlock &Entry = F.getEntryBlock(); + SplitBlock(&Entry, &*Entry.getFirstInsertionPt()); + + IRB.SetInsertPoint(Entry.getTerminator()); + std::string FunctionName = F.getName().str(); + auto *FnStr = IRB.CreateGlobalStringPtr(FunctionName); + IRB.CreateCall(CaliBeginRegion, {FnStr}); + + bool RetFound = false; + for (inst_iterator It = inst_begin(F), E = inst_end(F); It != E; ++It) + { + Instruction *I = &*It; + if (!isa(I) && !isa(I) && !isa(I)) + { + continue; + } + + if (isa(I)) + { + IRB.SetInsertPoint(I); + IRB.CreateCall(CaliEndRegion, {FnStr}); + RetFound = true; + } + + // This is a call instruction + CallBase *CB = dyn_cast(I); + if (CB && CB->doesNotReturn()) + { + IRB.SetInsertPoint(I); + IRB.CreateCall(CaliEndRegion, {FnStr}); + RetFound = true; + } + } + + // All functions need to have at least one exit block + if (!RetFound) + { + dbgs() << "Could not find return for " << FunctionName << "\n"; + abort(); + } + + return RetFound; +} +#endif + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index 672766cf..164b8e4c 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -20,6 +20,9 @@ #include "llvm/IR/Argument.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Support/Debug.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "../../parser/perfflow_parser.hpp" using namespace llvm; @@ -36,9 +39,18 @@ class WeaveCommon bool insertBefore(Module &m, Function &f, StringRef &a, int async, std::string &scope, std::string &flow, std::string pcut); +#ifdef PERFFLOWASPECT_WITH_CALIPER + bool instrumentCaliper(Module &m, Function &f); +#endif + public: bool modifyAnnotatedFunctions(Module &m); +#ifdef PERFFLOWASPECT_WITH_CALIPER + FunctionCallee CaliBeginRegion; + FunctionCallee CaliEndRegion; +#endif + }; } // End namespace diff --git a/src/python/perfflowaspect/advice_chrome.py b/src/python/perfflowaspect/advice_chrome.py index 336d0848..f7ffed2e 100644 --- a/src/python/perfflowaspect/advice_chrome.py +++ b/src/python/perfflowaspect/advice_chrome.py @@ -43,6 +43,9 @@ def cannonicalize_perfflow_options(): perfflow_options["log-event"] = "Verbose" if perfflow_options.get("log-format") is None: perfflow_options["log-format"] = "Array" + # Caliper is disabled by default + if perfflow_options.get("caliper-enable") is None: + perfflow_options["caliper-enable"] = "False" def parse_perfflow_options(): @@ -207,6 +210,12 @@ class ChromeTracingAdvice: logger = None + caliper_flag = perfflow_options["caliper-enable"] + if caliper_flag in ["True", "true", "TRUE"]: + enable_caliper = True + elif caliper_flag in ["False", "false", "FALSE"]: + enable_caliper = False + def __init__(self): pass @@ -361,7 +370,7 @@ def trace(*args, **kwargs): def around(func): @functools.wraps(func) def trace(*args, **kwargs): - # Obtain start timestamp for tracing consistency. + # Obtain start timestamp for tracing consistency. This will inclide the Caliper overhead. ts_start = time.time() * 1000000 if not ChromeTracingAdvice.enable_compact_log_event: @@ -373,9 +382,31 @@ def trace(*args, **kwargs): cpu_start = cpu_start[0] time_start = time.time() + # If Caliper is enabled, import pycaliper and call begin_region + # Conditional import preserves current behavior of t0001.t + if ChromeTracingAdvice.enable_caliper: + try: + from pycaliper.instrumentation import begin_region + except ImportError: + print("Caliper is enabled but pycaliper could not be imported.") + begin_region = None + # Call Caliper's begin region + begin_region(str(func)) + rc = func(*args, **kwargs) - # Obtain end timestamp to calculate durations. + # If Caliper is enabled, import pycaliper and call end_region + # Conditional import preserves current behavior of t0001.t + if ChromeTracingAdvice.enable_caliper: + try: + from pycaliper.instrumentation import end_region + except ImportError: + print("Caliper is enabled but pycaliper could not be imported.") + end_region = None + # Call Caliper's end region + end_region(str(func)) + + # Obtain end timestamp to calculate durations. This will include the Caliper overhead. ts_end = time.time() * 1000000 if ChromeTracingAdvice.enable_cpu_mem_usage: