From 45bfb8f51a0fad51e50e3bde73be9eb01c9228f3 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 22:47:18 +0100 Subject: [PATCH 01/29] fix conflict --- include/dlaf/eigensolver/eigensolver/impl.h | 51 +++++++++++++++++++-- include/dlaf/tune.h | 21 ++++++++- src/init.cpp | 5 ++ 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index f9ebc0c308..35ad662f8d 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -60,15 +60,49 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M if (uplo != blas::Uplo::Lower) DLAF_UNIMPLEMENTED(uplo); +#ifdef DLAF_WITH_HDF5 + std::optional file_eigensolver; + + if (getTuneParameters().debug_dump_eigensolver_data) { + file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), "eigensolver.h5"); + file_eigensolver->write(mat_a, "/input"); + } + + std::optional file_reduction_to_band; + + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), "red_to_band.h5"); + file_reduction_to_band->write(mat_a, "/input"); + } +#endif + auto mat_taus = reduction_to_band(grid, mat_a, band_size); + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file_reduction_to_band->write(mat_a, "/band"); + } + + std::optional file_band_to_tridiagonal; + + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), "band_to_tridiag.h5"); + file_band_to_tridiagonal->write(mat_a, "/band"); + } +#endif + auto ret = band_to_tridiagonal(grid, uplo, band_size, mat_a); #ifdef DLAF_WITH_HDF5 - std::optional file; + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file_band_to_tridiagonal->write(ret.tridiagonal, "/tridiag"); + } + + std::optional file_tridiag; if (getTuneParameters().debug_dump_trisolver_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), "trid-ref.h5"); - file->write(ret.tridiagonal, "/tridiag"); + file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), "tridiag.h5"); + file_tridiag->write(ret.tridiagonal, "/tridiag"); } #endif @@ -76,12 +110,19 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_trisolver_data) { - file->write(evals, "/evals"); - file->write(mat_e, "/evecs"); + file_tridiag->write(evals, "/evals"); + file_tridiag->write(mat_e, "/evecs"); } #endif bt_band_to_tridiagonal(grid, band_size, mat_e, ret.hh_reflectors); bt_reduction_to_band(grid, band_size, mat_e, mat_a, mat_taus); + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_eigensolver_data) { + file_eigensolver->write(evals, "/evals"); + file_eigensolver->write(mat_e, "/evecs"); + } +#endif } } diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index 7de7f2aa2c..aef99baeac 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -22,8 +22,24 @@ namespace dlaf { /// DLA-Future tuning parameters. /// /// Holds the value of the parameters that can be used to tune DLA-Future. +/// - debug_dump_eigensolver_data: +/// Enable dump of eigensolver input/output data to "eigensolver.h5" file that will be created in the +/// working folder (it should not exist before the execution). +/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. +/// Set with environment variable DLAF_DEBUG_DUMP_EIGENSOLVER_DATA. +/// - debug_dump_reduction_to_band_data: +/// Enable dump of reduction_to_band input/output data to "reduction_to_band.h5" file that will be +/// created in the working folder (it should not exist before the execution). +/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with +/// environment variable +/// DLAF_DEBUG_DUMP_EIGENSOLVER_DATA. +/// - debug_dump_band_to_tridiagonal_data: +/// Enable dump of band_to_trigiagonal input/output data to "band_to_tridiagonal.h5" file that will +/// be created in the working folder (it should not exist before the execution). +/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with +/// environment variable DLAF_DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA. /// - debug_dump_trisolver_data: -/// Enable dump of trisolver input/output data to "trid-ref.h5" file that will be created in the +/// Enable dump of trisolver input/output data to "tridiagonal.h5" file that will be created in the /// working folder (it should not exist before the execution). /// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. /// Set with environment variable DLAF_DEBUG_DUMP_TRISOLVER_DATA. @@ -81,6 +97,9 @@ struct TuneParameters { red2band_panel_nworkers = std::max(1, default_pool_thread_count / 2); tridiag_rank1_nworkers = default_pool_thread_count; } + bool debug_dump_eigensolver_data = false; + bool debug_dump_reduction_to_band_data = false; + bool debug_dump_band_to_tridiagonal_data = false; bool debug_dump_trisolver_data = false; std::size_t red2band_panel_nworkers = 1; std::size_t red2band_barrier_busy_wait_us = 1000; diff --git a/src/init.cpp b/src/init.cpp index 3fdba264b5..eca50d0dbb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -221,6 +221,11 @@ void updateConfiguration(const pika::program_options::variables_map& vm, configu updateConfigurationValue(vm, param.band_to_tridiag_1d_block_size_base, "BAND_TO_TRIDIAG_1D_BLOCK_SIZE_BASE", "band-to-tridiag-1d-block-size-base"); + updateConfigurationValue(vm, param.debug_dump_eigensolver_data, "DEBUG_DUMP_EIGENSOLVER_DATA", ""); + updateConfigurationValue(vm, param.debug_dump_reduction_to_band_data, + "DEBUG_DUMP_REDUCTION_TO_BAND_DATA", ""); + updateConfigurationValue(vm, param.debug_dump_band_to_tridiagonal_data, + "DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA", ""); updateConfigurationValue(vm, param.debug_dump_trisolver_data, "DEBUG_DUMP_TRISOLVER_DATA", ""); updateConfigurationValue(vm, param.tridiag_rank1_nworkers, "TRIDIAG_RANK1_NWORKERS", From ea673953c0f7dcf6f65593ca9124f77b4a2c5979 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Wed, 8 Nov 2023 10:06:54 +0100 Subject: [PATCH 02/29] enable multiple calls --- include/dlaf/eigensolver/eigensolver/impl.h | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 35ad662f8d..2e7d97d51d 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -56,6 +56,16 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M Matrix, D>& evals, Matrix& mat_e) { const SizeType band_size = getBandSize(mat_a.blockSize().rows()); +#ifdef DLAF_WITH_HDF5 + static size_t num_eigensolver_calls = 0; + + auto fname = [](std::string name){ + std::stringstream fn; + fn << name << num_eigensolver_calls << ".h5"; + return fn.str(); + }; +#endif + // need uplo check as reduction to band doesn't have the uplo argument yet. if (uplo != blas::Uplo::Lower) DLAF_UNIMPLEMENTED(uplo); @@ -64,14 +74,14 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M std::optional file_eigensolver; if (getTuneParameters().debug_dump_eigensolver_data) { - file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), "eigensolver.h5"); + file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("eigensolver")); file_eigensolver->write(mat_a, "/input"); } std::optional file_reduction_to_band; if (getTuneParameters().debug_dump_reduction_to_band_data) { - file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), "red_to_band.h5"); + file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("red_to_band")); file_reduction_to_band->write(mat_a, "/input"); } #endif @@ -86,7 +96,7 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M std::optional file_band_to_tridiagonal; if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), "band_to_tridiag.h5"); + file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("band_to_tridiag")); file_band_to_tridiagonal->write(mat_a, "/band"); } #endif @@ -101,7 +111,7 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M std::optional file_tridiag; if (getTuneParameters().debug_dump_trisolver_data) { - file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), "tridiag.h5"); + file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("tridiag")); file_tridiag->write(ret.tridiagonal, "/tridiag"); } #endif @@ -123,6 +133,8 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M file_eigensolver->write(evals, "/evals"); file_eigensolver->write(mat_e, "/evecs"); } + + num_eigensolver_calls++; #endif } } From 8b370d23ff04056d8de2c3a6e11e5702cbca6061 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 22:48:59 +0100 Subject: [PATCH 03/29] fix conflicts --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 21 +++++++ include/dlaf/eigensolver/eigensolver/impl.h | 62 +++---------------- .../dlaf/eigensolver/reduction_to_band/impl.h | 20 ++++++ .../dlaf/eigensolver/tridiag_solver/impl.h | 22 +++++++ include/dlaf/tune.h | 8 +-- src/init.cpp | 3 +- 6 files changed, 76 insertions(+), 60 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index a2c55d940a..80fa39db55 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1034,6 +1035,17 @@ TridiagResult BandToTridiag::call_L( // Should be dispatched to local implementation if (1x1) grid. DLAF_ASSERT(grid.size() != comm::Size2D(1, 1), grid); +#ifdef DLAF_WITH_HDF5 + static size_t num_b2t_calls = 0; + std::string fname = "band_to_tridiag-" + std::to_string(num_b2t_calls) + ".h5"; + std::optional file; + + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_a, "/input"); + } +#endif + // note: A is square and has square blocksize SizeType size = mat_a.size().cols(); SizeType n = mat_a.nrTiles().cols(); @@ -1524,6 +1536,15 @@ TridiagResult BandToTridiag::call_L( mat_trid.readwrite(index))); } +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_trid, "/tridiaginal"); + } + + num_b2t_calls++; +#endif + return {std::move(mat_trid), std::move(mat_v)}; } } diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 2e7d97d51d..9f5c2bf069 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -56,82 +56,34 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M Matrix, D>& evals, Matrix& mat_e) { const SizeType band_size = getBandSize(mat_a.blockSize().rows()); -#ifdef DLAF_WITH_HDF5 - static size_t num_eigensolver_calls = 0; - - auto fname = [](std::string name){ - std::stringstream fn; - fn << name << num_eigensolver_calls << ".h5"; - return fn.str(); - }; -#endif - // need uplo check as reduction to band doesn't have the uplo argument yet. if (uplo != blas::Uplo::Lower) DLAF_UNIMPLEMENTED(uplo); #ifdef DLAF_WITH_HDF5 - std::optional file_eigensolver; + static size_t num_eigensolver_calls = 0; + std::string fname = "eigensolver-" + std::to_string(num_eigensolver_calls) + ".h5"; + std::optional file; if (getTuneParameters().debug_dump_eigensolver_data) { - file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("eigensolver")); - file_eigensolver->write(mat_a, "/input"); - } - - std::optional file_reduction_to_band; - - if (getTuneParameters().debug_dump_reduction_to_band_data) { - file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("red_to_band")); - file_reduction_to_band->write(mat_a, "/input"); + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_a, "/input"); } #endif auto mat_taus = reduction_to_band(grid, mat_a, band_size); -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_reduction_to_band_data) { - file_reduction_to_band->write(mat_a, "/band"); - } - - std::optional file_band_to_tridiagonal; - - if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("band_to_tridiag")); - file_band_to_tridiagonal->write(mat_a, "/band"); - } -#endif - auto ret = band_to_tridiagonal(grid, uplo, band_size, mat_a); -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file_band_to_tridiagonal->write(ret.tridiagonal, "/tridiag"); - } - - std::optional file_tridiag; - - if (getTuneParameters().debug_dump_trisolver_data) { - file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("tridiag")); - file_tridiag->write(ret.tridiagonal, "/tridiag"); - } -#endif - tridiagonal_eigensolver(grid, ret.tridiagonal, evals, mat_e); -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_trisolver_data) { - file_tridiag->write(evals, "/evals"); - file_tridiag->write(mat_e, "/evecs"); - } -#endif - bt_band_to_tridiagonal(grid, band_size, mat_e, ret.hh_reflectors); bt_reduction_to_band(grid, band_size, mat_e, mat_a, mat_taus); #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_eigensolver_data) { - file_eigensolver->write(evals, "/evals"); - file_eigensolver->write(mat_e, "/evecs"); + file->write(evals, "/evals"); + file->write(mat_e, "/evecs"); } num_eigensolver_calls++; diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index c61151102f..794b67f67d 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -1129,6 +1130,17 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr auto mpi_col_chain = grid.col_communicator_pipeline(); auto mpi_col_chain_panel = grid.col_communicator_pipeline(); +#ifdef DLAF_WITH_HDF5 + static size_t num_reduction_to_band_calls = 0; + std::string fname = "reduction_to_band-" + std::to_string(num_reduction_to_band_calls) + ".h5"; + std::optional file; + + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_a, "/input"); + } +#endif + const auto& dist = mat_a.distribution(); const comm::Index2D rank = dist.rankIndex(); @@ -1423,6 +1435,14 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr v.reset(); } +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file->write(mat_a, "/band"); + } + + num_reduction_to_band_calls++; +#endif + return mat_taus; } } diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index fb6c584f4c..afa3e3a355 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -451,6 +452,17 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix& tridiag, Matrix& evals, Matrix, D>& evecs) { +#ifdef DLAF_WITH_HDF5 + static size_t num_tridiag_solver_calls = 0; + std::string fname = "tridiag_solver-" + std::to_string(num_tridiag_solver_calls) + ".h5"; + std::optional file; + + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(tridiag, "/input"); + } +#endif + Matrix real_evecs(evecs.distribution()); TridiagSolver::call(grid, tridiag, evals, real_evecs); @@ -459,6 +471,16 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix(real_evecs.read(tile_wrt_local), evecs.readwrite(tile_wrt_local)); } + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(evecs, "/evecs"); + file->write(evals, "/evals"); + } + + num_tridiag_solver_calls++; +#endif } } diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index aef99baeac..82608600b0 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -38,11 +38,11 @@ namespace dlaf { /// be created in the working folder (it should not exist before the execution). /// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with /// environment variable DLAF_DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA. -/// - debug_dump_trisolver_data: -/// Enable dump of trisolver input/output data to "tridiagonal.h5" file that will be created in the -/// working folder (it should not exist before the execution). +/// - debug_dump_tridiag_solver_data: +/// Enable dump of tridiagonal solver input/output data to "tridiagonal.h5" file that will before +/// created in the working folder (it should not exist before the execution). /// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. -/// Set with environment variable DLAF_DEBUG_DUMP_TRISOLVER_DATA. +/// Set with environment variable DLAF_DEBUG_DUMP_TRIDIAG_SOLVER_DATA. /// - red2band_panel_nworkers: /// The maximum number of threads to use for computing the panel in the reduction to band algorithm. /// Set with --dlaf:red2band-panel-nworkers or env variable DLAF_RED2BAND_PANEL_NWORKERS. diff --git a/src/init.cpp b/src/init.cpp index eca50d0dbb..488d5a179e 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -226,7 +226,8 @@ void updateConfiguration(const pika::program_options::variables_map& vm, configu "DEBUG_DUMP_REDUCTION_TO_BAND_DATA", ""); updateConfigurationValue(vm, param.debug_dump_band_to_tridiagonal_data, "DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA", ""); - updateConfigurationValue(vm, param.debug_dump_trisolver_data, "DEBUG_DUMP_TRISOLVER_DATA", ""); + updateConfigurationValue(vm, param.debug_dump_tridiag_solver_data, "DEBUG_DUMP_TRDIAG_SOLVER_DATA", + ""); updateConfigurationValue(vm, param.tridiag_rank1_nworkers, "TRIDIAG_RANK1_NWORKERS", "tridiag-rank1-nworkers"); From 5f1e2db4131de0b58e22cd5bb85d5603ba4a6af3 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 1 Dec 2023 10:38:09 +0100 Subject: [PATCH 04/29] fixes --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 1 - include/dlaf/eigensolver/tridiag_solver/impl.h | 1 - src/init.cpp | 5 ++--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index 80fa39db55..326cb4439a 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -1538,7 +1538,6 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); file->write(mat_trid, "/tridiaginal"); } diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index afa3e3a355..c381ee9fc8 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -474,7 +474,6 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrixwrite(evecs, "/evecs"); file->write(evals, "/evals"); } diff --git a/src/init.cpp b/src/init.cpp index 488d5a179e..3a52329d89 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -43,8 +43,7 @@ bool& initialized() { return i; } -template -struct Init { +template struct Init { // Initialization and finalization does nothing by default. Behaviour can be // overridden for backends. static void initialize(const configuration&) {} @@ -226,7 +225,7 @@ void updateConfiguration(const pika::program_options::variables_map& vm, configu "DEBUG_DUMP_REDUCTION_TO_BAND_DATA", ""); updateConfigurationValue(vm, param.debug_dump_band_to_tridiagonal_data, "DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA", ""); - updateConfigurationValue(vm, param.debug_dump_tridiag_solver_data, "DEBUG_DUMP_TRDIAG_SOLVER_DATA", + updateConfigurationValue(vm, param.debug_dump_tridiag_solver_data, "DEBUG_DUMP_TRIDIAG_SOLVER_DATA", ""); updateConfigurationValue(vm, param.tridiag_rank1_nworkers, "TRIDIAG_RANK1_NWORKERS", From 35960e18db12346a64dab576a1790508e32f17ad Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 1 Dec 2023 10:43:10 +0100 Subject: [PATCH 05/29] formatting --- src/init.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index 3a52329d89..2eb38d7f38 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -43,7 +43,8 @@ bool& initialized() { return i; } -template struct Init { +template +struct Init { // Initialization and finalization does nothing by default. Behaviour can be // overridden for backends. static void initialize(const configuration&) {} From 5c7220d4ea80d19ce0dd026f3f126a669f88b433 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 22:53:39 +0100 Subject: [PATCH 06/29] fix conflict --- .../dlaf/eigensolver/tridiag_solver/impl.h | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index c381ee9fc8..633dd563f2 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -364,6 +364,17 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix file; + + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(tridiag, "/input"); + } +#endif + // If the matrix is composed by a single tile simply call stedc. if (evecs.nrTiles().linear_size() == 1) { if constexpr (D == Device::CPU) { @@ -442,6 +453,15 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix(row_task_chain, 0, n, ws_h.i1, ws_hm.e0, ws_hm.e2); copy(ws_hm.e2, evecs); + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file->write(evecs, "/evecs"); + file->write(evals, "/evals"); + } + + num_tridiag_solver_calls++; +#endif } // \overload TridiagSolver::call() @@ -450,19 +470,8 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix -void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix& tridiag, +void TridiagSolver::call(comm::CommunicatorGrid grid, Matrix& tridiag, Matrix& evals, Matrix, D>& evecs) { -#ifdef DLAF_WITH_HDF5 - static size_t num_tridiag_solver_calls = 0; - std::string fname = "tridiag_solver-" + std::to_string(num_tridiag_solver_calls) + ".h5"; - std::optional file; - - if (getTuneParameters().debug_dump_tridiag_solver_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); - file->write(tridiag, "/input"); - } -#endif - Matrix real_evecs(evecs.distribution()); TridiagSolver::call(grid, tridiag, evals, real_evecs); @@ -471,15 +480,6 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix(real_evecs.read(tile_wrt_local), evecs.readwrite(tile_wrt_local)); } - -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_tridiag_solver_data) { - file->write(evecs, "/evecs"); - file->write(evals, "/evals"); - } - - num_tridiag_solver_calls++; -#endif } } From e8c9e8536b7373a89e9ca29a13686f6e8eb1fe44 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 23:03:19 +0100 Subject: [PATCH 07/29] fix name --- include/dlaf/tune.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index 82608600b0..30a2781e8c 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -100,7 +100,7 @@ struct TuneParameters { bool debug_dump_eigensolver_data = false; bool debug_dump_reduction_to_band_data = false; bool debug_dump_band_to_tridiagonal_data = false; - bool debug_dump_trisolver_data = false; + bool debug_dump_tridiag_solver_data = false; std::size_t red2band_panel_nworkers = 1; std::size_t red2band_barrier_busy_wait_us = 1000; std::size_t tridiag_rank1_nworkers = 1; From fd438ffcd0838a57bf3cc51fe8696094cfc9c7af Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 23:18:42 +0100 Subject: [PATCH 08/29] fix ref --- include/dlaf/eigensolver/tridiag_solver/impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index 633dd563f2..b97e8ae510 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -470,7 +470,7 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix -void TridiagSolver::call(comm::CommunicatorGrid grid, Matrix& tridiag, +void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix& tridiag, Matrix& evals, Matrix, D>& evecs) { Matrix real_evecs(evecs.distribution()); TridiagSolver::call(grid, tridiag, evals, real_evecs); From 2c62631e101936fc4950ce3a8153214f1f3dca75 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 8 Apr 2024 18:01:24 +0200 Subject: [PATCH 09/29] Update include/dlaf/eigensolver/band_to_tridiag/mc.h Co-authored-by: Mikael Simberg --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index 326cb4439a..c94ebb6994 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -1538,7 +1538,7 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file->write(mat_trid, "/tridiaginal"); + file->write(mat_trid, "/tridiagonal"); } num_b2t_calls++; From 709c5b0e7a32cdbf3e1b2ea5c08106bbc9abf682 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 11 Apr 2024 11:19:25 +0200 Subject: [PATCH 10/29] add printout when nrefls==0 --- include/dlaf/eigensolver/reduction_to_band/impl.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index 794b67f67d..124ee9b6e2 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -1157,6 +1157,13 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr comm::Index2D(mat_a.sourceRankIndex().col(), 0))); if (nrefls == 0) +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file->write(mat_a, "/band"); + } + + num_reduction_to_band_calls++; +#endif return mat_taus; Matrix mat_taus_retiled = From b4bdd4b0784d4b91a169dc6e1718bd11137a5949 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 11 Apr 2024 11:28:40 +0200 Subject: [PATCH 11/29] fix --- include/dlaf/eigensolver/reduction_to_band/impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index 124ee9b6e2..412eab3bfb 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -1156,7 +1156,7 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr comm::Index2D(mat_a.rankIndex().col(), 0), comm::Index2D(mat_a.sourceRankIndex().col(), 0))); - if (nrefls == 0) + if (nrefls == 0) { #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_reduction_to_band_data) { file->write(mat_a, "/band"); @@ -1165,6 +1165,7 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr num_reduction_to_band_calls++; #endif return mat_taus; + } Matrix mat_taus_retiled = mat_taus.retiledSubPipeline(LocalTileSize(mat_a.blockSize().cols() / band_size, 1)); From 822d1ac6f477eea4307b5106dbbc1a8b162a5b44 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 22:47:18 +0100 Subject: [PATCH 12/29] fix conflict --- include/dlaf/eigensolver/eigensolver/impl.h | 51 +++++++++++++++++++-- include/dlaf/tune.h | 21 ++++++++- src/init.cpp | 5 ++ 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index f9ebc0c308..35ad662f8d 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -60,15 +60,49 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M if (uplo != blas::Uplo::Lower) DLAF_UNIMPLEMENTED(uplo); +#ifdef DLAF_WITH_HDF5 + std::optional file_eigensolver; + + if (getTuneParameters().debug_dump_eigensolver_data) { + file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), "eigensolver.h5"); + file_eigensolver->write(mat_a, "/input"); + } + + std::optional file_reduction_to_band; + + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), "red_to_band.h5"); + file_reduction_to_band->write(mat_a, "/input"); + } +#endif + auto mat_taus = reduction_to_band(grid, mat_a, band_size); + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file_reduction_to_band->write(mat_a, "/band"); + } + + std::optional file_band_to_tridiagonal; + + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), "band_to_tridiag.h5"); + file_band_to_tridiagonal->write(mat_a, "/band"); + } +#endif + auto ret = band_to_tridiagonal(grid, uplo, band_size, mat_a); #ifdef DLAF_WITH_HDF5 - std::optional file; + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file_band_to_tridiagonal->write(ret.tridiagonal, "/tridiag"); + } + + std::optional file_tridiag; if (getTuneParameters().debug_dump_trisolver_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), "trid-ref.h5"); - file->write(ret.tridiagonal, "/tridiag"); + file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), "tridiag.h5"); + file_tridiag->write(ret.tridiagonal, "/tridiag"); } #endif @@ -76,12 +110,19 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_trisolver_data) { - file->write(evals, "/evals"); - file->write(mat_e, "/evecs"); + file_tridiag->write(evals, "/evals"); + file_tridiag->write(mat_e, "/evecs"); } #endif bt_band_to_tridiagonal(grid, band_size, mat_e, ret.hh_reflectors); bt_reduction_to_band(grid, band_size, mat_e, mat_a, mat_taus); + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_eigensolver_data) { + file_eigensolver->write(evals, "/evals"); + file_eigensolver->write(mat_e, "/evecs"); + } +#endif } } diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index 7de7f2aa2c..aef99baeac 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -22,8 +22,24 @@ namespace dlaf { /// DLA-Future tuning parameters. /// /// Holds the value of the parameters that can be used to tune DLA-Future. +/// - debug_dump_eigensolver_data: +/// Enable dump of eigensolver input/output data to "eigensolver.h5" file that will be created in the +/// working folder (it should not exist before the execution). +/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. +/// Set with environment variable DLAF_DEBUG_DUMP_EIGENSOLVER_DATA. +/// - debug_dump_reduction_to_band_data: +/// Enable dump of reduction_to_band input/output data to "reduction_to_band.h5" file that will be +/// created in the working folder (it should not exist before the execution). +/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with +/// environment variable +/// DLAF_DEBUG_DUMP_EIGENSOLVER_DATA. +/// - debug_dump_band_to_tridiagonal_data: +/// Enable dump of band_to_trigiagonal input/output data to "band_to_tridiagonal.h5" file that will +/// be created in the working folder (it should not exist before the execution). +/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with +/// environment variable DLAF_DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA. /// - debug_dump_trisolver_data: -/// Enable dump of trisolver input/output data to "trid-ref.h5" file that will be created in the +/// Enable dump of trisolver input/output data to "tridiagonal.h5" file that will be created in the /// working folder (it should not exist before the execution). /// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. /// Set with environment variable DLAF_DEBUG_DUMP_TRISOLVER_DATA. @@ -81,6 +97,9 @@ struct TuneParameters { red2band_panel_nworkers = std::max(1, default_pool_thread_count / 2); tridiag_rank1_nworkers = default_pool_thread_count; } + bool debug_dump_eigensolver_data = false; + bool debug_dump_reduction_to_band_data = false; + bool debug_dump_band_to_tridiagonal_data = false; bool debug_dump_trisolver_data = false; std::size_t red2band_panel_nworkers = 1; std::size_t red2band_barrier_busy_wait_us = 1000; diff --git a/src/init.cpp b/src/init.cpp index 3fdba264b5..eca50d0dbb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -221,6 +221,11 @@ void updateConfiguration(const pika::program_options::variables_map& vm, configu updateConfigurationValue(vm, param.band_to_tridiag_1d_block_size_base, "BAND_TO_TRIDIAG_1D_BLOCK_SIZE_BASE", "band-to-tridiag-1d-block-size-base"); + updateConfigurationValue(vm, param.debug_dump_eigensolver_data, "DEBUG_DUMP_EIGENSOLVER_DATA", ""); + updateConfigurationValue(vm, param.debug_dump_reduction_to_band_data, + "DEBUG_DUMP_REDUCTION_TO_BAND_DATA", ""); + updateConfigurationValue(vm, param.debug_dump_band_to_tridiagonal_data, + "DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA", ""); updateConfigurationValue(vm, param.debug_dump_trisolver_data, "DEBUG_DUMP_TRISOLVER_DATA", ""); updateConfigurationValue(vm, param.tridiag_rank1_nworkers, "TRIDIAG_RANK1_NWORKERS", From 4a2f301e128b336c93fe7267587356be7020ea3b Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Wed, 8 Nov 2023 10:06:54 +0100 Subject: [PATCH 13/29] enable multiple calls --- include/dlaf/eigensolver/eigensolver/impl.h | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 35ad662f8d..2e7d97d51d 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -56,6 +56,16 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M Matrix, D>& evals, Matrix& mat_e) { const SizeType band_size = getBandSize(mat_a.blockSize().rows()); +#ifdef DLAF_WITH_HDF5 + static size_t num_eigensolver_calls = 0; + + auto fname = [](std::string name){ + std::stringstream fn; + fn << name << num_eigensolver_calls << ".h5"; + return fn.str(); + }; +#endif + // need uplo check as reduction to band doesn't have the uplo argument yet. if (uplo != blas::Uplo::Lower) DLAF_UNIMPLEMENTED(uplo); @@ -64,14 +74,14 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M std::optional file_eigensolver; if (getTuneParameters().debug_dump_eigensolver_data) { - file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), "eigensolver.h5"); + file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("eigensolver")); file_eigensolver->write(mat_a, "/input"); } std::optional file_reduction_to_band; if (getTuneParameters().debug_dump_reduction_to_band_data) { - file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), "red_to_band.h5"); + file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("red_to_band")); file_reduction_to_band->write(mat_a, "/input"); } #endif @@ -86,7 +96,7 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M std::optional file_band_to_tridiagonal; if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), "band_to_tridiag.h5"); + file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("band_to_tridiag")); file_band_to_tridiagonal->write(mat_a, "/band"); } #endif @@ -101,7 +111,7 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M std::optional file_tridiag; if (getTuneParameters().debug_dump_trisolver_data) { - file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), "tridiag.h5"); + file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("tridiag")); file_tridiag->write(ret.tridiagonal, "/tridiag"); } #endif @@ -123,6 +133,8 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M file_eigensolver->write(evals, "/evals"); file_eigensolver->write(mat_e, "/evecs"); } + + num_eigensolver_calls++; #endif } } From 4ba3e3e4b4d09abb296b5470f55c04fa2a7e326b Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 22:48:59 +0100 Subject: [PATCH 14/29] fix conflicts --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 21 +++++++ include/dlaf/eigensolver/eigensolver/impl.h | 62 +++---------------- .../dlaf/eigensolver/reduction_to_band/impl.h | 20 ++++++ .../dlaf/eigensolver/tridiag_solver/impl.h | 22 +++++++ include/dlaf/tune.h | 8 +-- src/init.cpp | 3 +- 6 files changed, 76 insertions(+), 60 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index a2c55d940a..80fa39db55 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1034,6 +1035,17 @@ TridiagResult BandToTridiag::call_L( // Should be dispatched to local implementation if (1x1) grid. DLAF_ASSERT(grid.size() != comm::Size2D(1, 1), grid); +#ifdef DLAF_WITH_HDF5 + static size_t num_b2t_calls = 0; + std::string fname = "band_to_tridiag-" + std::to_string(num_b2t_calls) + ".h5"; + std::optional file; + + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_a, "/input"); + } +#endif + // note: A is square and has square blocksize SizeType size = mat_a.size().cols(); SizeType n = mat_a.nrTiles().cols(); @@ -1524,6 +1536,15 @@ TridiagResult BandToTridiag::call_L( mat_trid.readwrite(index))); } +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_trid, "/tridiaginal"); + } + + num_b2t_calls++; +#endif + return {std::move(mat_trid), std::move(mat_v)}; } } diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 2e7d97d51d..9f5c2bf069 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -56,82 +56,34 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M Matrix, D>& evals, Matrix& mat_e) { const SizeType band_size = getBandSize(mat_a.blockSize().rows()); -#ifdef DLAF_WITH_HDF5 - static size_t num_eigensolver_calls = 0; - - auto fname = [](std::string name){ - std::stringstream fn; - fn << name << num_eigensolver_calls << ".h5"; - return fn.str(); - }; -#endif - // need uplo check as reduction to band doesn't have the uplo argument yet. if (uplo != blas::Uplo::Lower) DLAF_UNIMPLEMENTED(uplo); #ifdef DLAF_WITH_HDF5 - std::optional file_eigensolver; + static size_t num_eigensolver_calls = 0; + std::string fname = "eigensolver-" + std::to_string(num_eigensolver_calls) + ".h5"; + std::optional file; if (getTuneParameters().debug_dump_eigensolver_data) { - file_eigensolver = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("eigensolver")); - file_eigensolver->write(mat_a, "/input"); - } - - std::optional file_reduction_to_band; - - if (getTuneParameters().debug_dump_reduction_to_band_data) { - file_reduction_to_band = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("red_to_band")); - file_reduction_to_band->write(mat_a, "/input"); + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_a, "/input"); } #endif auto mat_taus = reduction_to_band(grid, mat_a, band_size); -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_reduction_to_band_data) { - file_reduction_to_band->write(mat_a, "/band"); - } - - std::optional file_band_to_tridiagonal; - - if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file_band_to_tridiagonal = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("band_to_tridiag")); - file_band_to_tridiagonal->write(mat_a, "/band"); - } -#endif - auto ret = band_to_tridiagonal(grid, uplo, band_size, mat_a); -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file_band_to_tridiagonal->write(ret.tridiagonal, "/tridiag"); - } - - std::optional file_tridiag; - - if (getTuneParameters().debug_dump_trisolver_data) { - file_tridiag = matrix::internal::FileHDF5(grid.fullCommunicator(), fname("tridiag")); - file_tridiag->write(ret.tridiagonal, "/tridiag"); - } -#endif - tridiagonal_eigensolver(grid, ret.tridiagonal, evals, mat_e); -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_trisolver_data) { - file_tridiag->write(evals, "/evals"); - file_tridiag->write(mat_e, "/evecs"); - } -#endif - bt_band_to_tridiagonal(grid, band_size, mat_e, ret.hh_reflectors); bt_reduction_to_band(grid, band_size, mat_e, mat_a, mat_taus); #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_eigensolver_data) { - file_eigensolver->write(evals, "/evals"); - file_eigensolver->write(mat_e, "/evecs"); + file->write(evals, "/evals"); + file->write(mat_e, "/evecs"); } num_eigensolver_calls++; diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index c61151102f..794b67f67d 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -1129,6 +1130,17 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr auto mpi_col_chain = grid.col_communicator_pipeline(); auto mpi_col_chain_panel = grid.col_communicator_pipeline(); +#ifdef DLAF_WITH_HDF5 + static size_t num_reduction_to_band_calls = 0; + std::string fname = "reduction_to_band-" + std::to_string(num_reduction_to_band_calls) + ".h5"; + std::optional file; + + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(mat_a, "/input"); + } +#endif + const auto& dist = mat_a.distribution(); const comm::Index2D rank = dist.rankIndex(); @@ -1423,6 +1435,14 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr v.reset(); } +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file->write(mat_a, "/band"); + } + + num_reduction_to_band_calls++; +#endif + return mat_taus; } } diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index fb6c584f4c..afa3e3a355 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -451,6 +452,17 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix& tridiag, Matrix& evals, Matrix, D>& evecs) { +#ifdef DLAF_WITH_HDF5 + static size_t num_tridiag_solver_calls = 0; + std::string fname = "tridiag_solver-" + std::to_string(num_tridiag_solver_calls) + ".h5"; + std::optional file; + + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(tridiag, "/input"); + } +#endif + Matrix real_evecs(evecs.distribution()); TridiagSolver::call(grid, tridiag, evals, real_evecs); @@ -459,6 +471,16 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix(real_evecs.read(tile_wrt_local), evecs.readwrite(tile_wrt_local)); } + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(evecs, "/evecs"); + file->write(evals, "/evals"); + } + + num_tridiag_solver_calls++; +#endif } } diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index aef99baeac..82608600b0 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -38,11 +38,11 @@ namespace dlaf { /// be created in the working folder (it should not exist before the execution). /// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with /// environment variable DLAF_DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA. -/// - debug_dump_trisolver_data: -/// Enable dump of trisolver input/output data to "tridiagonal.h5" file that will be created in the -/// working folder (it should not exist before the execution). +/// - debug_dump_tridiag_solver_data: +/// Enable dump of tridiagonal solver input/output data to "tridiagonal.h5" file that will before +/// created in the working folder (it should not exist before the execution). /// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. -/// Set with environment variable DLAF_DEBUG_DUMP_TRISOLVER_DATA. +/// Set with environment variable DLAF_DEBUG_DUMP_TRIDIAG_SOLVER_DATA. /// - red2band_panel_nworkers: /// The maximum number of threads to use for computing the panel in the reduction to band algorithm. /// Set with --dlaf:red2band-panel-nworkers or env variable DLAF_RED2BAND_PANEL_NWORKERS. diff --git a/src/init.cpp b/src/init.cpp index eca50d0dbb..488d5a179e 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -226,7 +226,8 @@ void updateConfiguration(const pika::program_options::variables_map& vm, configu "DEBUG_DUMP_REDUCTION_TO_BAND_DATA", ""); updateConfigurationValue(vm, param.debug_dump_band_to_tridiagonal_data, "DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA", ""); - updateConfigurationValue(vm, param.debug_dump_trisolver_data, "DEBUG_DUMP_TRISOLVER_DATA", ""); + updateConfigurationValue(vm, param.debug_dump_tridiag_solver_data, "DEBUG_DUMP_TRDIAG_SOLVER_DATA", + ""); updateConfigurationValue(vm, param.tridiag_rank1_nworkers, "TRIDIAG_RANK1_NWORKERS", "tridiag-rank1-nworkers"); From 338e51ea231ec204ccc2e2543e7e51d6bc915c0d Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 1 Dec 2023 10:38:09 +0100 Subject: [PATCH 15/29] fixes --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 1 - include/dlaf/eigensolver/tridiag_solver/impl.h | 1 - src/init.cpp | 5 ++--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index 80fa39db55..326cb4439a 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -1538,7 +1538,6 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); file->write(mat_trid, "/tridiaginal"); } diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index afa3e3a355..c381ee9fc8 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -474,7 +474,6 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrixwrite(evecs, "/evecs"); file->write(evals, "/evals"); } diff --git a/src/init.cpp b/src/init.cpp index 488d5a179e..3a52329d89 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -43,8 +43,7 @@ bool& initialized() { return i; } -template -struct Init { +template struct Init { // Initialization and finalization does nothing by default. Behaviour can be // overridden for backends. static void initialize(const configuration&) {} @@ -226,7 +225,7 @@ void updateConfiguration(const pika::program_options::variables_map& vm, configu "DEBUG_DUMP_REDUCTION_TO_BAND_DATA", ""); updateConfigurationValue(vm, param.debug_dump_band_to_tridiagonal_data, "DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA", ""); - updateConfigurationValue(vm, param.debug_dump_tridiag_solver_data, "DEBUG_DUMP_TRDIAG_SOLVER_DATA", + updateConfigurationValue(vm, param.debug_dump_tridiag_solver_data, "DEBUG_DUMP_TRIDIAG_SOLVER_DATA", ""); updateConfigurationValue(vm, param.tridiag_rank1_nworkers, "TRIDIAG_RANK1_NWORKERS", From 2c265fa52f2179ffda921cf2996dbfa2fbbfba74 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 1 Dec 2023 10:43:10 +0100 Subject: [PATCH 16/29] formatting --- src/init.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index 3a52329d89..2eb38d7f38 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -43,7 +43,8 @@ bool& initialized() { return i; } -template struct Init { +template +struct Init { // Initialization and finalization does nothing by default. Behaviour can be // overridden for backends. static void initialize(const configuration&) {} From 3785c2e5f3a17893bcb1e4dc7ae8b7367364dd3f Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 22:53:39 +0100 Subject: [PATCH 17/29] fix conflict --- .../dlaf/eigensolver/tridiag_solver/impl.h | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index c381ee9fc8..633dd563f2 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -364,6 +364,17 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix file; + + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file->write(tridiag, "/input"); + } +#endif + // If the matrix is composed by a single tile simply call stedc. if (evecs.nrTiles().linear_size() == 1) { if constexpr (D == Device::CPU) { @@ -442,6 +453,15 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix(row_task_chain, 0, n, ws_h.i1, ws_hm.e0, ws_hm.e2); copy(ws_hm.e2, evecs); + +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_tridiag_solver_data) { + file->write(evecs, "/evecs"); + file->write(evals, "/evals"); + } + + num_tridiag_solver_calls++; +#endif } // \overload TridiagSolver::call() @@ -450,19 +470,8 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix -void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix& tridiag, +void TridiagSolver::call(comm::CommunicatorGrid grid, Matrix& tridiag, Matrix& evals, Matrix, D>& evecs) { -#ifdef DLAF_WITH_HDF5 - static size_t num_tridiag_solver_calls = 0; - std::string fname = "tridiag_solver-" + std::to_string(num_tridiag_solver_calls) + ".h5"; - std::optional file; - - if (getTuneParameters().debug_dump_tridiag_solver_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); - file->write(tridiag, "/input"); - } -#endif - Matrix real_evecs(evecs.distribution()); TridiagSolver::call(grid, tridiag, evals, real_evecs); @@ -471,15 +480,6 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix(real_evecs.read(tile_wrt_local), evecs.readwrite(tile_wrt_local)); } - -#ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_tridiag_solver_data) { - file->write(evecs, "/evecs"); - file->write(evals, "/evals"); - } - - num_tridiag_solver_calls++; -#endif } } From 01114f2972d9739adccdd575ff7aea7f26ffb49f Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 23:03:19 +0100 Subject: [PATCH 18/29] fix name --- include/dlaf/tune.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index 82608600b0..30a2781e8c 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -100,7 +100,7 @@ struct TuneParameters { bool debug_dump_eigensolver_data = false; bool debug_dump_reduction_to_band_data = false; bool debug_dump_band_to_tridiagonal_data = false; - bool debug_dump_trisolver_data = false; + bool debug_dump_tridiag_solver_data = false; std::size_t red2band_panel_nworkers = 1; std::size_t red2band_barrier_busy_wait_us = 1000; std::size_t tridiag_rank1_nworkers = 1; From 4865bc1fd82bbd4b6318f12a1eaf6fae3808b982 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 25 Mar 2024 23:18:42 +0100 Subject: [PATCH 19/29] fix ref --- include/dlaf/eigensolver/tridiag_solver/impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index 633dd563f2..b97e8ae510 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -470,7 +470,7 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix -void TridiagSolver::call(comm::CommunicatorGrid grid, Matrix& tridiag, +void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix& tridiag, Matrix& evals, Matrix, D>& evecs) { Matrix real_evecs(evecs.distribution()); TridiagSolver::call(grid, tridiag, evals, real_evecs); From 73bbe3df64b216baee23eddb0f1a22ee3e658de1 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 11 Apr 2024 11:19:25 +0200 Subject: [PATCH 20/29] add printout when nrefls==0 --- include/dlaf/eigensolver/reduction_to_band/impl.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index 794b67f67d..124ee9b6e2 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -1157,6 +1157,13 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr comm::Index2D(mat_a.sourceRankIndex().col(), 0))); if (nrefls == 0) +#ifdef DLAF_WITH_HDF5 + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file->write(mat_a, "/band"); + } + + num_reduction_to_band_calls++; +#endif return mat_taus; Matrix mat_taus_retiled = From a46c984b99d0d7f2cb8c00db063683c3cc1a3398 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 11 Apr 2024 11:28:40 +0200 Subject: [PATCH 21/29] fix --- include/dlaf/eigensolver/reduction_to_band/impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index 124ee9b6e2..412eab3bfb 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -1156,7 +1156,7 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr comm::Index2D(mat_a.rankIndex().col(), 0), comm::Index2D(mat_a.sourceRankIndex().col(), 0))); - if (nrefls == 0) + if (nrefls == 0) { #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_reduction_to_band_data) { file->write(mat_a, "/band"); @@ -1165,6 +1165,7 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr num_reduction_to_band_calls++; #endif return mat_taus; + } Matrix mat_taus_retiled = mat_taus.retiledSubPipeline(LocalTileSize(mat_a.blockSize().cols() / band_size, 1)); From 84bbca7f3b8acd3f2a4a8708def84966f20a33de Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Mon, 8 Apr 2024 18:01:24 +0200 Subject: [PATCH 22/29] Update include/dlaf/eigensolver/band_to_tridiag/mc.h Co-authored-by: Mikael Simberg --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index 326cb4439a..c94ebb6994 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -1538,7 +1538,7 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file->write(mat_trid, "/tridiaginal"); + file->write(mat_trid, "/tridiagonal"); } num_b2t_calls++; From 5d90c9c471e951e8aa2e3a0921919c0957c7be15 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Tue, 23 Apr 2024 18:18:02 +0200 Subject: [PATCH 23/29] remove old comments --- include/dlaf/tune.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/dlaf/tune.h b/include/dlaf/tune.h index 30a2781e8c..f5450d7cfb 100644 --- a/include/dlaf/tune.h +++ b/include/dlaf/tune.h @@ -25,23 +25,19 @@ namespace dlaf { /// - debug_dump_eigensolver_data: /// Enable dump of eigensolver input/output data to "eigensolver.h5" file that will be created in the /// working folder (it should not exist before the execution). -/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. /// Set with environment variable DLAF_DEBUG_DUMP_EIGENSOLVER_DATA. /// - debug_dump_reduction_to_band_data: /// Enable dump of reduction_to_band input/output data to "reduction_to_band.h5" file that will be /// created in the working folder (it should not exist before the execution). -/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with /// environment variable /// DLAF_DEBUG_DUMP_EIGENSOLVER_DATA. /// - debug_dump_band_to_tridiagonal_data: /// Enable dump of band_to_trigiagonal input/output data to "band_to_tridiagonal.h5" file that will /// be created in the working folder (it should not exist before the execution). -/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. Set with /// environment variable DLAF_DEBUG_DUMP_BAND_TO_TRIDIAGONAL_DATA. /// - debug_dump_tridiag_solver_data: /// Enable dump of tridiagonal solver input/output data to "tridiagonal.h5" file that will before /// created in the working folder (it should not exist before the execution). -/// WARNING: just a single execution can be dumped on disk, and any subsequent call fails. /// Set with environment variable DLAF_DEBUG_DUMP_TRIDIAG_SOLVER_DATA. /// - red2band_panel_nworkers: /// The maximum number of threads to use for computing the panel in the reduction to band algorithm. From 57f47dc7aa393ae154b24e6f857937d223897200 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Tue, 23 Apr 2024 18:41:31 +0200 Subject: [PATCH 24/29] add type in file name and make counters atomic --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 6 +++-- include/dlaf/eigensolver/eigensolver/impl.h | 5 ++-- .../dlaf/eigensolver/reduction_to_band/impl.h | 5 ++-- include/dlaf/matrix/hdf5.h | 26 +++++++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index c94ebb6994..ad334e64d2 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -10,6 +10,8 @@ #pragma once +#include + #include #include @@ -1036,8 +1038,8 @@ TridiagResult BandToTridiag::call_L( DLAF_ASSERT(grid.size() != comm::Size2D(1, 1), grid); #ifdef DLAF_WITH_HDF5 - static size_t num_b2t_calls = 0; - std::string fname = "band_to_tridiag-" + std::to_string(num_b2t_calls) + ".h5"; + static std::atomic num_b2t_calls = 0; + std::string fname = "band_to_tridiag-" + matrix::internal::TypeToString::value + "-" + std::to_string(num_b2t_calls) + ".h5"; std::optional file; if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 9f5c2bf069..17ab14f14a 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -9,6 +9,7 @@ // #pragma once +#include #include #include #include @@ -61,8 +62,8 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M DLAF_UNIMPLEMENTED(uplo); #ifdef DLAF_WITH_HDF5 - static size_t num_eigensolver_calls = 0; - std::string fname = "eigensolver-" + std::to_string(num_eigensolver_calls) + ".h5"; + static std::atomic num_eigensolver_calls = 0; + std::string fname = "eigensolver-" + matrix::internal::TypeToString::value + "-" + std::to_string(num_eigensolver_calls) + ".h5"; std::optional file; if (getTuneParameters().debug_dump_eigensolver_data) { diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index 412eab3bfb..c170bfc751 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -9,6 +9,7 @@ // #pragma once +#include #include #include #include @@ -1131,8 +1132,8 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr auto mpi_col_chain_panel = grid.col_communicator_pipeline(); #ifdef DLAF_WITH_HDF5 - static size_t num_reduction_to_band_calls = 0; - std::string fname = "reduction_to_band-" + std::to_string(num_reduction_to_band_calls) + ".h5"; + static std::atomic num_reduction_to_band_calls = 0; + std::string fname = "reduction_to_band-" + matrix::internal::TypeToString::value + "-" + std::to_string(num_reduction_to_band_calls) + ".h5"; std::optional file; if (getTuneParameters().debug_dump_reduction_to_band_data) { diff --git a/include/dlaf/matrix/hdf5.h b/include/dlaf/matrix/hdf5.h index 50e8b2d334..067b054167 100644 --- a/include/dlaf/matrix/hdf5.h +++ b/include/dlaf/matrix/hdf5.h @@ -53,6 +53,32 @@ const H5::PredType& hdf5_datatype>::type = hdf5_datatype::typ template struct hdf5_datatype : public hdf5_datatype {}; +// Type to string mappings +template +struct TypeToString { + static inline const std::string value = typeid(T).name(); +}; + +template<> +struct TypeToString { + static inline const std::string value = "s"; +}; + +template<> +struct TypeToString { + static inline const std::string value = "d"; +}; + +template<> +struct TypeToString> { + static inline const std::string value = "c"; +}; + +template<> +struct TypeToString> { + static inline const std::string value = "z"; +}; + // Helper function that for each local tile index in @p dist, gets a sender of a tile with // @p get_tile and sends it to a function that takes care of the mapping between file and memory. // Then, this function, passes all required arguments to @p dataset_op which should be either From 112b615810b423e2a852b5b5bbfbd6c25b21de42 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 25 Apr 2024 14:29:16 +0000 Subject: [PATCH 25/29] update --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 6 ++++-- include/dlaf/eigensolver/eigensolver/impl.h | 6 ++++-- include/dlaf/eigensolver/reduction_to_band/impl.h | 7 +++++-- include/dlaf/eigensolver/tridiag_solver/impl.h | 9 ++++++--- include/dlaf/matrix/hdf5.h | 13 ++++++++----- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index ad334e64d2..1b616ace46 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -11,6 +11,7 @@ #pragma once #include +#include #include #include @@ -1039,11 +1040,12 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 static std::atomic num_b2t_calls = 0; - std::string fname = "band_to_tridiag-" + matrix::internal::TypeToString::value + "-" + std::to_string(num_b2t_calls) + ".h5"; + std::stringstream fname; + fname << "band_to_tridiag-" << matrix::internal::TypeToString::value << "-" << std::to_string(num_b2t_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname.str()); file->write(mat_a, "/input"); } #endif diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 17ab14f14a..812cff218e 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -63,11 +64,12 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M #ifdef DLAF_WITH_HDF5 static std::atomic num_eigensolver_calls = 0; - std::string fname = "eigensolver-" + matrix::internal::TypeToString::value + "-" + std::to_string(num_eigensolver_calls) + ".h5"; + std::stringstream fname; + fname << "eigensolver-" << matrix::internal::TypeToString::value << "-" << std::to_string(num_eigensolver_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_eigensolver_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname.str()); file->write(mat_a, "/input"); } #endif diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index c170bfc751..e09761b12e 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -1133,11 +1134,12 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr #ifdef DLAF_WITH_HDF5 static std::atomic num_reduction_to_band_calls = 0; - std::string fname = "reduction_to_band-" + matrix::internal::TypeToString::value + "-" + std::to_string(num_reduction_to_band_calls) + ".h5"; + std::stringstream fname; + fname << "reduction_to_band-" << matrix::internal::TypeToString::value << "-" << std::to_string(num_reduction_to_band_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_reduction_to_band_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname.str()); file->write(mat_a, "/input"); } #endif @@ -1165,6 +1167,7 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr num_reduction_to_band_calls++; #endif + return mat_taus; } diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index b97e8ae510..01cdd2fa53 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -10,6 +10,8 @@ #pragma once #include +#include +#include #include #include @@ -365,12 +367,13 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix num_tridiag_solver_calls = 0; + std::stringstream fname; + fname << "tridiag_solver-" << matrix::internal::TypeToString_v << std::to_string(num_tridiag_solver_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_tridiag_solver_data) { - file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname); + file = matrix::internal::FileHDF5(grid.fullCommunicator(), fname.str()); file->write(tridiag, "/input"); } #endif diff --git a/include/dlaf/matrix/hdf5.h b/include/dlaf/matrix/hdf5.h index 067b054167..865dc9a900 100644 --- a/include/dlaf/matrix/hdf5.h +++ b/include/dlaf/matrix/hdf5.h @@ -56,27 +56,30 @@ struct hdf5_datatype : public hdf5_datatype {}; // Type to string mappings template struct TypeToString { - static inline const std::string value = typeid(T).name(); + static inline constexpr std::string_view value = typeid(T).name(); }; +template +inline constexpr std::string_view TypeToString_v = TypeToString::value; + template<> struct TypeToString { - static inline const std::string value = "s"; + static inline constexpr std::string_view value = "s"; }; template<> struct TypeToString { - static inline const std::string value = "d"; + static inline constexpr std::string value = "d"; }; template<> struct TypeToString> { - static inline const std::string value = "c"; + static inline constexpr std::string_view value = "c"; }; template<> struct TypeToString> { - static inline const std::string value = "z"; + static inline constexpr std::string_view value = "z"; }; // Helper function that for each local tile index in @p dist, gets a sender of a tile with From 9eb3b273ce13976b78fbbf7d75caeaab0b325efb Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 25 Apr 2024 14:31:07 +0000 Subject: [PATCH 26/29] format --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 3 ++- include/dlaf/eigensolver/eigensolver/impl.h | 3 ++- include/dlaf/eigensolver/reduction_to_band/impl.h | 11 ++++++----- include/dlaf/eigensolver/tridiag_solver/impl.h | 3 ++- include/dlaf/matrix/hdf5.h | 10 +++++----- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index 1b616ace46..ebdb1a3719 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -1041,7 +1041,8 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 static std::atomic num_b2t_calls = 0; std::stringstream fname; - fname << "band_to_tridiag-" << matrix::internal::TypeToString::value << "-" << std::to_string(num_b2t_calls) << ".h5"; + fname << "band_to_tridiag-" << matrix::internal::TypeToString::value << "-" + << std::to_string(num_b2t_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_band_to_tridiagonal_data) { diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 812cff218e..8df28e9476 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -65,7 +65,8 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M #ifdef DLAF_WITH_HDF5 static std::atomic num_eigensolver_calls = 0; std::stringstream fname; - fname << "eigensolver-" << matrix::internal::TypeToString::value << "-" << std::to_string(num_eigensolver_calls) << ".h5"; + fname << "eigensolver-" << matrix::internal::TypeToString::value << "-" + << std::to_string(num_eigensolver_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_eigensolver_data) { diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index e09761b12e..ed4dffc188 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -1135,7 +1135,8 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr #ifdef DLAF_WITH_HDF5 static std::atomic num_reduction_to_band_calls = 0; std::stringstream fname; - fname << "reduction_to_band-" << matrix::internal::TypeToString::value << "-" << std::to_string(num_reduction_to_band_calls) << ".h5"; + fname << "reduction_to_band-" << matrix::internal::TypeToString::value << "-" + << std::to_string(num_reduction_to_band_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_reduction_to_band_data) { @@ -1161,11 +1162,11 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr if (nrefls == 0) { #ifdef DLAF_WITH_HDF5 - if (getTuneParameters().debug_dump_reduction_to_band_data) { - file->write(mat_a, "/band"); - } + if (getTuneParameters().debug_dump_reduction_to_band_data) { + file->write(mat_a, "/band"); + } - num_reduction_to_band_calls++; + num_reduction_to_band_calls++; #endif return mat_taus; diff --git a/include/dlaf/eigensolver/tridiag_solver/impl.h b/include/dlaf/eigensolver/tridiag_solver/impl.h index 01cdd2fa53..0e1f7c4c94 100644 --- a/include/dlaf/eigensolver/tridiag_solver/impl.h +++ b/include/dlaf/eigensolver/tridiag_solver/impl.h @@ -369,7 +369,8 @@ void TridiagSolver::call(comm::CommunicatorGrid& grid, Matrix num_tridiag_solver_calls = 0; std::stringstream fname; - fname << "tridiag_solver-" << matrix::internal::TypeToString_v << std::to_string(num_tridiag_solver_calls) << ".h5"; + fname << "tridiag_solver-" + << matrix::internal::TypeToString_v << std::to_string(num_tridiag_solver_calls) << ".h5"; std::optional file; if (getTuneParameters().debug_dump_tridiag_solver_data) { diff --git a/include/dlaf/matrix/hdf5.h b/include/dlaf/matrix/hdf5.h index 865dc9a900..1161e98215 100644 --- a/include/dlaf/matrix/hdf5.h +++ b/include/dlaf/matrix/hdf5.h @@ -54,7 +54,7 @@ template struct hdf5_datatype : public hdf5_datatype {}; // Type to string mappings -template +template struct TypeToString { static inline constexpr std::string_view value = typeid(T).name(); }; @@ -62,22 +62,22 @@ struct TypeToString { template inline constexpr std::string_view TypeToString_v = TypeToString::value; -template<> +template <> struct TypeToString { static inline constexpr std::string_view value = "s"; }; -template<> +template <> struct TypeToString { static inline constexpr std::string value = "d"; }; -template<> +template <> struct TypeToString> { static inline constexpr std::string_view value = "c"; }; -template<> +template <> struct TypeToString> { static inline constexpr std::string_view value = "z"; }; From ca00c874dca8ad96cc42d21727d6b70e68e54742 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 25 Apr 2024 18:37:04 +0000 Subject: [PATCH 27/29] add missing changes, remove unrelated changes --- CMakeLists.txt | 2 +- ci/common-ci.yml | 2 +- ci/docker/codecov.Dockerfile | 1 - ci/docker/deploy.Dockerfile | 1 - include/dlaf/matrix/hdf5.h | 4 ++-- src/CMakeLists.txt | 5 ----- 6 files changed, 4 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 577099933a..6c1c2edbc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ cmake_minimum_required(VERSION 3.22) -project(DLAF VERSION 0.4.0) +project(DLAF VERSION 0.1.0) # --------------------------------------------------------------------------- # CMake configurations diff --git a/ci/common-ci.yml b/ci/common-ci.yml index 38c635c488..626f81edc2 100644 --- a/ci/common-ci.yml +++ b/ci/common-ci.yml @@ -34,7 +34,7 @@ stages: reports: dotenv: build.env variables: - SPACK_SHA: 4b0479159feed0d9bcd6f1d75b166a6ac67f9602 + SPACK_SHA: dcc4423a9d0219a26ceeb52e329ed956e41f4d2c SPACK_DLAF_REPO: ./spack DOCKER_BUILD_ARGS: '[ "BASE_IMAGE", diff --git a/ci/docker/codecov.Dockerfile b/ci/docker/codecov.Dockerfile index d9b401d376..b59aebe309 100644 --- a/ci/docker/codecov.Dockerfile +++ b/ci/docker/codecov.Dockerfile @@ -29,7 +29,6 @@ RUN spack repo rm --scope site dlaf && \ spack -e ci develop --no-clone -p ${SOURCE} dla-future@master && \ spack -e ci concretize -f && \ mkdir ${BUILD} && \ - mkdir -p $(dirname $(spack -e ci location -b dla-future)) && \ ln -s ${BUILD} `spack -e ci location -b dla-future` && \ spack -e ci --config "config:flags:keep_werror:all" install --jobs ${NUM_PROCS} --keep-stage --verbose diff --git a/ci/docker/deploy.Dockerfile b/ci/docker/deploy.Dockerfile index f7ea668704..78ba89dd06 100644 --- a/ci/docker/deploy.Dockerfile +++ b/ci/docker/deploy.Dockerfile @@ -26,7 +26,6 @@ RUN spack repo rm --scope site dlaf && \ spack -e ci develop --no-clone -p ${SOURCE} dla-future@master && \ spack -e ci concretize -f && \ mkdir ${BUILD} && \ - mkdir -p $(dirname $(spack -e ci location -b dla-future)) && \ ln -s ${BUILD} `spack -e ci location -b dla-future` && \ spack -e ci --config "config:flags:keep_werror:all" install --jobs ${NUM_PROCS} --keep-stage --verbose diff --git a/include/dlaf/matrix/hdf5.h b/include/dlaf/matrix/hdf5.h index 1161e98215..f0148a213f 100644 --- a/include/dlaf/matrix/hdf5.h +++ b/include/dlaf/matrix/hdf5.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include @@ -69,7 +69,7 @@ struct TypeToString { template <> struct TypeToString { - static inline constexpr std::string value = "d"; + static inline constexpr std::string_view value = "d"; }; template <> diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cdc1f1fd0c..986fde4ec1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -353,11 +353,6 @@ configure_package_config_file( INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${CMAKE_PROJECT_NAME} ) -write_basic_package_version_file( - DLAFConfigVersion.cmake VERSION ${PACKAGE_VERSION} COMPATIBILITY AnyNewerVersion -) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/DLAFConfig.cmake - ${CMAKE_CURRENT_BINARY_DIR}/DLAFConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${CMAKE_PROJECT_NAME} ) From f3fe0f184aad37ab561513cc352aa16fb13b3556 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Thu, 25 Apr 2024 18:39:01 +0000 Subject: [PATCH 28/29] add back string --- include/dlaf/matrix/hdf5.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/dlaf/matrix/hdf5.h b/include/dlaf/matrix/hdf5.h index f0148a213f..c45670e58f 100644 --- a/include/dlaf/matrix/hdf5.h +++ b/include/dlaf/matrix/hdf5.h @@ -14,6 +14,7 @@ #include #include +#include #include #include From 315f95a4eade9bc045e8a7ede52302ae87ebb3c7 Mon Sep 17 00:00:00 2001 From: Rocco Meli Date: Fri, 26 Apr 2024 10:23:16 +0200 Subject: [PATCH 29/29] Apply suggestions from code review Co-authored-by: Mikael Simberg --- include/dlaf/eigensolver/band_to_tridiag/mc.h | 2 +- include/dlaf/eigensolver/eigensolver/impl.h | 2 +- include/dlaf/eigensolver/reduction_to_band/impl.h | 2 +- include/dlaf/matrix/hdf5.h | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/dlaf/eigensolver/band_to_tridiag/mc.h b/include/dlaf/eigensolver/band_to_tridiag/mc.h index ebdb1a3719..86146d88d7 100644 --- a/include/dlaf/eigensolver/band_to_tridiag/mc.h +++ b/include/dlaf/eigensolver/band_to_tridiag/mc.h @@ -1041,7 +1041,7 @@ TridiagResult BandToTridiag::call_L( #ifdef DLAF_WITH_HDF5 static std::atomic num_b2t_calls = 0; std::stringstream fname; - fname << "band_to_tridiag-" << matrix::internal::TypeToString::value << "-" + fname << "band_to_tridiag-" << matrix::internal::TypeToString_v << "-" << std::to_string(num_b2t_calls) << ".h5"; std::optional file; diff --git a/include/dlaf/eigensolver/eigensolver/impl.h b/include/dlaf/eigensolver/eigensolver/impl.h index 8df28e9476..e39a246005 100644 --- a/include/dlaf/eigensolver/eigensolver/impl.h +++ b/include/dlaf/eigensolver/eigensolver/impl.h @@ -65,7 +65,7 @@ void Eigensolver::call(comm::CommunicatorGrid& grid, blas::Uplo uplo, M #ifdef DLAF_WITH_HDF5 static std::atomic num_eigensolver_calls = 0; std::stringstream fname; - fname << "eigensolver-" << matrix::internal::TypeToString::value << "-" + fname << "eigensolver-" << matrix::internal::TypeToString_v << "-" << std::to_string(num_eigensolver_calls) << ".h5"; std::optional file; diff --git a/include/dlaf/eigensolver/reduction_to_band/impl.h b/include/dlaf/eigensolver/reduction_to_band/impl.h index ed4dffc188..148078d422 100644 --- a/include/dlaf/eigensolver/reduction_to_band/impl.h +++ b/include/dlaf/eigensolver/reduction_to_band/impl.h @@ -1135,7 +1135,7 @@ Matrix ReductionToBand::call(comm::CommunicatorGrid& gr #ifdef DLAF_WITH_HDF5 static std::atomic num_reduction_to_band_calls = 0; std::stringstream fname; - fname << "reduction_to_band-" << matrix::internal::TypeToString::value << "-" + fname << "reduction_to_band-" << matrix::internal::TypeToString_v << "-" << std::to_string(num_reduction_to_band_calls) << ".h5"; std::optional file; diff --git a/include/dlaf/matrix/hdf5.h b/include/dlaf/matrix/hdf5.h index c45670e58f..30be778aec 100644 --- a/include/dlaf/matrix/hdf5.h +++ b/include/dlaf/matrix/hdf5.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include