Replace CUDA with Kokkos library

Signed-off-by: Krzysztof Bieganski <kbieganski@antmicro.com>
Signed-off-by: Kamil Rakoczy <krakoczy@antmicro.com>

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "src/abc"]
 	path = third-party/abc
 	url = ../../The-OpenROAD-Project/abc.git
+[submodule "src/gpl2/kokkos-fft"]
+	path = src/gpl2/kokkos-fft
+	url = https://github.com/kokkos/kokkos-fft.git

include/ord/Design.h

@@ -70,7 +70,7 @@ class Replace;
 }
 
 namespace gpl2 {
-class GpuReplace;
+class DgReplace;
 }
 
 namespace dpl {
@@ -181,7 +181,7 @@ class Design
   ant::AntennaChecker* getAntennaChecker();
   grt::GlobalRouter* getGlobalRouter();
   gpl::Replace* getReplace();
-  gpl2::GpuReplace* getGpuReplace();
+  gpl2::DgReplace* getDgReplace();
   dpl::Opendp* getOpendp();
   mpl::MacroPlacer* getMacroPlacer();
   ppl::IOPlacer* getIOPlacer();

include/ord/OpenRoad.hh

@@ -108,7 +108,7 @@ class Replace;
 }
 
 namespace gpl2 {
-class GpuReplace;
+class DgReplace;
 }
 
 namespace rcx {
@@ -189,7 +189,7 @@ class OpenRoad
   rcx::Ext* getOpenRCX() { return extractor_; }
   drt::TritonRoute* getTritonRoute() { return detailed_router_; }
   gpl::Replace* getReplace() { return replace_; }
-  gpl2::GpuReplace* getGpuReplace() { return gpu_replace_; }
+  gpl2::DgReplace* getDgReplace() { return dg_replace_; }
   psm::PDNSim* getPDNSim() { return pdnsim_; }
   grt::GlobalRouter* getGlobalRouter() { return global_router_; }
   par::PartitionMgr* getPartitionMgr() { return partitionMgr_; }
@@ -285,7 +285,7 @@ class OpenRoad
   drt::TritonRoute* detailed_router_ = nullptr;
   ant::AntennaChecker* antenna_checker_ = nullptr;
   gpl::Replace* replace_ = nullptr;
-  gpl2::GpuReplace* gpu_replace_ = nullptr;
+  gpl2::DgReplace* dg_replace_ = nullptr;
   psm::PDNSim* pdnsim_ = nullptr;
   par::PartitionMgr* partitionMgr_ = nullptr;
   pdn::PdnGen* pdngen_ = nullptr;

src/CMakeLists.txt

@@ -190,7 +190,6 @@ add_compile_options(
   -Wredundant-decls
   -Wformat-security
   -Wno-sign-compare
-  -Wp,-D_GLIBCXX_ASSERTIONS
   $<$<CXX_COMPILER_ID:Clang>:-Wno-gnu-zero-variadic-macro-arguments>
   # Needed for floating point stability in FFT (fft_test will check this).
   # See also https://kristerw.github.io/2021/11/09/fp-contract/
@@ -248,6 +247,14 @@ endif()
 find_package(Threads REQUIRED)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 
+find_package(Kokkos)
+
+if (NOT Kokkos_FOUND)
+  add_compile_options(
+    -Wp,-D_GLIBCXX_ASSERTIONS
+  )
+endif()
+
 ################################################################
 
 # Build flow tools
@@ -265,7 +272,9 @@ add_subdirectory(dbSta)
 add_subdirectory(rsz)
 add_subdirectory(stt)
 add_subdirectory(gpl)
-add_subdirectory(gpl2)
+if (Kokkos_FOUND)
+  add_subdirectory(gpl2)
+endif()
 add_subdirectory(dpl)
 add_subdirectory(dpo)
 add_subdirectory(fin)
@@ -293,6 +302,9 @@ add_subdirectory(par)
 add_executable(openroad
   ${OPENROAD_SOURCE}
 )
+if (Kokkos_FOUND)
+  target_compile_definitions(openroad PUBLIC ENABLE_KOKKOS)
+endif()
 
 target_compile_options(openroad
   PRIVATE
@@ -323,7 +335,6 @@ target_link_libraries(openroad
   pad
   openroad_swig
   gpl
-  gpl2
   dpl
   dpo
   fin
@@ -356,6 +367,10 @@ target_link_libraries(openroad
   ${CMAKE_THREAD_LIBS_INIT}
 )
 
+if (Kokkos_FOUND)
+  target_link_libraries(openroad gpl2 Kokkos::kokkos)
+endif()
+
 target_compile_definitions(openroad PRIVATE BUILD_TYPE="${CMAKE_BUILD_TYPE}")
 
 # tclReadline
@@ -411,7 +426,6 @@ if (Python3_FOUND AND BUILD_PYTHON)
     ant_py
     grt_py
     gpl_py
-    gpl2_py
     dpl_py
     mpl_py
     ppl_py
@@ -428,6 +442,9 @@ if (Python3_FOUND AND BUILD_PYTHON)
     dft_py
     par_py
   )
+  if (Kokkos_FOUND)
+    target_link_libraries(openroad gpl2_py Kokkos::kokkos)
+  endif()
 
 else()
   message(STATUS "Python3 disabled")

src/Design.cc

@@ -257,10 +257,10 @@ gpl::Replace* Design::getReplace()
   return app->getReplace();
 }
 
-gpl2::GpuReplace* Design::getGpuReplace()
+gpl2::DgReplace* Design::getDgReplace()
 {
   auto app = OpenRoad::openRoad();
-  return app->getGpuReplace();
+  return app->getDgReplace();
 }
 
 dpl::Opendp* Design::getOpendp()

src/Main.cc

@@ -71,6 +71,10 @@
 #include "sta/StringUtil.hh"
 #include "utl/Logger.h"
 
+#ifdef ENABLE_KOKKOS
+#include <Kokkos_Core.hpp>
+#endif
+
 using sta::findCmdLineFlag;
 using sta::findCmdLineKey;
 using sta::is_regular_file;
@@ -309,6 +313,9 @@ int main(int argc, char* argv[])
   // Set argc to 1 so Tcl_Main doesn't source any files.
   // Tcl_Main never returns.
   Tcl_Main(1, argv, ord::tclAppInit);
+#ifdef ENABLE_KOKKOS
+  Kokkos::finalize();
+#endif
   return 0;
 }
 
@@ -371,6 +378,9 @@ static int tclAppInit(int& argc,
     }
 #endif
 
+#ifdef ENABLE_KOKKOS
+    Kokkos::initialize();
+#endif
     ord::initOpenRoad(interp);
 
     if (!findCmdLineFlag(argc, argv, "-no_splash")) {
@@ -432,6 +442,9 @@ static int tclAppInit(int& argc,
             int result = sourceTclFile(cmd_file, false, false, interp);
             if (exit_after_cmd_file) {
               int exit_code = (result == TCL_OK) ? EXIT_SUCCESS : EXIT_FAILURE;
+#ifdef ENABLE_KOKKOS
+              Kokkos::finalize();
+#endif
               exit(exit_code);
             }
           } else {

src/OpenRoad-py.i

@@ -56,7 +56,7 @@ const char *
 openroad_git_describe();
 
 const bool
-openroad_gpu_compiled();
+openroad_dg_compiled();
 
 const bool 
 openroad_python_compiled();

src/OpenRoad.cc

@@ -57,7 +57,9 @@
 #include "dst/MakeDistributed.h"
 #include "fin/MakeFinale.h"
 #include "gpl/MakeReplace.h"
-#include "gpl2/MakeGpuReplace.h"
+#ifdef ENABLE_KOKKOS
+#include "gpl2/MakeDgReplace.h"
+#endif
 #include "grt/MakeGlobalRouter.h"
 #include "gui/MakeGui.h"
 #include "ifp//MakeInitFloorplan.hh"
@@ -140,7 +142,9 @@ OpenRoad::~OpenRoad()
   deleteOpenRCX(extractor_);
   deleteTritonRoute(detailed_router_);
   deleteReplace(replace_);
-  deleteGpuReplace(gpu_replace_);
+#ifdef ENABLE_KOKKOS
+  deleteDgReplace(dg_replace_);
+#endif
   deleteFinale(finale_);
   deleteAntennaChecker(antenna_checker_);
   odb::dbDatabase::destroy(db_);
@@ -196,7 +200,9 @@ void OpenRoad::init(Tcl_Interp* tcl_interp)
   extractor_ = makeOpenRCX();
   detailed_router_ = makeTritonRoute();
   replace_ = makeReplace();
-  gpu_replace_ = makeGpuReplace();
+#ifdef ENABLE_KOKKOS
+  dg_replace_ = makeDgReplace();
+#endif
   pdnsim_ = makePDNSim();
   antenna_checker_ = makeAntennaChecker();
   partitionMgr_ = makePartitionMgr();
@@ -222,7 +228,9 @@ void OpenRoad::init(Tcl_Interp* tcl_interp)
   initDbVerilogNetwork(this);
   initIoplacer(this);
   initReplace(this);
-  initGpuReplace(this);
+#ifdef ENABLE_KOKKOS
+  initDgReplace(this);
+#endif
   initOpendp(this);
   initOptdp(this);
   initFinale(this);

src/OpenRoad.i

@@ -154,11 +154,11 @@ getReplace()
   return openroad->getReplace();
 }
 
-gpl2::GpuReplace*
-getGpuReplace()
+gpl2::DgReplace*
+getDgReplace()
 {
   OpenRoad *openroad = getOpenRoad();
-  return openroad->getGpuReplace();
+  return openroad->getDgReplace();
 }
 
 rcx::Ext *
@@ -311,7 +311,7 @@ openroad_git_describe()
 }
 
 const bool 
-openroad_gpu_compiled()
+openroad_dg_compiled()
 {
   return ord::OpenRoad::getGUICompileOption();
 }