[OpenCL/GPU] Kernel binary caching @open sesame 04/04 09:22

nntrainer/opencl/opencl_loader.cpp

@@ -73,7 +73,9 @@ void LoadOpenCLFunctions(void *libopencl) {
   LoadFunction(clEnqueueWriteBuffer);
   LoadFunction(clEnqueueReadBuffer);
   LoadFunction(clCreateProgramWithSource);
+  LoadFunction(clCreateProgramWithBinary);
   LoadFunction(clBuildProgram);
+  LoadFunction(clGetProgramInfo);
   LoadFunction(clGetProgramBuildInfo);
   LoadFunction(clRetainProgram);
   LoadFunction(clCreateKernel);
@@ -95,7 +97,9 @@ PFN_clCreateBuffer clCreateBuffer;
 PFN_clEnqueueWriteBuffer clEnqueueWriteBuffer;
 PFN_clEnqueueReadBuffer clEnqueueReadBuffer;
 PFN_clCreateProgramWithSource clCreateProgramWithSource;
+PFN_clCreateProgramWithBinary clCreateProgramWithBinary;
 PFN_clBuildProgram clBuildProgram;
+PFN_clGetProgramInfo clGetProgramInfo;
 PFN_clGetProgramBuildInfo clGetProgramBuildInfo;
 PFN_clRetainProgram clRetainProgram;
 PFN_clCreateKernel clCreateKernel;

nntrainer/opencl/opencl_loader.h

@@ -72,13 +72,24 @@ typedef cl_program(CL_API_CALL *PFN_clCreateProgramWithSource)(
   const char ** /**< strings */, const size_t * /**< lengths */,
   cl_int * /**< errcode_ret */);
 
+typedef cl_program(CL_API_CALL *PFN_clCreateProgramWithBinary)(
+  cl_context /**< context */, cl_uint /**< num_devices */,
+  const cl_device_id * /**< device_list */, const size_t * /**< lengths */,
+  const unsigned char ** /**< binaries */, cl_int * /**< binary_status */,
+  cl_int * /**< errcode_ret */);
+
 typedef cl_int(CL_API_CALL *PFN_clBuildProgram)(
   cl_program /**< program */, cl_uint /**< num_devices */,
   const cl_device_id * /**< device_list */, const char * /**< options */,
   void(CL_CALLBACK * /**< pfn_notify */)(cl_program /**< program */,
                                          void * /**< user_data */),
   void * /**< user_data */);
 
+typedef cl_int(CL_API_CALL *PFN_clGetProgramInfo)(
+  cl_program /**< program */, cl_program_info /**< param_name */,
+  size_t /**< param_value_size */, void * /**< param_value */,
+  size_t * /**< param_value_size_ret */);
+
 typedef cl_int(CL_API_CALL *PFN_clGetProgramBuildInfo)(
   cl_program /**< program */, cl_device_id /**< device */,
   cl_program_build_info /**< param_name */, size_t /**< param_value_size */,
@@ -128,7 +139,9 @@ extern PFN_clCreateBuffer clCreateBuffer;
 extern PFN_clEnqueueWriteBuffer clEnqueueWriteBuffer;
 extern PFN_clEnqueueReadBuffer clEnqueueReadBuffer;
 extern PFN_clCreateProgramWithSource clCreateProgramWithSource;
+extern PFN_clCreateProgramWithBinary clCreateProgramWithBinary;
 extern PFN_clBuildProgram clBuildProgram;
+extern PFN_clGetProgramInfo clGetProgramInfo;
 extern PFN_clGetProgramBuildInfo clGetProgramBuildInfo;
 extern PFN_clRetainProgram clRetainProgram;
 extern PFN_clCreateKernel clCreateKernel;

nntrainer/opencl/opencl_program.cpp

@@ -13,6 +13,8 @@
 
 #include "opencl_program.h"
 
+#include <cstring>
+#include <fstream>
 #include <string>
 
 #include "opencl_loader.h"
@@ -29,8 +31,8 @@ namespace nntrainer::opencl {
  * @return true if successful or false otherwise
  */
 bool Program::BuildProgram(cl_device_id device_id,
-                           const std::string &compiler_options) {
-
+                           const std::string &compiler_options,
+                           bool binaryCreated) {
   // clBuildProgram returns NULL with error code if fails
   const int error_code = clBuildProgram(
     program_, 0, nullptr, compiler_options.c_str(), nullptr, nullptr);
@@ -41,6 +43,105 @@ bool Program::BuildProgram(cl_device_id device_id,
     return false;
   }
 
+  // saving kernel binary
+  if (!binaryCreated)
+    return GetProgramInfo(device_id);
+
+  return true;
+}
+
+/**
+ * @brief Utility to get program info and save kernel binaries
+ *
+ * @param device_id OpenCL device id
+ * @return true if successful or false otherwise
+ */
+bool Program::GetProgramInfo(cl_device_id device_id) {
+  // since only GPU is being used
+  unsigned int num_devices = 1;
+
+  cl_int error_code = CL_SUCCESS;
+  size_t *binaries_size = NULL;
+  unsigned char **binaries_ptr = NULL;
+  char *kernel_names;
+
+  // Read the binary size
+  size_t binaries_size_alloc_size = sizeof(size_t) * num_devices;
+  binaries_size = (size_t *)malloc(binaries_size_alloc_size);
+  error_code = clGetProgramInfo(program_, CL_PROGRAM_BINARY_SIZES,
+                                binaries_size_alloc_size, binaries_size, NULL);
+
+  if (error_code != CL_SUCCESS) {
+    ml_loge("Failed to get program binary size. OpenCL error code: %d. %s",
+            error_code,
+            (GetProgramBuildInfo(device_id, CL_PROGRAM_BUILD_LOG)).c_str());
+    return false;
+  }
+
+  // Read the kernel name
+  size_t *kernel_names_size;
+  error_code = clGetProgramInfo(program_, CL_PROGRAM_KERNEL_NAMES, 0, NULL,
+                                kernel_names_size);
+
+  if (error_code != CL_SUCCESS) {
+    ml_loge("Failed to get program kernel name size. OpenCL error code: %d. %s",
+            error_code,
+            (GetProgramBuildInfo(device_id, CL_PROGRAM_BUILD_LOG)).c_str());
+    return false;
+  }
+
+  // since only one kernel is being saved via this function at a time
+  // kernel_names_size -> array with 1 element
+  kernel_names = (char *)malloc(kernel_names_size[0]);
+  error_code = clGetProgramInfo(program_, CL_PROGRAM_KERNEL_NAMES,
+                                kernel_names_size[0], kernel_names, NULL);
+
+  if (error_code != CL_SUCCESS) {
+    ml_loge("Failed to get program kernel names. OpenCL error code: %d. %s",
+            error_code,
+            (GetProgramBuildInfo(device_id, CL_PROGRAM_BUILD_LOG)).c_str());
+    return false;
+  } else {
+    ml_logi("Saving kernel binary for: %s", std::string(kernel_names).c_str());
+  }
+
+  // Read the binary
+  size_t binaries_ptr_alloc_size = sizeof(unsigned char *) * num_devices;
+  binaries_ptr = (unsigned char **)malloc(binaries_ptr_alloc_size);
+
+  memset(binaries_ptr, 0, binaries_ptr_alloc_size);
+  for (unsigned int i = 0; i < num_devices; ++i) {
+    binaries_ptr[i] = (unsigned char *)malloc(binaries_size[i]);
+  }
+
+  error_code = clGetProgramInfo(program_, CL_PROGRAM_BINARIES,
+                                binaries_ptr_alloc_size, binaries_ptr, NULL);
+
+  if (error_code != CL_SUCCESS) {
+    ml_loge("Failed to get program binary data. OpenCL error code: %d. %s",
+            error_code,
+            (GetProgramBuildInfo(device_id, CL_PROGRAM_BUILD_LOG)).c_str());
+    return false;
+  }
+
+  // Write the binary to file
+  for (unsigned int i = 0; i < num_devices; ++i) {
+    std::ofstream fs(std::string(kernel_names) + "_kernel.bin",
+                     std::ios::out | std::ios::binary | std::ios::app);
+    fs.write((char *)binaries_ptr[i], binaries_size[i]);
+    fs.close();
+  }
+
+  // cleanup
+  if (binaries_ptr) {
+    for (unsigned int i = 0; i < num_devices; ++i) {
+      free(binaries_ptr[i]);
+    }
+    free(binaries_ptr);
+  }
+  free(binaries_size);
+  free(kernel_names);
+
   return true;
 }
 
@@ -103,6 +204,41 @@ bool Program::CreateCLProgram(const cl_context &context,
   return BuildProgram(device_id, compiler_options);
 }
 
+/**
+ * @brief Create OpenCL program from pre compiled binary
+ *
+ * @param context OpenCL context
+ * @param device_id OpenCL device id
+ * @param size binary file size
+ * @param binary data saved as binary
+ * @param binary_name name of binary file for logging
+ * @param compiler_options string compiler options
+ * @return true if successful or false otherwise
+ */
+bool Program::CreateCLProgramWithBinary(const cl_context &context,
+                                        const cl_device_id &device_id,
+                                        size_t size, unsigned char *binary,
+                                        std::string binary_name,
+                                        const std::string &compiler_options) {
+
+  int error_code;
+  const cl_device_id device_list[] = {device_id};
+  const size_t lengths[] = {size};
+  const unsigned char *binaries[] = {binary};
+
+  program_ = clCreateProgramWithBinary(context, 1, device_list, lengths,
+                                       binaries, NULL, &error_code);
+  if (!program_ || error_code != CL_SUCCESS) {
+    ml_loge("Failed to create compute program. OpenCL error code: %d",
+            error_code);
+    return false;
+  }
+
+  ml_logi("Loaded program from binary for: %s", binary_name.c_str());
+
+  return BuildProgram(device_id, compiler_options, true);
+}
+
 /**
  * @brief Get the Program object
  *

nntrainer/opencl/opencl_program.h

@@ -35,8 +35,16 @@ class Program {
    * @param compiler_options string compiler options
    * @return true if successful or false otherwise
    */
-  bool BuildProgram(cl_device_id device_id,
-                    const std::string &compiler_options);
+  bool BuildProgram(cl_device_id device_id, const std::string &compiler_options,
+                    bool binaryCreated = false);
+
+  /**
+   * @brief Utility to get program info and save kernel binaries
+   *
+   * @param device_id OpenCL device id
+   * @return true if successful or false otherwise
+   */
+  bool GetProgramInfo(cl_device_id device_id);
 
   /**
    * @brief Get the information on the program build
@@ -62,6 +70,22 @@ class Program {
                        const std::string &code,
                        const std::string &compiler_options);
 
+  /**
+   * @brief Create OpenCL program from pre compiled binary
+   *
+   * @param context OpenCL context
+   * @param device_id OpenCL device id
+   * @param size binary file size
+   * @param binary data saved as binary
+   * @param binary_name name of binary file for logging
+   * @param compiler_options string compiler options
+   * @return true if successful or false otherwise
+   */
+  bool CreateCLProgramWithBinary(const cl_context &context,
+                                 const cl_device_id &device_id, size_t size,
+                                 unsigned char *binary, std::string binary_name,
+                                 const std::string &compiler_options);
+
   /**
    * @brief Get the Program object
    *