Introduce the DPC++ and LevelZero device driver

CMakeLists.txt

@@ -5,6 +5,9 @@ include(CMakeDependentOption)
 include(CMakePushCheckState)
 include(GNUInstallDirs)
 
+# Help debug dependencies with custom targets that PTG uses a lot
+SET(GRAPHVIZ_CUSTOM_TARGETS TRUE)
+
 # The current version number
 #   This uses the numbering scheme from libtool -version_number c:r:a
 #      see http://www.sourceware.org/autobook/autobook/autobook_61.html
@@ -173,6 +176,12 @@ option(PARSEC_GPU_HIP_ALLOC_PER_TILE
   "Tile based allocation engine for GPU memory (instead of internal management
   of a complete allocation)" OFF)
 mark_as_advanced(PARSEC_GPU_HIP_ALLOC_PER_TILE)
+option(PARSEC_GPU_WITH_LEVEL_ZERO
+        "Enable GPU support using LEVEL_ZERO kernels" ON)
+option(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE
+        "Tile based allocation engine for GPU memory (instead of internal management
+  of a complete allocation)" OFF)
+mark_as_advanced(PARSEC_GPU_LEVEL_ZERO_ALLOC_PER_TILE)
 option(PARSEC_GPU_WITH_OPENCL
   "Enable GPU support using OpenCL kernels" OFF)
 mark_as_advanced(PARSEC_GPU_WITH_OPENCL) # Hide this as it is not supported yet
@@ -737,6 +746,19 @@ int main(int argc, char *argv[]) {
     endif()
     set(PARSEC_HAVE_HIP ${HIP_NOT_CUDA_FOUND} CACHE BOOL "True if PaRSEC provide support for HIP")
   endif( PARSEC_GPU_WITH_HIP )
+  if( PARSEC_GPU_WITH_LEVEL_ZERO )
+    find_package(level-zero)
+    find_package(DPCPP)
+    set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero")
+    if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP)
+      include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/")
+      set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero")
+      message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}")
+      message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}")
+    else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP)
+      message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}")
+    endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP)
+  endif( PARSEC_GPU_WITH_LEVEL_ZERO )
 
   find_package(AYUDAME QUIET)
   set(PARSEC_HAVE_AYUDAME ${AYUDAME_FOUND})
@@ -915,6 +937,13 @@ include(AddDocumentedFiles)
 
 add_subdirectory(parsec)
 
+#
+# Add dependency to Level-Zero if it is enabled
+#
+if(PARSEC_HAVE_LEVEL_ZERO)
+    target_link_libraries(parsec PRIVATE level_zero::ze_loader)
+endif(PARSEC_HAVE_LEVEL_ZERO)
+
 #
 # Now continue with compiling the tests.
 #
@@ -1237,6 +1266,8 @@ install(FILES
   ${PROJECT_SOURCE_DIR}/cmake_modules/FindPAPI.cmake
   ${PROJECT_SOURCE_DIR}/cmake_modules/FindOTF2.cmake
   ${PROJECT_SOURCE_DIR}/cmake_modules/FindHWLOC.cmake
+  ${PROJECT_SOURCE_DIR}/cmake_modules/Findlevel-zero.cmake
+  ${PROJECT_SOURCE_DIR}/cmake_modules/FindDPCPP.cmake
   ${PROJECT_SOURCE_DIR}/cmake_modules/ParsecCompilePTG.cmake
   DESTINATION ${PARSEC_INSTALL_CMAKEDIR})
 

cmake_modules/FindDPCPP.cmake

@@ -0,0 +1,44 @@
+if(DPCPP_EXECUTABLE)
+    get_filename_component( _dpcpp_path ${DPCPP_EXECUTABLE} PATH )
+else(DPCPP_EXECUTABLE)
+    set(_dpcpp_path "")
+endif(DPCPP_EXECUTABLE)
+find_program( DPCPP_EXECUTABLE NAMES dpcpp HINTS ${_dpcpp_path} ENV PATH )
+mark_as_advanced( DPCPP_EXECUTABLE )
+
+if( DPCPP_EXECUTABLE )
+    execute_process(COMMAND ${DPCPP_EXECUTABLE} --version
+            RESULT_VARIABLE _res
+            OUTPUT_VARIABLE _out
+            ERROR_VARIABLE _err)
+    if( _res EQUAL 0 )
+        string(REGEX MATCH "([^\n]+)" _ ${_out})
+        message(STATUS "Found dpcpp: ${DPCPP_EXECUTABLE} version ${CMAKE_MATCH_1}")
+
+        get_filename_component( _dpcpp_dir ${DPCPP_EXECUTABLE} DIRECTORY )
+        get_filename_component( _dpcpp_lib_dir "${_dpcpp_dir}/../lib" ABSOLUTE )
+
+        find_library(SYCL_LIBRARY sycl HINTS "${_dpcpp_lib_dir}" "${SYCL_LIBRARY_DIR}" ENV "SYCL_LIBRARY_DIR")
+        if( SYCL_LIBRARY )
+            message(STATUS "SYCL library: ${SYCL_LIBRARY}")
+
+            get_filename_component( _dpcpp_inc_dir "${_dpcpp_dir}/../include" ABSOLUTE )
+            find_file(_sycl_include_file "sycl.hpp" HINTS "${_dpcpp_inc_dir}" "${_dpcpp_inc_dir}/sycl/CL" "${SYCL_INCLUDE_DIR}" ENV "SYCL_INCLUDE_DIR")
+            if(_sycl_include_file)
+                get_filename_component(SYCL_INCLUDE_DIR "${_sycl_include_file}" DIRECTORY)
+                message(STATUS "SYCL include dir: ${SYCL_INCLUDE_DIR}")
+                set(PARSEC_HAVE_DPCPP "TRUE" CACHE BOOL "True if PaRSEC provide support for dpc++")
+            else(_sycl_include_file)
+                message(WARNING "Found DPC++, and SYCL library, but could not find SYCL include directory. Define SYCL_INCLUDE_DIR to enable DPC++ support")
+                set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++")
+            endif(_sycl_include_file)
+        else( SYCL_LIBRARY )
+            message(WARNING "Found DPC++, but did not find SYCL library. Define SYCL_LIBRARY_DIR to enable DPC++ support")
+            set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++")
+        endif( SYCL_LIBRARY )
+    else( _res EQUAL 0 )
+        set(PARSEC_HAVE_DPCPP "FALSE" CACHE BOOL "True if PaRSEC provide support for dpc++")
+        message(WARNING "${DPCPP_EXECUTABLE} does not work: 'dpcpp -v' returned ${_res}, with error ${_err}")
+        message(WARNING "dpc++ support is disabled, set DPCPP_EXECUTABLE to the path of dpcpp to enable DPC++ support, and/or fix your environment to run dpcpp")
+    endif( _res EQUAL 0 )
+endif( DPCPP_EXECUTABLE )

cmake_modules/Findlevel-zero.cmake

@@ -0,0 +1,46 @@
+if(LEVEL_ZERO_ROOT_DIR)
+    message(STATUS "Trying to locate level-zero library and headers under ${LEVEL_ZERO_ROOT_DIR}")
+    find_library(ZE_LOADER_LIBRARY "ze_loader" HINTS "${LEVEL_ZERO_ROOT_DIR}/lib" "${LEVEL_ZERO_ROOT_DIR}/lib64" NO_DEFAULT_PATH)
+    find_path (LEVEL_ZERO_INCLUDE_DIR NAMES "level_zero/ze_api.h" PATHS "${LEVEL_ZERO_ROOT_DIR}/include" NO_DEFAULT_PATH)
+
+    if(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR)
+	    get_filename_component(ZE_LOADER_LIBRARY_DIR ${ZE_LOADER_LIBRARY} DIRECTORY)
+	    check_library_exists("ze_loader" "zeInit" ${ZE_LOADER_LIBRARY_DIR} ZE_LOADER_HAVE_ZEINIT)
+
+        if(ZE_LOADER_HAVE_ZEINIT)
+            message(STATUS "Found ze_loader library in ${ZE_LOADER_LIBRARY} and level_zero/ze_api.h in ${LEVEL_ZERO_INCLUDE_DIR}")
+            add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL)
+            set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${ZE_LOADER_LIBRARY}")
+            set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h")
+            include_directories("${LEVEL_ZERO_INCLUDE_DIR}/") 
+            set(LEVEL_ZERO_FOUND TRUE)
+        else(ZE_LOADER_HAVE_ZEINIT)
+            if(NOT ZE_LOADER_HAVE_ZEINIT)
+                message(WARNING "Found ze_loader library under ${ZE_LOADER_LIBRARY}, but could not find symbol zeInit in this library -- falling back to package config search")
+            endif(NOT ZE_LOADER_HAVE_ZEINIT)
+        endif(ZE_LOADER_HAVE_ZEINIT)
+    else(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR)
+	    if(NOT ZE_LOADER_LIBRARY)
+		    message(WARNING "Could not find ze_loader library under provided LEVEL_ZERO_ROOT_DIR='${LEVEL_ZERO_ROOT_DIR}' (tried subdirectories lib/ and lib64/) -- falling back to package config search")
+	    endif(NOT ZE_LOADER_LIBRARY)
+	    if(NOT LEVEL_ZERO_INCLUDE_DIR)
+		    message(WARNING "Cound not find level_zero/ze_api.h under provided LEVEL_ZERO_ROOT_DIR=${LEVEL_ZERO_ROOT_DIR}' (tried subdirectory include) -- falling back to package config search")
+	    endif(NOT LEVEL_ZERO_INCLUDE_DIR)
+    endif(ZE_LOADER_LIBRARY AND LEVEL_ZERO_INCLUDE_DIR)
+endif(LEVEL_ZERO_ROOT_DIR)
+
+if(NOT LEVEL_ZERO_FOUND)
+    find_package(PkgConfig QUIET)
+
+    if(PKG_CONFIG_FOUND)
+        pkg_check_modules(LEVEL_ZERO level-zero)
+        if(LEVEL_ZERO_FOUND)
+            pkg_get_variable(LEVEL_ZERO_LIBRARY_DIR level-zero libdir)
+            pkg_get_variable(LEVEL_ZERO_INCLUDE_DIR level-zero includedir)
+            add_library(level_zero::ze_loader UNKNOWN IMPORTED GLOBAL)
+            set_property(TARGET level_zero::ze_loader PROPERTY IMPORTED_LOCATION "${LEVEL_ZERO_LIBRARY_DIR}/libze_loader.so")
+            set_property(TARGET level_zero::ze_loader PROPERTY INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}/level_zero/ze_api.h")
+            include_directories("${LEVEL_ZERO_INCLUDE_DIR}/")
+        endif(LEVEL_ZERO_FOUND)
+    endif(PKG_CONFIG_FOUND)
+endif(NOT LEVEL_ZERO_FOUND)

cmake_modules/PaRSECConfig.cmake.in

@@ -73,6 +73,23 @@ if(@PARSEC_HAVE_HIP@)
   SET(PARSEC_HAVE_HIP TRUE)
 endif(@PARSEC_HAVE_HIP@)
 
+if(@PARSEC_HAVE_LEVEL_ZERO@)
+  if(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "")
+    set(LEVEL_ZERO_ROOT_DIR "@LEVEL_ZERO_ROOT_DIR@")
+  endif(NOT "@LEVEL_ZERO_ROOT_DIR@" EQUAL "")
+  find_package(level-zero)
+  find_package(DPCPP)
+  set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel level-zero")
+  if (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP)
+    include_directories("${LEVEL_ZERO_INCLUDE_DIR}/level_zero/")
+    set(PARSEC_HAVE_LEVEL_ZERO ${LEVEL_ZERO_FOUND} CACHE BOOL "True if PaRSEC provide support for Intel Level Zero")
+    message(STATUS "Found Intel level-zero ${LEVEL_ZERO_VERSION} in -I${LEVEL_ZERO_INCLUDE_DIR} / -L${LEVEL_ZERO_LIBRARY_DIR}")
+    message(STATUS "Found dpcpp in ${DPCPP_EXECUTABLE}")
+  else(LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP)
+    message(STATUS "Could not find both level-zero and DPCPP: level-zero says ${LEVEL_ZERO_FOUND}, DPCPP says ${DPCPP_EXECUTABLE}")
+  endif (LEVEL_ZERO_FOUND AND PARSEC_HAVE_DPCPP)
+endif(@PARSEC_HAVE_LEVEL_ZERO@)
+
 if(@PARSEC_PROF_TRACE@)
   # Nothing exportable here, if this test succeed then PaRSEC supports tracing
 endif(@PARSEC_PROF_TRACE@)

cmake_modules/ParsecCompilePTG.cmake

@@ -7,7 +7,7 @@
 #
 function(target_ptg_source_ex)
   set(options DEBUG LINE FORCE_PROFILE)
-  set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H FUNCTION_NAME DEP_MANAGEMENT)
+  set(oneValueArgs TARGET MODE SOURCE DESTINATION DESTINATION_C DESTINATION_H DESTINATION_DPCPP FUNCTION_NAME DEP_MANAGEMENT)
   set(multipleValueArgs WARNINGS IGNORE_PROPERTIES PTGPP_FLAGS)
   cmake_parse_arguments(PARSEC_PTGPP "${options}" "${oneValueArgs}"
           "${multiValueArgs}" ${ARGN} )
@@ -51,6 +51,9 @@ function(target_ptg_source_ex)
   else()
     set(outname_h "${outname}.h")
   endif()
+  if(DEFINED PARSEC_PTGPP_DESTINATION_DPCPP)
+      set(outname_dpcpp "${PARSEC_PTGPP_DESTINATION_DPCPP}")
+  endif()
 
   if(DEFINED PARSEC_PTGPP_FUNCTION_NAME)
     set(fnname "${PARSEC_PTGPP_FUNCTION_NAME}")
@@ -91,12 +94,21 @@ function(target_ptg_source_ex)
   # its cmake source_file name, yet we depend on the source_file name as it is how cmake tracks it
   get_property(location SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY LOCATION)
 
-  add_custom_command(
+  if(DEFINED outname_dpcpp)
+    add_custom_command(
+          OUTPUT ${outname_h} ${outname_c} ${outname_dpcpp}
+          COMMAND $<TARGET_FILE:PaRSEC::parsec-ptgpp> ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -S ${outname_dpcpp} -f ${fnname}
+          MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE}
+          DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp)
+    add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c} ${outname_dpcpp})
+  else(DEFINED outname_dpcpp)
+    add_custom_command(
           OUTPUT ${outname_h} ${outname_c}
           COMMAND $<TARGET_FILE:PaRSEC::parsec-ptgpp> ${_ptgpp_flags} -E -i ${location} -C ${outname_c} -H ${outname_h} -f ${fnname}
           MAIN_DEPENDENCY ${PARSEC_PTGPP_SOURCE}
           DEPENDS ${PARSEC_PTGPP_SOURCE} PaRSEC::parsec-ptgpp)
-  add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c})
+    add_custom_target(ptgpp_${target}.${outname} DEPENDS ${outname_h} ${outname_c})
+  endif(DEFINED outname_dpcpp)
 
   # Copy the properties to the generated files
   get_property(cflags     SOURCE ${PARSEC_PTGPP_SOURCE} PROPERTY COMPILE_OPTIONS)
@@ -110,11 +122,24 @@ function(target_ptg_source_ex)
           COMPILE_OPTIONS "${cflags}"
           INCLUDE_DIRECTORIES "${includes}"
           COMPILE_DEFINITIONS "${defs}")
+  if(DEFINED outname_dpcpp)
+    set_source_files_properties("${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}"
+            TARGET_DIRECTORY ${target}
+            PROPERTIES
+            GENERATED 1
+            COMPILE_OPTIONS "${cflags}"
+            INCLUDE_DIRECTORIES "${includes}"
+            COMPILE_DEFINITIONS "${defs}")
+  endif(DEFINED outname_dpcpp)
 
   # make sure we produce .h before we build other .c in the target
   add_dependencies(${target} ptgpp_${target}.${outname})
   # add to the target
-  target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}")
+  if(DEFINED outname_dpcpp)
+    target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c};${CMAKE_CURRENT_BINARY_DIR}/${outname_dpcpp}")
+  else(DEFINED outname_dpcpp)
+    target_sources(${target} ${PARSEC_PTGPP_MODE} "${CMAKE_CURRENT_BINARY_DIR}/${outname_h};${CMAKE_CURRENT_BINARY_DIR}/${outname_c}")
+  endif(DEFINED outname_dpcpp)
 
   get_target_property(_includes ${target} INCLUDE_DIRECTORIES)
   list(FIND _includes "${CMAKE_CURRENT_BINARY_DIR}" _i1)

parsec/data.c

@@ -94,8 +94,7 @@ static void parsec_data_destruct(parsec_data_t* obj )
 #endif  /* defined(PARSEC_DEBUG_PARANOID) */
             assert(obj->super.obj_reference_count > 1);
             parsec_data_copy_detach( obj, copy, i );
-            if ( !(device->type & PARSEC_DEV_CUDA)
-              && !(device->type & PARSEC_DEV_HIP) ) {
+            if ( !(device->type & PARSEC_DEV_GPU) ){
                 /**
                  * GPU copies are normally stored in LRU lists, and must be
                  * destroyed by the release list to free the memory on the device

parsec/include/parsec/parsec_options.h.in

@@ -123,6 +123,8 @@
 #cmakedefine PARSEC_HAVE_HWLOC
 #cmakedefine PARSEC_HAVE_PAPI
 #cmakedefine PARSEC_HAVE_CUDA
+#cmakedefine PARSEC_HAVE_LEVEL_ZERO
+#cmakedefine PARSEC_HAVE_DPCPP
 #cmakedefine PARSEC_HAVE_HIP
 #cmakedefine PARSEC_HAVE_OPENCL
 #cmakedefine PARSEC_HAVE_MPI

parsec/interfaces/dtd/insert_function.c

@@ -41,6 +41,9 @@
 #if defined(PARSEC_HAVE_CUDA)
 #include "parsec/mca/device/cuda/device_cuda.h"
 #endif  /* defined(PARSEC_HAVE_CUDA) */
+#if defined(PARSEC_HAVE_LEVEL_ZERO)
+#include "parsec/mca/device/level_zero/device_level_zero.h"
+#endif /* defined(PARSEC_HAVE_LEVEL_ZERO) */
 
 #include "parsec/mca/mca_repository.h"
 #include "parsec/constants.h"
@@ -1494,6 +1497,9 @@ parsec_dtd_startup(parsec_context_t *context,
         // If CUDA is enabled, let the CUDA device activated for this
         // taskpool.
         if( PARSEC_DEV_CUDA == device->type ) continue;
+        // If LEVEL_ZERO is enabled, let the LEVEL_ZERO device activated for this
+        // taskpool.
+        if( PARSEC_DEV_LEVEL_ZERO == device->type ) continue;
         if( NULL != device->taskpool_register )
             if( PARSEC_SUCCESS !=
                 device->taskpool_register(device, (parsec_taskpool_t *)tp)) {
@@ -2334,6 +2340,12 @@ static parsec_hook_return_t parsec_dtd_gpu_task_submit(parsec_execution_stream_t
         gpu_task->stage_in  = parsec_default_cuda_stage_in;
         gpu_task->stage_out = parsec_default_cuda_stage_out;
         return parsec_cuda_kernel_scheduler(es, gpu_task, dev_index);
+#endif
+#if defined(PARSEC_HAVE_LEVEL_ZERO)
+    case PARSEC_DEV_LEVEL_ZERO:
+        gpu_task->stage_in = parsec_default_level_zero_stage_in;
+        gpu_task->stage_out = parsec_default_level_zero_stage_out;
+        return parsec_level_zero_kernel_scheduler(es, gpu_task, dev_index);
 #endif
     default:
         parsec_fatal("DTD scheduling on device type %d: this is not a valid GPU device type in this build", device->type);
@@ -2404,8 +2416,10 @@ int parsec_dtd_task_class_add_chore(parsec_taskpool_t *tp,
     if(PARSEC_DEV_CUDA == device_type) {
         incarnations[i].hook = parsec_dtd_gpu_task_submit;
         dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function;
-    }
-    else {
+    } else if(PARSEC_DEV_LEVEL_ZERO == device_type) {
+        incarnations[i].hook = parsec_dtd_gpu_task_submit;
+        dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)function;
+    } else {
         dtd_tc->cpu_func_ptr = function;
         incarnations[i].hook = parsec_dtd_cpu_task_submit;
     }
@@ -3289,8 +3303,11 @@ __parsec_dtd_taskpool_create_task(parsec_taskpool_t *tp,
                 /* Special case for CUDA: we need an intermediate */
                 (*incarnations)[0].hook = parsec_dtd_gpu_task_submit;
                 dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer;
-            }
-            else {
+            } if( device_type == PARSEC_DEV_LEVEL_ZERO ) {
+                /* Special case for LEVEL ZERO: we need an intermediate */
+                (*incarnations)[0].hook = parsec_dtd_gpu_task_submit;
+                dtd_tc->gpu_func_ptr = (parsec_advance_task_function_t)fpointer;
+            } else {
                 /* Default case: the user-provided function is directly the hook to call */
                 (*incarnations)[0].hook = fpointer; // We can directly call the CPU hook
                 dtd_tc->cpu_func_ptr = fpointer;

parsec/interfaces/ptg/ptg-compiler/jdf.h

@@ -99,6 +99,7 @@ typedef struct jdf_compiler_global_args {
     char *input;
     char *output_c;
     char *output_h;
+    char *output_dpcpp;
     char *output_o;
     char *funcid;
     jdf_warning_mask_t wmask;
@@ -143,10 +144,17 @@ extern int   jdfdebug;
  *  We remember the line number in the JDF file where this external code was found
  */
 typedef struct jdf_external_entry {
-    struct jdf_object_t       super;
+    struct jdf_object_t        super;
+    struct jdf_external_entry *next;
+    char                      *language;
     char                      *external_code;
 } jdf_external_entry_t;
 
+typedef struct jdf_code_string {
+    char  *language;
+    char  *string;
+} jdf_code_string_t;
+
 /** A global is a variable name, optionally an expression to define it,
  *  and a line number associated with it for error printing purposes
  */