1set(LIBOMPTARGET_BUILD_DEVICERTL_BCLIB TRUE CACHE BOOL 2 "Can be set to false to disable building this library.") 3 4if (NOT LIBOMPTARGET_BUILD_DEVICERTL_BCLIB) 5 message(STATUS "Not building DeviceRTL: Disabled by LIBOMPTARGET_BUILD_DEVICERTL_BCLIB") 6 return() 7endif() 8 9# Check to ensure the host system is a supported host architecture. 10if(NOT ${CMAKE_SIZEOF_VOID_P} EQUAL "8") 11 message(STATUS "Not building DeviceRTL: Runtime does not support 32-bit hosts") 12 return() 13endif() 14 15if (LLVM_DIR) 16 # Builds that use pre-installed LLVM have LLVM_DIR set. 17 # A standalone or LLVM_ENABLE_RUNTIMES=openmp build takes this route 18 find_program(CLANG_TOOL clang PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH) 19 find_program(PACKAGER_TOOL clang-offload-packager PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH) 20 find_program(LINK_TOOL llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH) 21 find_program(OPT_TOOL opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH) 22 if ((NOT CLANG_TOOL) OR (NOT LINK_TOOL) OR (NOT OPT_TOOL) OR (NOT PACKAGER_TOOL)) 23 message(STATUS "Not building DeviceRTL. Missing clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL}, opt: ${OPT_TOOL}, or clang-offload-packager: ${PACKAGER_TOOL}") 24 return() 25 else() 26 message(STATUS "Building DeviceRTL. Using clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL} and opt: ${OPT_TOOL}") 27 endif() 28elseif (LLVM_TOOL_CLANG_BUILD AND NOT CMAKE_CROSSCOMPILING AND NOT OPENMP_STANDALONE_BUILD) 29 # LLVM in-tree builds may use CMake target names to discover the tools. 30 # A LLVM_ENABLE_PROJECTS=openmp build takes this route 31 set(CLANG_TOOL $<TARGET_FILE:clang>) 32 set(PACKAGER_TOOL $<TARGET_FILE:clang-offload-packager>) 33 set(LINK_TOOL $<TARGET_FILE:llvm-link>) 34 set(OPT_TOOL $<TARGET_FILE:opt>) 35 message(STATUS "Building DeviceRTL. Using clang from in-tree build") 36else() 37 message(STATUS "Not building DeviceRTL. No appropriate clang found") 38 return() 39endif() 40 41set(devicertl_base_directory ${CMAKE_CURRENT_SOURCE_DIR}) 42set(include_directory ${devicertl_base_directory}/include) 43set(source_directory ${devicertl_base_directory}/src) 44 45set(include_files 46 ${include_directory}/Allocator.h 47 ${include_directory}/Configuration.h 48 ${include_directory}/Debug.h 49 ${include_directory}/Interface.h 50 ${include_directory}/LibC.h 51 ${include_directory}/Mapping.h 52 ${include_directory}/Profiling.h 53 ${include_directory}/State.h 54 ${include_directory}/Synchronization.h 55 ${include_directory}/DeviceTypes.h 56 ${include_directory}/DeviceUtils.h 57 ${include_directory}/Workshare.h 58) 59 60set(src_files 61 ${source_directory}/Allocator.cpp 62 ${source_directory}/Configuration.cpp 63 ${source_directory}/Debug.cpp 64 ${source_directory}/Kernel.cpp 65 ${source_directory}/LibC.cpp 66 ${source_directory}/Mapping.cpp 67 ${source_directory}/Misc.cpp 68 ${source_directory}/Parallelism.cpp 69 ${source_directory}/Profiling.cpp 70 ${source_directory}/Reduction.cpp 71 ${source_directory}/State.cpp 72 ${source_directory}/Synchronization.cpp 73 ${source_directory}/Tasking.cpp 74 ${source_directory}/DeviceUtils.cpp 75 ${source_directory}/Workshare.cpp 76) 77 78# We disable the slp vectorizer during the runtime optimization to avoid 79# vectorized accesses to the shared state. Generally, those are "good" but 80# the optimizer pipeline (esp. Attributor) does not fully support vectorized 81# instructions yet and we end up missing out on way more important constant 82# propagation. That said, we will run the vectorizer again after the runtime 83# has been linked into the user program. 84set(clang_opt_flags -O3 -mllvm -openmp-opt-disable -DSHARED_SCRATCHPAD_SIZE=512 -mllvm -vectorize-slp=false ) 85set(link_opt_flags -O3 -openmp-opt-disable -attributor-enable=module -vectorize-slp=false ) 86set(link_export_flag -passes=internalize -internalize-public-api-file=${source_directory}/exports) 87 88# If the user built with the GPU C library enabled we will use that instead. 89if(${LIBOMPTARGET_GPU_LIBC_SUPPORT}) 90 list(APPEND clang_opt_flags -DOMPTARGET_HAS_LIBC) 91endif() 92 93# Prepend -I to each list element 94set (LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL "${LIBOMPTARGET_LLVM_INCLUDE_DIRS}") 95list(TRANSFORM LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL PREPEND "-I") 96 97# Set flags for LLVM Bitcode compilation. 98set(bc_flags -c -foffload-lto -std=c++17 -fvisibility=hidden 99 ${clang_opt_flags} --offload-device-only 100 -nocudalib -nogpulib -nogpuinc -nostdlibinc 101 -fopenmp -fopenmp-cuda-mode 102 -Wno-unknown-cuda-version -Wno-openmp-target 103 -DOMPTARGET_DEVICE_RUNTIME 104 -I${include_directory} 105 -I${devicertl_base_directory}/../include 106 -I${devicertl_base_directory}/../../libc 107 ${LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL} 108) 109 110# first create an object target 111add_library(omptarget.devicertl.all_objs OBJECT IMPORTED) 112function(compileDeviceRTLLibrary target_name target_triple) 113 set(target_bc_flags ${ARGN}) 114 115 set(bc_files "") 116 foreach(src ${src_files}) 117 get_filename_component(infile ${src} ABSOLUTE) 118 get_filename_component(outfile ${src} NAME) 119 set(outfile "${outfile}-${target_name}.bc") 120 set(depfile "${outfile}.d") 121 122 # Passing an empty CPU to -march= suppressed target specific metadata. 123 add_custom_command(OUTPUT ${outfile} 124 COMMAND ${CLANG_TOOL} 125 ${bc_flags} 126 -fopenmp-targets=${target_triple} 127 -Xopenmp-target=${target_triple} -march= 128 ${target_bc_flags} 129 -MD -MF ${depfile} 130 ${infile} -o ${outfile} 131 DEPENDS ${infile} 132 DEPFILE ${depfile} 133 COMMENT "Building LLVM bitcode ${outfile}" 134 VERBATIM 135 ) 136 if(TARGET clang) 137 # Add a file-level dependency to ensure that clang is up-to-date. 138 # By default, add_custom_command only builds clang if the 139 # executable is missing. 140 add_custom_command(OUTPUT ${outfile} 141 DEPENDS clang 142 APPEND 143 ) 144 endif() 145 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${outfile}) 146 147 list(APPEND bc_files ${outfile}) 148 endforeach() 149 150 set(bclib_name "libomptarget-${target_name}.bc") 151 152 # Link to a bitcode library. 153 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 154 COMMAND ${LINK_TOOL} 155 -o ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} ${bc_files} 156 DEPENDS ${bc_files} 157 COMMENT "Linking LLVM bitcode ${bclib_name}" 158 ) 159 160 if(TARGET llvm-link) 161 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 162 DEPENDS llvm-link 163 APPEND) 164 endif() 165 166 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 167 COMMAND ${OPT_TOOL} ${link_export_flag} ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 168 -o ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 169 DEPENDS ${source_directory}/exports ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 170 COMMENT "Internalizing LLVM bitcode ${bclib_name}" 171 ) 172 if(TARGET opt) 173 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 174 DEPENDS opt 175 APPEND) 176 endif() 177 178 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 179 COMMAND ${OPT_TOOL} ${link_opt_flags} ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 180 -o ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 181 DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 182 COMMENT "Optimizing LLVM bitcode ${bclib_name}" 183 ) 184 if(TARGET opt) 185 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 186 DEPENDS opt 187 APPEND) 188 endif() 189 190 set(bclib_target_name "omptarget-${target_name}-bc") 191 add_custom_target(${bclib_target_name} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}) 192 193 # Copy library to destination. 194 add_custom_command(TARGET ${bclib_target_name} POST_BUILD 195 COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 196 ${LIBOMPTARGET_LIBRARY_DIR}) 197 add_dependencies(omptarget.devicertl.${target_name} ${bclib_target_name}) 198 199 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${bclib_name} ${LIBOMPTARGET_LIBRARY_DIR}/${bclib_name}) 200 201 # Install bitcode library under the lib destination folder. 202 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} DESTINATION "${OFFLOAD_INSTALL_LIBDIR}") 203 204 set(target_feature "") 205 if("${target_triple}" STREQUAL "nvptx64-nvidia-cuda") 206 set(target_feature "feature=+ptx63") 207 endif() 208 209 # Package the bitcode in the bitcode and embed it in an ELF for the static library 210 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name} 211 COMMAND ${PACKAGER_TOOL} -o ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name} 212 "--image=file=${CMAKE_CURRENT_BINARY_DIR}/${bclib_name},${target_feature},triple=${target_triple},arch=generic,kind=openmp" 213 DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 214 COMMENT "Packaging LLVM offloading binary ${bclib_name}.out" 215 ) 216 if(TARGET clang-offload-packager) 217 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name} 218 DEPENDS clang-offload-packager 219 APPEND) 220 endif() 221 222 set(output_name "${CMAKE_CURRENT_BINARY_DIR}/devicertl-${target_name}.o") 223 add_custom_command(OUTPUT ${output_name} 224 COMMAND ${CLANG_TOOL} --std=c++17 -c -nostdlib 225 -Xclang -fembed-offload-object=${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name} 226 -o ${output_name} 227 ${source_directory}/Stub.cpp 228 DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/packaged_${bclib_name} ${source_directory}/Stub.cpp 229 COMMENT "Embedding LLVM offloading binary in devicertl-${target_name}.o" 230 VERBATIM 231 ) 232 if(TARGET clang) 233 add_custom_command(OUTPUT ${output_name} 234 DEPENDS clang 235 APPEND) 236 endif() 237 238 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${output_name}) 239 set_property(TARGET omptarget.devicertl.all_objs APPEND PROPERTY IMPORTED_OBJECTS ${output_name}) 240 241 if (CMAKE_EXPORT_COMPILE_COMMANDS) 242 set(ide_target_name omptarget-ide-${target_name}) 243 add_library(${ide_target_name} STATIC EXCLUDE_FROM_ALL ${src_files}) 244 target_compile_options(${ide_target_name} PRIVATE 245 -fopenmp-targets=${target_triple} -Xopenmp-target=${target_triple} -march= 246 -fopenmp -fopenmp-cuda-mode -mllvm -openmp-opt-disable 247 -foffload-lto -fvisibility=hidden --offload-device-only 248 -nocudalib -nogpulib -nogpuinc -nostdlibinc -Wno-unknown-cuda-version 249 ) 250 target_compile_definitions(${ide_target_name} PRIVATE SHARED_SCRATCHPAD_SIZE=512) 251 target_include_directories(${ide_target_name} PRIVATE 252 ${include_directory} 253 ${devicertl_base_directory}/../../libc 254 ${devicertl_base_directory}/../include 255 ${LIBOMPTARGET_LLVM_INCLUDE_DIRS} 256 ) 257 install(TARGETS ${ide_target_name} EXCLUDE_FROM_ALL) 258 endif() 259endfunction() 260 261add_custom_target(omptarget.devicertl.amdgpu) 262compileDeviceRTLLibrary(amdgpu amdgcn-amd-amdhsa -Xclang -mcode-object-version=none) 263 264add_custom_target(omptarget.devicertl.nvptx) 265compileDeviceRTLLibrary(nvptx nvptx64-nvidia-cuda --cuda-feature=+ptx63) 266 267# Archive all the object files generated above into a static library 268add_library(omptarget.devicertl STATIC) 269set_target_properties(omptarget.devicertl PROPERTIES 270 ARCHIVE_OUTPUT_DIRECTORY "${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}" 271 LINKER_LANGUAGE CXX 272) 273target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs) 274 275install(TARGETS omptarget.devicertl ARCHIVE DESTINATION ${OFFLOAD_INSTALL_LIBDIR}) 276