1 #include "sycldevice.hpp" 2 #include <limits> // for std::numeric_limits 3 #include <csetjmp> // for MPI sycl device awareness 4 #include <csignal> // SIGSEGV 5 #include <vector> 6 #include <CL/sycl.hpp> 7 8 namespace Petsc 9 { 10 11 namespace device 12 { 13 14 namespace sycl 15 { 16 17 // definition for static 18 std::array<Device::DeviceInternal *, PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {}; 19 Device::DeviceInternal **Device::devices_ = &Device::devices_array_[1]; 20 int Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE; 21 bool Device::initialized_ = false; 22 23 static std::jmp_buf MPISyclAwareJumpBuffer; 24 static bool MPISyclAwareJumpBufferSet; 25 26 // internal "impls" class for SyclDevice. Each instance represents a single sycl device 27 class PETSC_NODISCARD Device::DeviceInternal { 28 const int id_; // -1 for the host device; 0 and up for gpu devices 29 bool devInitialized_; 30 const ::sycl::device syclDevice_; 31 32 public: 33 // default constructor 34 DeviceInternal(int id) noexcept : id_(id), devInitialized_(false), syclDevice_(chooseSYCLDevice_(id)) { } 35 int id() const { return id_; } 36 bool initialized() const { return devInitialized_; } 37 38 PetscErrorCode initialize() noexcept 39 { 40 PetscFunctionBegin; 41 if (initialized()) PetscFunctionReturn(PETSC_SUCCESS); 42 if (syclDevice_.is_gpu() && use_gpu_aware_mpi) { 43 if (!isMPISyclAware_()) { 44 PetscCall((*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n")); 45 PetscCall((*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n")); 46 PETSCABORT(PETSC_COMM_SELF, PETSC_ERR_LIB); 47 } 48 } 49 devInitialized_ = true; 50 PetscFunctionReturn(PETSC_SUCCESS); 51 } 52 53 PetscErrorCode view(PetscViewer viewer) const noexcept 54 { 55 MPI_Comm comm; 56 PetscMPIInt rank; 57 PetscBool iascii; 58 59 PetscFunctionBegin; 60 PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d being viewed before it was initialized or configured", id()); 61 PetscCall(PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer), PETSCVIEWERASCII, &iascii)); 62 PetscCall(PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer), &comm)); 63 if (iascii) { 64 PetscViewer sviewer; 65 66 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 67 PetscCall(PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sviewer)); 68 PetscCall(PetscViewerASCIIPrintf(sviewer, "[%d] device: %s\n", rank, syclDevice_.get_info<::sycl::info::device::name>().c_str())); 69 PetscCall(PetscViewerASCIIPushTab(sviewer)); 70 PetscCall(PetscViewerASCIIPrintf(sviewer, "-> Device vendor: %s\n", syclDevice_.get_info<::sycl::info::device::vendor>().c_str())); 71 PetscCall(PetscViewerASCIIPopTab(sviewer)); 72 PetscCall(PetscViewerFlush(sviewer)); 73 PetscCall(PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sviewer)); 74 PetscCall(PetscViewerFlush(viewer)); 75 } 76 PetscFunctionReturn(PETSC_SUCCESS); 77 } 78 79 PetscErrorCode getattribute(PetscDeviceAttribute attr, void *value) const noexcept 80 { 81 PetscFunctionBegin; 82 PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d not initialized", id()); 83 switch (attr) { 84 case PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK: 85 *static_cast<std::size_t *>(value) = syclDevice_.get_info<::sycl::info::device::local_mem_size>(); 86 case PETSC_DEVICE_ATTR_MAX: 87 break; 88 } 89 PetscFunctionReturn(PETSC_SUCCESS); 90 } 91 92 private: 93 static ::sycl::device chooseSYCLDevice_(int id) 94 { 95 if (id == PETSC_SYCL_DEVICE_HOST) { 96 return ::sycl::device(::sycl::host_selector()); 97 } else { 98 return ::sycl::device::get_devices(::sycl::info::device_type::gpu)[id]; 99 } 100 } 101 102 // Is the underlying MPI aware of sycl (GPU) devices? 103 bool isMPISyclAware_() noexcept 104 { 105 const int bufSize = 2; 106 const int hbuf[bufSize] = {1, 0}; 107 int *dbuf = nullptr; 108 bool awareness = false; 109 const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode { 110 if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer, 1); 111 return PetscSignalHandlerDefault(signal, ptr); 112 }; 113 114 PetscFunctionBegin; 115 auto Q = ::sycl::queue(syclDevice_); 116 dbuf = ::sycl::malloc_device<int>(bufSize, Q); 117 Q.memcpy(dbuf, hbuf, sizeof(int) * bufSize).wait(); 118 PetscCallAbort(PETSC_COMM_SELF, PetscPushSignalHandler(SyclSignalHandler, nullptr)); 119 MPISyclAwareJumpBufferSet = true; 120 if (setjmp(MPISyclAwareJumpBuffer)) { 121 // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware 122 awareness = false; 123 PetscStackPop; 124 } else if (!MPI_Allreduce(dbuf, dbuf + 1, 1, MPI_INT, MPI_SUM, PETSC_COMM_SELF)) awareness = true; 125 MPISyclAwareJumpBufferSet = false; 126 PetscCallAbort(PETSC_COMM_SELF, PetscPopSignalHandler()); 127 ::sycl::free(dbuf, Q); 128 PetscFunctionReturn(awareness); 129 } 130 }; 131 132 PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscBool *defaultView, PetscDeviceInitType *defaultInitType) noexcept 133 { 134 auto id = *defaultDeviceId; 135 auto initType = *defaultInitType; 136 auto view = *defaultView, flg = PETSC_FALSE; 137 PetscInt ngpus; 138 139 PetscFunctionBegin; 140 if (initialized_) PetscFunctionReturn(PETSC_SUCCESS); 141 initialized_ = true; 142 PetscCall(PetscRegisterFinalize(finalize_)); 143 PetscOptionsBegin(comm, nullptr, "PetscDevice sycl Options", "Sys"); 144 PetscCall(base_type::PetscOptionDeviceInitialize(PetscOptionsObject, &initType, nullptr)); 145 PetscCall(base_type::PetscOptionDeviceSelect(PetscOptionsObject, "Which sycl device to use? Pass -2 for host, PETSC_DECIDE (" PetscStringize(PETSC_DECIDE) ") to let PETSc decide, 0 and up for GPUs", "PetscDeviceCreate()", id, &id, nullptr, -2, std::numeric_limits<decltype(ngpus)>::max())); 146 static_assert(PETSC_DECIDE == -1, "Expect PETSC_DECIDE to be -1"); 147 PetscCall(base_type::PetscOptionDeviceView(PetscOptionsObject, &view, &flg)); 148 PetscOptionsEnd(); 149 150 // post-process the options and lay the groundwork for initialization if needs be 151 std::vector<::sycl::device> gpu_devices = ::sycl::device::get_devices(::sycl::info::device_type::gpu); 152 ngpus = static_cast<PetscInt>(gpu_devices.size()); 153 PetscCheck(ngpus || id < 0, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there is no GPU", (int)id); 154 PetscCheck(ngpus <= 0 || id < ngpus, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there are only %d GPU", (int)id, (int)ngpus); 155 156 if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */ 157 else { 158 PetscCall(PetscDeviceCheckDeviceCount_Internal(ngpus)); 159 if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */ 160 if (ngpus) { 161 PetscMPIInt rank; 162 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 163 id = rank % ngpus; 164 } else id = PETSC_SYCL_DEVICE_HOST; 165 } 166 if (view) initType = PETSC_DEVICE_INIT_EAGER; 167 } 168 169 if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form 170 171 defaultDevice_ = static_cast<decltype(defaultDevice_)>(id); 172 PetscCheck(initType != PETSC_DEVICE_INIT_EAGER || id != PETSC_SYCL_DEVICE_NONE, comm, PETSC_ERR_USER_INPUT, "Cannot eagerly initialize sycl devices as you disabled them by -device_enable_sycl none"); 173 // record the results of the initialization 174 *defaultDeviceId = id; 175 *defaultView = view; 176 *defaultInitType = initType; 177 PetscFunctionReturn(PETSC_SUCCESS); 178 } 179 180 PetscErrorCode Device::finalize_() noexcept 181 { 182 PetscFunctionBegin; 183 if (!initialized_) PetscFunctionReturn(PETSC_SUCCESS); 184 for (auto &&devPtr : devices_array_) delete devPtr; 185 defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default 186 initialized_ = false; 187 PetscFunctionReturn(PETSC_SUCCESS); 188 } 189 190 PetscErrorCode Device::init_device_id_(PetscInt *inid) const noexcept 191 { 192 const auto id = *inid == PETSC_DECIDE ? defaultDevice_ : *inid; 193 194 PetscFunctionBegin; 195 PetscCheck(defaultDevice_ != PETSC_SYCL_DEVICE_NONE, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Trying to retrieve a SYCL PetscDevice when it has been disabled"); 196 PetscCheck(!(id < PETSC_SYCL_DEVICE_HOST) && !(id - PETSC_SYCL_DEVICE_HOST >= PETSC_DEVICE_MAX_DEVICES), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Only supports %zu number of devices but trying to get device with id %" PetscInt_FMT, 197 devices_array_.size(), id); 198 if (!devices_[id]) devices_[id] = new DeviceInternal(id); 199 PetscCheck(id == devices_[id]->id(), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Entry %" PetscInt_FMT " contains device with mismatching id %" PetscInt_FMT, id, devices_[id]->id()); 200 PetscCall(devices_[id]->initialize()); 201 *inid = id; 202 PetscFunctionReturn(PETSC_SUCCESS); 203 } 204 205 PetscErrorCode Device::view_device_(PetscDevice device, PetscViewer viewer) noexcept 206 { 207 PetscFunctionBegin; 208 PetscCall(devices_[device->deviceId]->view(viewer)); 209 PetscFunctionReturn(PETSC_SUCCESS); 210 } 211 212 PetscErrorCode Device::get_attribute_(PetscInt id, PetscDeviceAttribute attr, void *value) noexcept 213 { 214 PetscFunctionBegin; 215 PetscCall(devices_[id]->getattribute(attr, value)); 216 PetscFunctionReturn(PETSC_SUCCESS); 217 } 218 219 } // namespace sycl 220 221 } // namespace device 222 223 } // namespace Petsc 224