1 #include "../../interface/sycldevice.hpp" 2 #include <csetjmp> // for MPI sycl device awareness 3 #include <csignal> // SIGSEGV 4 #include <vector> 5 #include <CL/sycl.hpp> 6 7 #if PetscDefined(USE_LOG) 8 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 9 #else 10 #define PetscLogInitialize() 0 11 #endif 12 13 namespace Petsc { 14 15 namespace Device { 16 17 namespace SYCL { 18 19 // definition for static 20 std::array<Device::DeviceInternal *, PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {}; 21 Device::DeviceInternal **Device::devices_ = &Device::devices_array_[1]; 22 int Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE; 23 bool Device::initialized_ = false; 24 25 static std::jmp_buf MPISyclAwareJumpBuffer; 26 static bool MPISyclAwareJumpBufferSet; 27 28 // internal "impls" class for SyclDevice. Each instance represents a single sycl device 29 class PETSC_NODISCARD Device::DeviceInternal { 30 const int id_; // -1 for the host device; 0 and up for gpu devices 31 bool devInitialized_; 32 const sycl::device syclDevice_; 33 34 public: 35 // default constructor 36 DeviceInternal(int id) noexcept : id_(id), devInitialized_(false), syclDevice_(chooseSYCLDevice_(id)) { } 37 int id() const { return id_; } 38 bool initialized() const { return devInitialized_; } 39 40 PETSC_NODISCARD PetscErrorCode initialize() noexcept { 41 PetscFunctionBegin; 42 if (devInitialized_) PetscFunctionReturn(0); 43 if (syclDevice_.is_gpu() && use_gpu_aware_mpi) { 44 if (!isMPISyclAware_()) { 45 (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n"); 46 (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n"); 47 PETSCABORT(PETSC_COMM_SELF, PETSC_ERR_LIB); 48 } 49 } 50 devInitialized_ = true; 51 PetscFunctionReturn(0); 52 } 53 54 PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept { 55 MPI_Comm comm; 56 PetscMPIInt rank; 57 PetscBool iascii; 58 59 PetscFunctionBegin; 60 PetscCheck(devInitialized_, PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d being viewed before it was initialized or configured", id_); 61 PetscCall(PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer), PETSCVIEWERASCII, &iascii)); 62 PetscCall(PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer), &comm)); 63 if (iascii) { 64 PetscViewer sviewer; 65 66 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 67 PetscCall(PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sviewer)); 68 PetscCall(PetscViewerASCIIPrintf(sviewer, "[%d] device: %s\n", rank, syclDevice_.get_info<sycl::info::device::name>().c_str())); 69 PetscCall(PetscViewerASCIIPushTab(sviewer)); 70 PetscCall(PetscViewerASCIIPrintf(sviewer, "-> Device vendor: %s\n", syclDevice_.get_info<sycl::info::device::vendor>().c_str())); 71 PetscCall(PetscViewerASCIIPopTab(sviewer)); 72 PetscCall(PetscViewerFlush(sviewer)); 73 PetscCall(PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sviewer)); 74 PetscCall(PetscViewerFlush(viewer)); 75 } 76 PetscFunctionReturn(0); 77 } 78 79 PETSC_NODISCARD PetscErrorCode getattribute(PetscDeviceAttribute attr, void *value) const noexcept { 80 PetscFunctionBegin; 81 PetscCheck(initialized(), PETSC_COMM_SELF, PETSC_ERR_COR, "Device %d not initialized", id()); 82 switch (attr) { 83 case PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK: *static_cast<std::size_t *>(value) = syclDevice_.get_info<sycl::info::device::local_mem_size>(); 84 case PETSC_DEVICE_ATTR_MAX: break; 85 } 86 PetscFunctionReturn(0); 87 } 88 89 private: 90 static sycl::device chooseSYCLDevice_(int id) { 91 if (id == PETSC_SYCL_DEVICE_HOST) { 92 return sycl::device(sycl::host_selector()); 93 } else { 94 return sycl::device::get_devices(sycl::info::device_type::gpu)[id]; 95 } 96 } 97 98 // Is the underlying MPI aware of sycl (GPU) devices? 99 bool isMPISyclAware_() noexcept { 100 const int bufSize = 2; 101 const int hbuf[bufSize] = {1, 0}; 102 int *dbuf = nullptr; 103 bool awareness = false; 104 const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode { 105 if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer, 1); 106 return PetscSignalHandlerDefault(signal, ptr); 107 }; 108 109 PetscFunctionBegin; 110 auto Q = sycl::queue(syclDevice_); 111 dbuf = sycl::malloc_device<int>(bufSize, Q); 112 Q.memcpy(dbuf, hbuf, sizeof(int) * bufSize).wait(); 113 PetscCallAbort(PETSC_COMM_SELF, PetscPushSignalHandler(SyclSignalHandler, nullptr)); 114 MPISyclAwareJumpBufferSet = true; 115 if (setjmp(MPISyclAwareJumpBuffer)) { 116 // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware 117 awareness = false; 118 PetscStackPop; 119 } else if (!MPI_Allreduce(dbuf, dbuf + 1, 1, MPI_INT, MPI_SUM, PETSC_COMM_SELF)) awareness = true; 120 MPISyclAwareJumpBufferSet = false; 121 PetscCallAbort(PETSC_COMM_SELF, PetscPopSignalHandler()); 122 sycl::free(dbuf, Q); 123 PetscFunctionReturn(awareness); 124 } 125 }; 126 127 PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscDeviceInitType *defaultInitType) noexcept { 128 PetscInt initType = *defaultInitType, id = *defaultDeviceId; 129 PetscBool view = PETSC_FALSE, flg; 130 PetscInt ngpus; 131 132 PetscFunctionBegin; 133 if (initialized_) PetscFunctionReturn(0); 134 initialized_ = true; 135 PetscCall(PetscRegisterFinalize(finalize_)); 136 137 PetscOptionsBegin(comm, nullptr, "PetscDevice SYCL Options", "Sys"); 138 PetscCall(PetscOptionsEList("-device_enable_sycl", "How (or whether) to initialize a device", "SyclDevice::initialize()", PetscDeviceInitTypes, 3, PetscDeviceInitTypes[initType], &initType, nullptr)); 139 PetscCall(PetscOptionsRangeInt("-device_select_sycl", "Which sycl device to use? Pass -2 for host, PETSC_DECIDE (-1) to let PETSc decide, 0 and up for GPUs", "PetscDeviceCreate", id, &id, nullptr, -2, std::numeric_limits<decltype(ngpus)>::max())); 140 PetscCall(PetscOptionsBool("-device_view_sycl", "Display device information and assignments (forces eager initialization)", nullptr, view, &view, &flg)); 141 PetscOptionsEnd(); 142 143 // post-process the options and lay the groundwork for initialization if needs be 144 std::vector<sycl::device> gpu_devices = sycl::device::get_devices(sycl::info::device_type::gpu); 145 ngpus = static_cast<PetscInt>(gpu_devices.size()); 146 PetscCheck(ngpus || id < 0, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there is no GPU", (int)id); 147 PetscCheck(ngpus <= 0 || id < ngpus, comm, PETSC_ERR_USER_INPUT, "You specified a sycl gpu device with -device_select_sycl %d but there are only %d GPU", (int)id, (int)ngpus); 148 149 if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */ 150 else { 151 PetscCall(PetscDeviceCheckDeviceCount_Internal(ngpus)); 152 if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */ 153 if (ngpus) { 154 PetscMPIInt rank; 155 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 156 id = rank % ngpus; 157 } else id = PETSC_SYCL_DEVICE_HOST; 158 } 159 view = static_cast<decltype(view)>(view && flg); 160 if (view) initType = PETSC_DEVICE_INIT_EAGER; 161 } 162 163 if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form 164 165 defaultDevice_ = static_cast<decltype(defaultDevice_)>(id); 166 PetscCheck(initType != PETSC_DEVICE_INIT_EAGER || id != PETSC_SYCL_DEVICE_NONE, comm, PETSC_ERR_USER_INPUT, "Cannot eagerly initialize sycl devices as you disabled them by -device_enable_sycl none"); 167 168 if (initType == PETSC_DEVICE_INIT_EAGER) { 169 devices_[defaultDevice_] = new DeviceInternal(defaultDevice_); 170 PetscCall(devices_[defaultDevice_]->initialize()); 171 if (view) { 172 PetscViewer viewer; 173 PetscCall(PetscLogInitialize()); 174 PetscCall(PetscViewerASCIIGetStdout(comm, &viewer)); 175 PetscCall(devices_[defaultDevice_]->view(viewer)); 176 } 177 } 178 179 // record the results of the initialization 180 *defaultInitType = static_cast<PetscDeviceInitType>(initType); 181 *defaultDeviceId = id; 182 PetscFunctionReturn(0); 183 } 184 185 PetscErrorCode Device::finalize_() noexcept { 186 PetscFunctionBegin; 187 if (!initialized_) PetscFunctionReturn(0); 188 for (auto &&devPtr : devices_array_) delete devPtr; 189 defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default 190 initialized_ = false; 191 PetscFunctionReturn(0); 192 } 193 194 PetscErrorCode Device::getDevice(PetscDevice device, PetscInt id) const noexcept { 195 PetscFunctionBegin; 196 PetscCheck(defaultDevice_ != PETSC_SYCL_DEVICE_NONE, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Trying to retrieve a SYCL PetscDevice when it has been disabled"); 197 if (id == PETSC_DECIDE) id = defaultDevice_; 198 PetscCheck(!(id < PETSC_SYCL_DEVICE_HOST) && !(id - PETSC_SYCL_DEVICE_HOST >= PETSC_DEVICE_MAX_DEVICES), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Only supports %zu number of devices but trying to get device with id %" PetscInt_FMT, 199 devices_array_.size(), id); 200 if (devices_[id]) { 201 PetscCheck(id == devices_[id]->id(), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Entry %" PetscInt_FMT " contains device with mismatching id %" PetscInt_FMT, id, devices_[id]->id()); 202 } else devices_[id] = new DeviceInternal(id); 203 PetscCall(devices_[id]->initialize()); 204 device->deviceId = devices_[id]->id(); // technically id = devices_[id]->id_ here 205 device->ops->createcontext = create_; 206 device->ops->configure = this->configureDevice; 207 device->ops->view = this->viewDevice; 208 device->ops->getattribute = this->getAttribute; 209 PetscFunctionReturn(0); 210 } 211 212 PetscErrorCode Device::configureDevice(PetscDevice device) noexcept { 213 PetscFunctionBegin; 214 // Nothing for now 215 PetscFunctionReturn(0); 216 } 217 218 PetscErrorCode Device::viewDevice(PetscDevice device, PetscViewer viewer) noexcept { 219 PetscFunctionBegin; 220 PetscCall(devices_[device->deviceId]->view(viewer)); 221 PetscFunctionReturn(0); 222 } 223 224 PetscErrorCode Device::getAttribute(PetscDevice device, PetscDeviceAttribute attr, void *value) noexcept { 225 PetscFunctionBegin; 226 PetscCall(devices_[device->deviceId]->getattribute(attr, value)); 227 PetscFunctionReturn(0); 228 } 229 230 } // namespace SYCL 231 232 } // namespace Device 233 234 } // namespace Petsc 235