1 #include "../../interface/sycldevice.hpp" 2 #include <csetjmp> // for MPI sycl device awareness 3 #include <csignal> // SIGSEGV 4 #include <vector> 5 #include <CL/sycl.hpp> 6 7 #if PetscDefined(USE_LOG) 8 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 9 #else 10 #define PetscLogInitialize() 0 11 #endif 12 13 namespace Petsc 14 { 15 16 namespace Device 17 { 18 19 namespace SYCL 20 { 21 22 // definition for static 23 std::array<Device::DeviceInternal*,PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {}; 24 Device::DeviceInternal** Device::devices_ = &Device::devices_array_[1]; 25 int Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE; 26 bool Device::initialized_ = false; 27 28 static std::jmp_buf MPISyclAwareJumpBuffer; 29 static bool MPISyclAwareJumpBufferSet; 30 31 // internal "impls" class for SyclDevice. Each instance represents a single sycl device 32 class PETSC_NODISCARD Device::DeviceInternal 33 { 34 const int id_; // -1 for the host device; 0 and up for gpu devices 35 bool devInitialized_; 36 const sycl::device syclDevice_; 37 38 public: 39 // default constructor 40 DeviceInternal(int id) noexcept : id_(id),devInitialized_(false),syclDevice_(chooseSYCLDevice_(id)){} 41 int id() const {return id_;} 42 bool initialized() const {return devInitialized_;} 43 44 PETSC_NODISCARD PetscErrorCode initialize() noexcept 45 { 46 PetscFunctionBegin; 47 if (devInitialized_) PetscFunctionReturn(0); 48 if (syclDevice_.is_gpu() && use_gpu_aware_mpi) { 49 if (!isMPISyclAware_()) { 50 (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n"); 51 (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n"); 52 PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB); 53 } 54 } 55 devInitialized_ = true; 56 PetscFunctionReturn(0); 57 } 58 59 PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept 60 { 61 PetscErrorCode ierr; 62 MPI_Comm comm; 63 PetscMPIInt rank; 64 PetscBool iascii; 65 66 PetscFunctionBegin; 67 PetscCheckFalse(!devInitialized_,PETSC_COMM_SELF,PETSC_ERR_COR,"Device %d being viewed before it was initialized or configured",id_); 68 ierr = PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer),PETSCVIEWERASCII,&iascii);CHKERRQ(ierr); 69 ierr = PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer),&comm);CHKERRQ(ierr); 70 if (iascii) { 71 PetscViewer sviewer; 72 73 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 74 ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr); 75 ierr = PetscViewerASCIIPrintf(sviewer,"[%d] device: %s\n",rank,syclDevice_.get_info<sycl::info::device::name>().c_str());CHKERRQ(ierr); 76 ierr = PetscViewerASCIIPushTab(sviewer);CHKERRQ(ierr); 77 ierr = PetscViewerASCIIPrintf(sviewer,"-> Device vendor: %s\n",syclDevice_.get_info<sycl::info::device::vendor>().c_str());CHKERRQ(ierr); 78 ierr = PetscViewerASCIIPopTab(sviewer);CHKERRQ(ierr); 79 ierr = PetscViewerFlush(sviewer);CHKERRQ(ierr); 80 ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr); 81 ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); 82 } 83 PetscFunctionReturn(0); 84 } 85 86 private: 87 static sycl::device chooseSYCLDevice_(int id) 88 { 89 if (id == PETSC_SYCL_DEVICE_HOST) { 90 return sycl::device(sycl::host_selector()); 91 } else { 92 return sycl::device::get_devices(sycl::info::device_type::gpu)[id]; 93 } 94 } 95 96 // Is the underlying MPI aware of sycl (GPU) devices? 97 bool isMPISyclAware_() noexcept 98 { 99 PetscErrorCode ierr; 100 const int bufSize = 2; 101 const int hbuf[bufSize] = {1,0}; 102 int *dbuf = nullptr; 103 bool awareness = false; 104 const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode { 105 if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer,1); 106 return PetscSignalHandlerDefault(signal,ptr); 107 }; 108 109 PetscFunctionBegin; 110 auto Q = sycl::queue(syclDevice_); 111 dbuf = sycl::malloc_device<int>(bufSize,Q); 112 Q.memcpy(dbuf,hbuf,sizeof(int)*bufSize).wait(); 113 ierr = PetscPushSignalHandler(SyclSignalHandler,nullptr);CHKERRABORT(PETSC_COMM_SELF,ierr); 114 MPISyclAwareJumpBufferSet = true; 115 if (setjmp(MPISyclAwareJumpBuffer)) { 116 // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware 117 awareness = false; 118 PetscStackPop; 119 } else if (!MPI_Allreduce(dbuf,dbuf+1,1,MPI_INT,MPI_SUM,PETSC_COMM_SELF)) awareness = true; 120 MPISyclAwareJumpBufferSet = false; 121 ierr = PetscPopSignalHandler();CHKERRABORT(PETSC_COMM_SELF,ierr); 122 sycl::free(dbuf,Q); 123 PetscFunctionReturn(awareness); 124 } 125 }; 126 127 PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscDeviceInitType *defaultInitType) noexcept 128 { 129 PetscInt initType = *defaultInitType,id = *defaultDeviceId; 130 PetscBool view = PETSC_FALSE,flg; 131 PetscInt ngpus; 132 PetscErrorCode ierr; 133 134 PetscFunctionBegin; 135 if (initialized_) PetscFunctionReturn(0); 136 initialized_ = true; 137 ierr = PetscRegisterFinalize(finalize_);CHKERRQ(ierr); 138 139 ierr = PetscOptionsBegin(comm,nullptr,"PetscDevice SYCL Options","Sys");CHKERRQ(ierr); 140 ierr = PetscOptionsEList("-device_enable_sycl","How (or whether) to initialize a device","SyclDevice::initialize()",PetscDeviceInitTypes,3,PetscDeviceInitTypes[initType],&initType,nullptr);CHKERRQ(ierr); 141 ierr = PetscOptionsRangeInt("-device_select_sycl","Which sycl device to use? Pass -2 for host, PETSC_DECIDE (-1) to let PETSc decide, 0 and up for GPUs","PetscDeviceCreate",id,&id,nullptr,-2,std::numeric_limits<decltype(ngpus)>::max());CHKERRQ(ierr); 142 ierr = PetscOptionsBool("-device_view_sycl","Display device information and assignments (forces eager initialization)",nullptr,view,&view,&flg);CHKERRQ(ierr); 143 ierr = PetscOptionsEnd();CHKERRQ(ierr); 144 145 // post-process the options and lay the groundwork for initialization if needs be 146 std::vector<sycl::device> gpu_devices = sycl::device::get_devices(sycl::info::device_type::gpu); 147 ngpus = static_cast<PetscInt>(gpu_devices.size()); 148 PetscCheckFalse(ngpus == 0 && id >= 0,comm,PETSC_ERR_USER_INPUT,"You specified a sycl gpu device with -device_select_sycl %d but there is no GPU", (int)id); 149 PetscCheckFalse(ngpus > 0 && id >= ngpus,comm,PETSC_ERR_USER_INPUT,"You specified a sycl gpu device with -device_select_sycl %d but there are only %d GPU", (int)id, (int)ngpus); 150 151 if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */ 152 else { 153 ierr = PetscDeviceCheckDeviceCount_Internal(ngpus);CHKERRQ(ierr); 154 if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */ 155 if (ngpus) { 156 PetscMPIInt rank; 157 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 158 id = rank % ngpus; 159 } else id = PETSC_SYCL_DEVICE_HOST; 160 } 161 view = static_cast<decltype(view)>(view && flg); 162 if (view) initType = PETSC_DEVICE_INIT_EAGER; 163 } 164 165 if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form 166 167 defaultDevice_ = static_cast<decltype(defaultDevice_)>(id); 168 PetscCheckFalse(initType == PETSC_DEVICE_INIT_EAGER && id == PETSC_SYCL_DEVICE_NONE,comm,PETSC_ERR_USER_INPUT,"Cannot eagerly initialize sycl devices as you disabled them by -device_enable_sycl none"); 169 170 if (initType == PETSC_DEVICE_INIT_EAGER) { 171 devices_[defaultDevice_] = new DeviceInternal(defaultDevice_); 172 ierr = devices_[defaultDevice_]->initialize();CHKERRQ(ierr); 173 if (view) { 174 PetscViewer viewer; 175 ierr = PetscLogInitialize();CHKERRQ(ierr); 176 ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr); 177 ierr = devices_[defaultDevice_]->view(viewer);CHKERRQ(ierr); 178 } 179 } 180 181 // record the results of the initialization 182 *defaultInitType = static_cast<PetscDeviceInitType>(initType); 183 *defaultDeviceId = id; 184 PetscFunctionReturn(0); 185 } 186 187 PetscErrorCode Device::finalize_() noexcept 188 { 189 PetscFunctionBegin; 190 if (!initialized_) PetscFunctionReturn(0); 191 for (auto&& devPtr : devices_array_) delete devPtr; 192 defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default 193 initialized_ = false; 194 PetscFunctionReturn(0); 195 } 196 197 PetscErrorCode Device::getDevice(PetscDevice device, PetscInt id) const noexcept 198 { 199 PetscErrorCode ierr; 200 201 PetscFunctionBegin; 202 PetscCheckFalse(defaultDevice_ == PETSC_SYCL_DEVICE_NONE,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Trying to retrieve a SYCL PetscDevice when it has been disabled"); 203 if (id == PETSC_DECIDE) id = defaultDevice_; 204 PetscCheckFalse((id < PETSC_SYCL_DEVICE_HOST) || (id-PETSC_SYCL_DEVICE_HOST >= PETSC_DEVICE_MAX_DEVICES),PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Only supports %zu number of devices but trying to get device with id %" PetscInt_FMT,devices_array_.size(),id); 205 if (devices_[id]) { 206 PetscCheckFalse(id != devices_[id]->id(),PETSC_COMM_SELF,PETSC_ERR_PLIB,"Entry %" PetscInt_FMT " contains device with mismatching id %" PetscInt_FMT,id,devices_[id]->id()); 207 } else devices_[id] = new DeviceInternal(id); 208 ierr = devices_[id]->initialize();CHKERRQ(ierr); 209 device->deviceId = devices_[id]->id(); // technically id = devices_[id]->id_ here 210 device->ops->createcontext = create_; 211 device->ops->configure = this->configureDevice; 212 device->ops->view = this->viewDevice; 213 PetscFunctionReturn(0); 214 } 215 216 PetscErrorCode Device::configureDevice(PetscDevice device) noexcept 217 { 218 PetscFunctionBegin; 219 // Nothing for now 220 PetscFunctionReturn(0); 221 } 222 223 PetscErrorCode Device::viewDevice(PetscDevice device, PetscViewer viewer) noexcept 224 { 225 PetscErrorCode ierr; 226 227 PetscFunctionBegin; 228 ierr = devices_[device->deviceId]->view(viewer);CHKERRQ(ierr); 229 PetscFunctionReturn(0); 230 } 231 232 } // namespace SYCL 233 234 } // namespace Device 235 236 } // namespace Petsc 237