1 #include "../../interface/sycldevice.hpp" 2 #include <csetjmp> // for MPI sycl device awareness 3 #include <csignal> // SIGSEGV 4 #include <vector> 5 #include <CL/sycl.hpp> 6 7 #if PetscDefined(USE_LOG) 8 PETSC_INTERN PetscErrorCode PetscLogInitialize(void); 9 #else 10 #define PetscLogInitialize() 0 11 #endif 12 13 namespace Petsc 14 { 15 16 // definition for static 17 std::array<SyclDevice::SyclDeviceInternal*,PETSC_DEVICE_MAX_DEVICES> SyclDevice::devices_array_ = {}; 18 SyclDevice::SyclDeviceInternal** SyclDevice::devices_ = &SyclDevice::devices_array_[1]; 19 int SyclDevice::defaultDevice_ = PETSC_SYCL_DEVICE_NONE; 20 bool SyclDevice::initialized_ = false; 21 22 static std::jmp_buf MPISyclAwareJumpBuffer; 23 static bool MPISyclAwareJumpBufferSet; 24 25 // internal "impls" class for SyclDevice. Each instance represents a single sycl device 26 class PETSC_NODISCARD SyclDevice::SyclDeviceInternal 27 { 28 const int id_; // -1 for the host device; 0 and up for gpu devices 29 bool devInitialized_; 30 const sycl::device syclDevice_; 31 32 public: 33 // default constructor 34 SyclDeviceInternal(int id) noexcept : id_(id),devInitialized_(false),syclDevice_(chooseSYCLDevice_(id)){} 35 int id() const {return id_;} 36 bool initialized() const {return devInitialized_;} 37 38 PETSC_NODISCARD PetscErrorCode initialize() noexcept 39 { 40 PetscFunctionBegin; 41 if (devInitialized_) PetscFunctionReturn(0); 42 if (syclDevice_.is_gpu() && use_gpu_aware_mpi) { 43 if (!isMPISyclAware_()) { 44 (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n"); 45 (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n"); 46 PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB); 47 } 48 } 49 devInitialized_ = true; 50 PetscFunctionReturn(0); 51 } 52 53 PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept 54 { 55 PetscErrorCode ierr; 56 MPI_Comm comm; 57 PetscMPIInt rank; 58 PetscBool iascii; 59 60 PetscFunctionBegin; 61 if (PetscUnlikely(!devInitialized_)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_COR,"Device %d being viewed before it was initialized or configured",id_); 62 ierr = PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer),PETSCVIEWERASCII,&iascii);CHKERRQ(ierr); 63 ierr = PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer),&comm);CHKERRQ(ierr); 64 if (iascii) { 65 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 66 ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr); 67 ierr = PetscViewerASCIISynchronizedPrintf(viewer,"[%d] device: %s\n",rank,syclDevice_.get_info<sycl::info::device::name>().c_str());CHKERRQ(ierr); 68 // flush the assignment information 69 ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); 70 ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr); 71 ierr = PetscViewerASCIISynchronizedPrintf(viewer,"-> Device vendor: %s\n",syclDevice_.get_info<sycl::info::device::vendor>().c_str());CHKERRQ(ierr); 72 ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr); 73 ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); 74 ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr); 75 } 76 PetscFunctionReturn(0); 77 } 78 79 private: 80 static sycl::device chooseSYCLDevice_(int id) 81 { 82 if (id == PETSC_SYCL_DEVICE_HOST) { 83 return sycl::device(sycl::host_selector()); 84 } else { 85 return sycl::device::get_devices(sycl::info::device_type::gpu)[id]; 86 } 87 } 88 89 // Is the underlying MPI aware of sycl (GPU) devices? 90 bool isMPISyclAware_() noexcept 91 { 92 PetscErrorCode ierr; 93 const int bufSize = 2; 94 const int hbuf[bufSize] = {1,0}; 95 int *dbuf = nullptr; 96 bool awareness = false; 97 const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode { 98 if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer,1); 99 return PetscSignalHandlerDefault(signal,ptr); 100 }; 101 102 PetscFunctionBegin; 103 auto Q = sycl::queue(syclDevice_); 104 dbuf = sycl::malloc_device<int>(bufSize,Q); 105 Q.memcpy(dbuf,hbuf,sizeof(int)*bufSize).wait(); 106 ierr = PetscPushSignalHandler(SyclSignalHandler,nullptr);CHKERRABORT(PETSC_COMM_SELF,ierr); 107 MPISyclAwareJumpBufferSet = true; 108 if (setjmp(MPISyclAwareJumpBuffer)) { 109 // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware 110 awareness = false; 111 PetscStackPop; 112 } else if (!MPI_Allreduce(dbuf,dbuf+1,1,MPI_INT,MPI_SUM,PETSC_COMM_SELF)) awareness = true; 113 MPISyclAwareJumpBufferSet = false; 114 ierr = PetscPopSignalHandler();CHKERRABORT(PETSC_COMM_SELF,ierr); 115 sycl::free(dbuf,Q); 116 PetscFunctionReturn(awareness); 117 } 118 }; 119 120 PetscErrorCode SyclDevice::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscDeviceInitType *defaultInitType) noexcept 121 { 122 PetscInt initType = *defaultInitType,id = *defaultDeviceId; 123 PetscBool view = PETSC_FALSE,flg; 124 PetscInt ngpus; 125 PetscErrorCode ierr; 126 127 PetscFunctionBegin; 128 if (initialized_) PetscFunctionReturn(0); 129 initialized_ = true; 130 ierr = PetscRegisterFinalize(finalize_);CHKERRQ(ierr); 131 132 ierr = PetscOptionsBegin(comm,nullptr,"PetscDevice SYCL Options","Sys");CHKERRQ(ierr); 133 ierr = PetscOptionsEList("-device_enable_sycl","How (or whether) to initialize a device","SyclDevice::initialize()",PetscDeviceInitTypes,3,PetscDeviceInitTypes[initType],&initType,nullptr);CHKERRQ(ierr); 134 ierr = PetscOptionsRangeInt("-device_select_sycl","Which sycl device to use? Pass -2 for host, PETSC_DECIDE (-1) to let PETSc decide, 0 and up for GPUs","PetscDeviceCreate",id,&id,nullptr,-2,std::numeric_limits<decltype(ngpus)>::max());CHKERRQ(ierr); 135 ierr = PetscOptionsBool("-device_view_sycl","Display device information and assignments (forces eager initialization)",nullptr,view,&view,&flg);CHKERRQ(ierr); 136 ierr = PetscOptionsEnd();CHKERRQ(ierr); 137 138 // post-process the options and lay the groundwork for initialization if needs be 139 std::vector<sycl::device> gpu_devices = sycl::device::get_devices(sycl::info::device_type::gpu); 140 ngpus = static_cast<PetscInt>(gpu_devices.size()); 141 if (PetscUnlikely(ngpus == 0 && id >= 0)) SETERRQ1(comm,PETSC_ERR_USER_INPUT,"You specified a sycl gpu device with -device_select_sycl %d but there is no GPU", (int)id); 142 if (PetscUnlikely(ngpus > 0 && id >= ngpus)) SETERRQ2(comm,PETSC_ERR_USER_INPUT,"You specified a sycl gpu device with -device_select_sycl %d but there are only %d GPU", (int)id, (int)ngpus); 143 144 if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */ 145 else { 146 ierr = PetscDeviceCheckDeviceCount_Internal(ngpus);CHKERRQ(ierr); 147 if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */ 148 if (ngpus) { 149 PetscMPIInt rank; 150 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 151 id = rank % ngpus; 152 } else id = PETSC_SYCL_DEVICE_HOST; 153 } 154 view = static_cast<decltype(view)>(view && flg); 155 if (view) initType = PETSC_DEVICE_INIT_EAGER; 156 } 157 158 if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form 159 160 defaultDevice_ = static_cast<decltype(defaultDevice_)>(id); 161 if (PetscUnlikely(initType == PETSC_DEVICE_INIT_EAGER && id == PETSC_SYCL_DEVICE_NONE)) SETERRQ(comm,PETSC_ERR_USER_INPUT,"Cannot eagerly initialize sycl devices as you disabled them by -device_enable_sycl none"); 162 163 if (initType == PETSC_DEVICE_INIT_EAGER) { 164 devices_[defaultDevice_] = new SyclDeviceInternal(defaultDevice_); 165 ierr = devices_[defaultDevice_]->initialize();CHKERRQ(ierr); 166 if (view) { 167 PetscViewer viewer; 168 ierr = PetscLogInitialize();CHKERRQ(ierr); 169 ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr); 170 ierr = devices_[defaultDevice_]->view(viewer);CHKERRQ(ierr); 171 } 172 } 173 174 // record the results of the initialization 175 *defaultInitType = static_cast<PetscDeviceInitType>(initType); 176 *defaultDeviceId = id; 177 PetscFunctionReturn(0); 178 } 179 180 PetscErrorCode SyclDevice::finalize_() noexcept 181 { 182 PetscFunctionBegin; 183 if (!initialized_) PetscFunctionReturn(0); 184 for (auto&& devPtr : devices_array_) delete devPtr; 185 defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default 186 initialized_ = false; 187 PetscFunctionReturn(0); 188 } 189 190 PetscErrorCode SyclDevice::getDevice(PetscDevice device, PetscInt id) const noexcept 191 { 192 PetscErrorCode ierr; 193 194 PetscFunctionBegin; 195 if (PetscUnlikely(defaultDevice_ == PETSC_SYCL_DEVICE_NONE)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Trying to retrieve a SYCL PetscDevice when it has been disabled"); 196 if (id == PETSC_DECIDE) id = defaultDevice_; 197 if ((id < PETSC_SYCL_DEVICE_HOST) || (id-PETSC_SYCL_DEVICE_HOST >= PETSC_DEVICE_MAX_DEVICES)) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Only supports %zu number of devices but trying to get device with id %" PetscInt_FMT,devices_array_.size(),id); 198 if (devices_[id]) { 199 if (id != devices_[id]->id()) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Entry %" PetscInt_FMT " contains device with mismatching id %" PetscInt_FMT,id,devices_[id]->id()); 200 } else devices_[id] = new SyclDeviceInternal(id); 201 ierr = devices_[id]->initialize();CHKERRQ(ierr); 202 device->deviceId = devices_[id]->id(); // technically id = devices_[id]->id_ here 203 device->ops->createcontext = create_; 204 device->ops->configure = this->configureDevice; 205 device->ops->view = this->viewDevice; 206 PetscFunctionReturn(0); 207 } 208 209 PetscErrorCode SyclDevice::configureDevice(PetscDevice device) noexcept 210 { 211 PetscFunctionBegin; 212 // Nothing for now 213 PetscFunctionReturn(0); 214 } 215 216 PetscErrorCode SyclDevice::viewDevice(PetscDevice device, PetscViewer viewer) noexcept 217 { 218 PetscErrorCode ierr; 219 220 PetscFunctionBegin; 221 ierr = devices_[device->deviceId]->view(viewer);CHKERRQ(ierr); 222 PetscFunctionReturn(0); 223 } 224 225 } // namespace Petsc 226