1 #ifndef IMPLDEVICEBASE_HPP 2 #define IMPLDEVICEBASE_HPP 3 4 #include <petsc/private/deviceimpl.h> 5 #include <petsc/private/viewerimpl.h> 6 7 #include <petsc/private/cpp/crtp.hpp> 8 #include <petsc/private/cpp/type_traits.hpp> 9 #include <petsc/private/cpp/utility.hpp> 10 #include <petsc/private/cpp/array.hpp> 11 12 #include <cstring> // for std::strlen 13 14 namespace Petsc 15 { 16 17 namespace device 18 { 19 20 namespace impl 21 { 22 23 template <typename Derived> // CRTP 24 class DeviceBase : public util::crtp<DeviceBase, Derived> { 25 public: 26 using derived_type = Derived; 27 using createContextFunction_t = PetscErrorCode (*)(PetscDeviceContext); 28 29 // default constructor 30 constexpr DeviceBase(createContextFunction_t f) noexcept : create_(f) { } 31 32 template <typename T = derived_type> 33 PETSC_NODISCARD static constexpr PetscDeviceType PETSC_DEVICE_IMPL() noexcept 34 { 35 return T::PETSC_DEVICE_IMPL_(); 36 } 37 38 PetscErrorCode getDevice(PetscDevice, PetscInt) noexcept; 39 static PetscErrorCode configureDevice(PetscDevice) noexcept; 40 static PetscErrorCode viewDevice(PetscDevice, PetscViewer) noexcept; 41 static PetscErrorCode getAttribute(PetscDevice, PetscDeviceAttribute, void *) noexcept; 42 43 protected: 44 // function to create a PetscDeviceContext (the (*create) function pointer usually set 45 // via XXXSetType() for other PETSc objects) 46 const createContextFunction_t create_; 47 48 // if you want the base class to handle the entire options query, has the same arguments as 49 // PetscOptionDeviceBasic 50 static PetscErrorCode PetscOptionDeviceAll(MPI_Comm, std::pair<PetscDeviceInitType, PetscBool> &, std::pair<PetscInt, PetscBool> &, std::pair<PetscBool, PetscBool> &) noexcept; 51 52 // if you want to start and end the options query yourself, but still want all the default 53 // options 54 static PetscErrorCode PetscOptionDeviceBasic(PetscOptionItems *, std::pair<PetscDeviceInitType, PetscBool> &, std::pair<PetscInt, PetscBool> &, std::pair<PetscBool, PetscBool> &) noexcept; 55 56 // option templates to follow, each one has two forms: 57 // - A simple form returning only the value and flag. This gives no control over the message, 58 // arguments to the options query or otherwise 59 // - A complex form, which allows you to pass most of the options query arguments *EXCEPT* 60 // - The options query function called 61 // - The option string 62 63 // option template for initializing the device 64 static PetscErrorCode PetscOptionDeviceInitialize(PetscOptionItems *, PetscDeviceInitType *, PetscBool *) noexcept; 65 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int> = 0> 66 static PetscErrorCode PetscOptionDeviceInitialize(PetscOptionItems *, T &&...) noexcept; 67 // option template for selecting the default device 68 static PetscErrorCode PetscOptionDeviceSelect(PetscOptionItems *, PetscInt *, PetscBool *) noexcept; 69 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int> = 0> 70 static PetscErrorCode PetscOptionDeviceSelect(PetscOptionItems *, T &&...) noexcept; 71 // option templates for viewing a device 72 static PetscErrorCode PetscOptionDeviceView(PetscOptionItems *, PetscBool *, PetscBool *) noexcept; 73 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int> = 0> 74 static PetscErrorCode PetscOptionDeviceView(PetscOptionItems *, T &&...) noexcept; 75 76 private: 77 // base function for all options templates above, they basically just reformat the arguments, 78 // create the option string and pass it off to this function 79 template <typename... T, typename F = PetscErrorCode (*)(PetscOptionItems *, const char *, T &&...)> 80 static PetscErrorCode PetscOptionDevice(F &&, PetscOptionItems *, const char[], T &&...) noexcept; 81 82 // default crtp implementations 83 static PetscErrorCode init_device_id_(PetscInt *id) noexcept 84 { 85 PetscFunctionBegin; 86 *id = 0; 87 PetscFunctionReturn(PETSC_SUCCESS); 88 } 89 90 static constexpr PetscErrorCode configure_device_(PetscDevice) noexcept { return PETSC_SUCCESS; } 91 static constexpr PetscErrorCode view_device_(PetscDevice, PetscViewer) noexcept { return PETSC_SUCCESS; } 92 }; 93 94 template <typename D> 95 inline PetscErrorCode DeviceBase<D>::getDevice(PetscDevice device, PetscInt id) noexcept 96 { 97 PetscFunctionBegin; 98 PetscCall(this->underlying().init_device_id_(&id)); 99 device->deviceId = id; 100 device->ops->createcontext = this->underlying().create_; 101 device->ops->configure = this->underlying().configureDevice; 102 device->ops->view = this->underlying().viewDevice; 103 device->ops->getattribute = this->underlying().getAttribute; 104 PetscFunctionReturn(PETSC_SUCCESS); 105 } 106 107 template <typename D> 108 inline PetscErrorCode DeviceBase<D>::configureDevice(PetscDevice device) noexcept 109 { 110 PetscFunctionBegin; 111 PetscCall(derived_type::configure_device_(device)); 112 PetscFunctionReturn(PETSC_SUCCESS); 113 } 114 115 template <typename D> 116 inline PetscErrorCode DeviceBase<D>::viewDevice(PetscDevice device, PetscViewer viewer) noexcept 117 { 118 PetscFunctionBegin; 119 PetscCall(derived_type::view_device_(device, viewer)); 120 PetscFunctionReturn(PETSC_SUCCESS); 121 } 122 123 template <typename D> 124 inline PetscErrorCode DeviceBase<D>::getAttribute(PetscDevice device, PetscDeviceAttribute attr, void *value) noexcept 125 { 126 PetscFunctionBegin; 127 PetscCall(derived_type::get_attribute_(device->deviceId, attr, value)); 128 PetscFunctionReturn(PETSC_SUCCESS); 129 } 130 131 template <typename D> 132 template <typename... T, typename F> 133 inline PetscErrorCode DeviceBase<D>::PetscOptionDevice(F &&OptionsFunction, PetscOptionItems *PetscOptionsObject, const char optstub[], T &&...args) noexcept 134 { 135 constexpr auto dtype = PETSC_DEVICE_IMPL(); 136 const auto implname = PetscDeviceTypes[dtype]; 137 auto buf = std::array<char, 128>{}; 138 constexpr auto buflen = buf.size() - 1; 139 140 PetscFunctionBegin; 141 if (PetscDefined(USE_DEBUG)) { 142 const auto len = std::strlen(optstub) + std::strlen(implname); 143 144 PetscCheck(len < buflen, PetscOptionsObject->comm, PETSC_ERR_PLIB, "char buffer is not large enough to hold '%s%s'; have %zu need %zu", optstub, implname, buflen, len); 145 } 146 PetscCall(PetscSNPrintf(buf.data(), buflen, "%s%s", optstub, implname)); 147 PetscCall(OptionsFunction(PetscOptionsObject, buf.data(), std::forward<T>(args)...)); 148 PetscFunctionReturn(PETSC_SUCCESS); 149 } 150 151 template <typename D> 152 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int>> 153 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceInitialize(PetscOptionItems *PetscOptionsObject, T &&...args) noexcept 154 { 155 PetscFunctionBegin; 156 PetscCall(PetscOptionDevice(PetscOptionsEList_Private, PetscOptionsObject, "-device_enable_", std::forward<T>(args)...)); 157 PetscFunctionReturn(PETSC_SUCCESS); 158 } 159 160 template <typename D> 161 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceInitialize(PetscOptionItems *PetscOptionsObject, PetscDeviceInitType *inittype, PetscBool *flag) noexcept 162 { 163 auto type = static_cast<PetscInt>(util::to_underlying(*inittype)); 164 165 PetscFunctionBegin; 166 PetscCall(PetscOptionDeviceInitialize(PetscOptionsObject, "How (or whether) to initialize a device", "PetscDeviceInitialize()", PetscDeviceInitTypes, 3, PetscDeviceInitTypes[type], &type, flag)); 167 *inittype = static_cast<PetscDeviceInitType>(type); 168 PetscFunctionReturn(PETSC_SUCCESS); 169 } 170 171 template <typename D> 172 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int>> 173 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceSelect(PetscOptionItems *PetscOptionsObject, T &&...args) noexcept 174 { 175 PetscFunctionBegin; 176 PetscCall(PetscOptionDevice(PetscOptionsInt_Private, PetscOptionsObject, "-device_select_", std::forward<T>(args)...)); 177 PetscFunctionReturn(PETSC_SUCCESS); 178 } 179 180 template <typename D> 181 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceSelect(PetscOptionItems *PetscOptionsObject, PetscInt *id, PetscBool *flag) noexcept 182 { 183 PetscFunctionBegin; 184 PetscCall(PetscOptionDeviceSelect(PetscOptionsObject, "Which device to use. Pass " PetscStringize(PETSC_DECIDE) " to have PETSc decide or (given they exist) [0-" PetscStringize(PETSC_DEVICE_MAX_DEVICES) ") for a specific device", "PetscDeviceCreate()", *id, id, flag, PETSC_DECIDE, PETSC_DEVICE_MAX_DEVICES)); 185 PetscFunctionReturn(PETSC_SUCCESS); 186 } 187 188 template <typename D> 189 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int>> 190 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceView(PetscOptionItems *PetscOptionsObject, T &&...args) noexcept 191 { 192 PetscFunctionBegin; 193 PetscCall(PetscOptionDevice(PetscOptionsBool_Private, PetscOptionsObject, "-device_view_", std::forward<T>(args)...)); 194 PetscFunctionReturn(PETSC_SUCCESS); 195 } 196 197 template <typename D> 198 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceView(PetscOptionItems *PetscOptionsObject, PetscBool *view, PetscBool *flag) noexcept 199 { 200 PetscFunctionBegin; 201 PetscCall(PetscOptionDeviceView(PetscOptionsObject, "Display device information and assignments (forces eager initialization)", "PetscDeviceView()", *view, view, flag)); 202 PetscFunctionReturn(PETSC_SUCCESS); 203 } 204 205 template <typename D> 206 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceBasic(PetscOptionItems *PetscOptionsObject, std::pair<PetscDeviceInitType, PetscBool> &initType, std::pair<PetscInt, PetscBool> &initId, std::pair<PetscBool, PetscBool> &initView) noexcept 207 { 208 PetscFunctionBegin; 209 PetscCall(PetscOptionDeviceInitialize(PetscOptionsObject, &initType.first, &initType.second)); 210 PetscCall(PetscOptionDeviceSelect(PetscOptionsObject, &initId.first, &initId.second)); 211 PetscCall(PetscOptionDeviceView(PetscOptionsObject, &initView.first, &initView.second)); 212 PetscFunctionReturn(PETSC_SUCCESS); 213 } 214 215 template <typename D> 216 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceAll(MPI_Comm comm, std::pair<PetscDeviceInitType, PetscBool> &initType, std::pair<PetscInt, PetscBool> &initId, std::pair<PetscBool, PetscBool> &initView) noexcept 217 { 218 constexpr char optname[] = "PetscDevice %s Options"; 219 constexpr auto dtype = PETSC_DEVICE_IMPL(); 220 const auto implname = PetscDeviceTypes[dtype]; 221 auto buf = std::array<char, 128>{}; 222 constexpr auto buflen = buf.size() - 1; // -1 to leave room for null 223 224 PetscFunctionBegin; 225 if (PetscDefined(USE_DEBUG)) { 226 // -3 since '%s' is replaced and dont count null char for optname 227 const auto len = std::strlen(implname) + PETSC_STATIC_ARRAY_LENGTH(optname) - 3; 228 229 PetscCheck(len < buflen, comm, PETSC_ERR_PLIB, "char buffer is not large enough to hold 'PetscDevice %s Options'; have %zu need %zu", implname, buflen, len); 230 } 231 PetscCall(PetscSNPrintf(buf.data(), buflen, optname, implname)); 232 PetscOptionsBegin(comm, nullptr, buf.data(), "Sys"); 233 PetscCall(PetscOptionDeviceBasic(PetscOptionsObject, initType, initId, initView)); 234 PetscOptionsEnd(); 235 PetscFunctionReturn(PETSC_SUCCESS); 236 } 237 238 } // namespace impl 239 240 } // namespace device 241 242 } // namespace Petsc 243 244 #define PETSC_DEVICE_IMPL_BASE_CLASS_HEADER(base_name, T) \ 245 using base_name = ::Petsc::device::impl::DeviceBase<T>; \ 246 friend base_name; \ 247 using base_name::base_name 248 249 #endif // IMPLDEVICEBASE_HPP 250