1 #ifndef IMPLDEVICEBASE_HPP 2 #define IMPLDEVICEBASE_HPP 3 4 #if defined(__cplusplus) 5 #include <petsc/private/deviceimpl.h> 6 #include <petsc/private/viewerimpl.h> 7 8 #include <petsc/private/cpp/crtp.hpp> 9 #include <petsc/private/cpp/type_traits.hpp> 10 #include <petsc/private/cpp/utility.hpp> 11 #include <petsc/private/cpp/array.hpp> 12 13 #include <cstring> // for std::strlen 14 15 namespace Petsc { 16 17 namespace device { 18 19 namespace impl { 20 21 template <typename Derived> // CRTP 22 class DeviceBase : public util::crtp<Derived, DeviceBase> { 23 public: 24 using derived_type = Derived; 25 using createContextFunction_t = PetscErrorCode (*)(PetscDeviceContext); 26 27 // default constructor 28 constexpr DeviceBase(createContextFunction_t f) noexcept : create_(f) { } 29 30 template <typename T = derived_type> 31 PETSC_NODISCARD static constexpr PetscDeviceType PETSC_DEVICE_IMPL() noexcept { 32 return T::PETSC_DEVICE_IMPL_(); 33 } 34 35 PETSC_NODISCARD PetscErrorCode getDevice(PetscDevice, PetscInt) noexcept; 36 PETSC_NODISCARD static PetscErrorCode configureDevice(PetscDevice) noexcept; 37 PETSC_NODISCARD static PetscErrorCode viewDevice(PetscDevice, PetscViewer) noexcept; 38 PETSC_NODISCARD static PetscErrorCode getAttribute(PetscDevice, PetscDeviceAttribute, void *) noexcept; 39 40 protected: 41 // function to create a PetscDeviceContext (the (*create) function pointer usually set 42 // via XXXSetType() for other PETSc objects) 43 const createContextFunction_t create_; 44 45 // if you want the base class to handle the entire options query, has the same arguments as 46 // PetscOptionDeviceBasic 47 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceAll(MPI_Comm, std::pair<PetscDeviceInitType, PetscBool> &, std::pair<PetscInt, PetscBool> &, std::pair<PetscBool, PetscBool> &) noexcept; 48 49 // if you want to start and end the options query yourself, but still want all the default 50 // options 51 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceBasic(PetscOptionItems *, std::pair<PetscDeviceInitType, PetscBool> &, std::pair<PetscInt, PetscBool> &, std::pair<PetscBool, PetscBool> &) noexcept; 52 53 // option templates to follow, each one has two forms: 54 // - A simple form returning only the value and flag. This gives no control over the message, 55 // arguments to the options query or otherwise 56 // - A complex form, which allows you to pass most of the options query arguments *EXCEPT* 57 // - The options query function called 58 // - The option string 59 60 // option template for initializing the device 61 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceInitialize(PetscOptionItems *, PetscDeviceInitType *, PetscBool *) noexcept; 62 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int> = 0> 63 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceInitialize(PetscOptionItems *, T &&...) noexcept; 64 // option template for selecting the default device 65 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceSelect(PetscOptionItems *, PetscInt *, PetscBool *) noexcept; 66 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int> = 0> 67 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceSelect(PetscOptionItems *, T &&...) noexcept; 68 // option templates for viewing a device 69 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceView(PetscOptionItems *, PetscBool *, PetscBool *) noexcept; 70 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int> = 0> 71 PETSC_NODISCARD static PetscErrorCode PetscOptionDeviceView(PetscOptionItems *, T &&...) noexcept; 72 73 private: 74 // base function for all options templates above, they basically just reformat the arguments, 75 // create the option string and pass it off to this function 76 template <typename... T, typename F = PetscErrorCode (*)(PetscOptionItems *, const char *, T &&...)> 77 PETSC_NODISCARD static PetscErrorCode PetscOptionDevice(F &&, PetscOptionItems *, const char[], T &&...) noexcept; 78 79 // default crtp implementations 80 PETSC_NODISCARD static PetscErrorCode init_device_id_(PetscInt *id) noexcept { 81 PetscFunctionBegin; 82 *id = 0; 83 PetscFunctionReturn(0); 84 } 85 86 PETSC_NODISCARD static constexpr PetscErrorCode configure_device_(PetscDevice) noexcept { return 0; } 87 PETSC_NODISCARD static constexpr PetscErrorCode view_device_(PetscDevice, PetscViewer) noexcept { return 0; } 88 }; 89 90 template <typename D> 91 inline PetscErrorCode DeviceBase<D>::getDevice(PetscDevice device, PetscInt id) noexcept { 92 PetscFunctionBegin; 93 PetscCall(this->underlying().init_device_id_(&id)); 94 device->deviceId = id; 95 device->ops->createcontext = this->underlying().create_; 96 device->ops->configure = this->underlying().configureDevice; 97 device->ops->view = this->underlying().viewDevice; 98 device->ops->getattribute = this->underlying().getAttribute; 99 PetscFunctionReturn(0); 100 } 101 102 template <typename D> 103 inline PetscErrorCode DeviceBase<D>::configureDevice(PetscDevice device) noexcept { 104 PetscFunctionBegin; 105 PetscCall(derived_type::configure_device_(device)); 106 PetscFunctionReturn(0); 107 } 108 109 template <typename D> 110 inline PetscErrorCode DeviceBase<D>::viewDevice(PetscDevice device, PetscViewer viewer) noexcept { 111 PetscFunctionBegin; 112 PetscCall(derived_type::view_device_(device, viewer)); 113 PetscFunctionReturn(0); 114 } 115 116 template <typename D> 117 inline PetscErrorCode DeviceBase<D>::getAttribute(PetscDevice device, PetscDeviceAttribute attr, void *value) noexcept { 118 PetscFunctionBegin; 119 PetscCall(derived_type::get_attribute_(device->deviceId, attr, value)); 120 PetscFunctionReturn(0); 121 } 122 123 template <typename D> 124 template <typename... T, typename F> 125 inline PetscErrorCode DeviceBase<D>::PetscOptionDevice(F &&OptionsFunction, PetscOptionItems *PetscOptionsObject, const char optstub[], T &&...args) noexcept { 126 constexpr auto dtype = PETSC_DEVICE_IMPL(); 127 const auto implname = PetscDeviceTypes[dtype]; 128 auto buf = std::array<char, 128>{}; 129 constexpr auto buflen = buf.size() - 1; 130 131 PetscFunctionBegin; 132 if (PetscDefined(USE_DEBUG)) { 133 const auto len = std::strlen(optstub) + std::strlen(implname); 134 135 PetscCheck(len < buflen, PetscOptionsObject->comm, PETSC_ERR_PLIB, "char buffer is not large enough to hold '%s%s'; have %zu need %zu", optstub, implname, buflen, len); 136 } 137 PetscCall(PetscSNPrintf(buf.data(), buflen, "%s%s", optstub, implname)); 138 PetscCall(OptionsFunction(PetscOptionsObject, buf.data(), std::forward<T>(args)...)); 139 PetscFunctionReturn(0); 140 } 141 142 template <typename D> 143 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int>> 144 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceInitialize(PetscOptionItems *PetscOptionsObject, T &&...args) noexcept { 145 PetscFunctionBegin; 146 PetscCall(PetscOptionDevice(PetscOptionsEList_Private, PetscOptionsObject, "-device_enable_", std::forward<T>(args)...)); 147 PetscFunctionReturn(0); 148 } 149 150 template <typename D> 151 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceInitialize(PetscOptionItems *PetscOptionsObject, PetscDeviceInitType *inittype, PetscBool *flag) noexcept { 152 auto type = static_cast<PetscInt>(util::integral_value(*inittype)); 153 154 PetscFunctionBegin; 155 PetscCall(PetscOptionDeviceInitialize(PetscOptionsObject, "How (or whether) to initialize a device", "PetscDeviceInitialize()", PetscDeviceInitTypes, 3, PetscDeviceInitTypes[type], &type, flag)); 156 *inittype = static_cast<PetscDeviceInitType>(type); 157 PetscFunctionReturn(0); 158 } 159 160 template <typename D> 161 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int>> 162 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceSelect(PetscOptionItems *PetscOptionsObject, T &&...args) noexcept { 163 PetscFunctionBegin; 164 PetscCall(PetscOptionDevice(PetscOptionsInt_Private, PetscOptionsObject, "-device_select_", std::forward<T>(args)...)); 165 PetscFunctionReturn(0); 166 } 167 168 template <typename D> 169 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceSelect(PetscOptionItems *PetscOptionsObject, PetscInt *id, PetscBool *flag) noexcept { 170 PetscFunctionBegin; 171 PetscCall(PetscOptionDeviceSelect(PetscOptionsObject, "Which device to use. Pass " PetscStringize(PETSC_DECIDE) " to have PETSc decide or (given they exist) [0-" PetscStringize(PETSC_DEVICE_MAX_DEVICES) ") for a specific device", "PetscDeviceCreate()", *id, id, flag, PETSC_DECIDE, PETSC_DEVICE_MAX_DEVICES)); 172 PetscFunctionReturn(0); 173 } 174 175 template <typename D> 176 template <typename... T, util::enable_if_t<sizeof...(T) >= 3, int>> 177 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceView(PetscOptionItems *PetscOptionsObject, T &&...args) noexcept { 178 PetscFunctionBegin; 179 PetscCall(PetscOptionDevice(PetscOptionsBool_Private, PetscOptionsObject, "-device_view_", std::forward<T>(args)...)); 180 PetscFunctionReturn(0); 181 } 182 183 template <typename D> 184 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceView(PetscOptionItems *PetscOptionsObject, PetscBool *view, PetscBool *flag) noexcept { 185 PetscFunctionBegin; 186 PetscCall(PetscOptionDeviceView(PetscOptionsObject, "Display device information and assignments (forces eager initialization)", "PetscDeviceView()", *view, view, flag)); 187 PetscFunctionReturn(0); 188 } 189 190 template <typename D> 191 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceBasic(PetscOptionItems *PetscOptionsObject, std::pair<PetscDeviceInitType, PetscBool> &initType, std::pair<PetscInt, PetscBool> &initId, std::pair<PetscBool, PetscBool> &initView) noexcept { 192 PetscFunctionBegin; 193 PetscCall(PetscOptionDeviceInitialize(PetscOptionsObject, &initType.first, &initType.second)); 194 PetscCall(PetscOptionDeviceSelect(PetscOptionsObject, &initId.first, &initId.second)); 195 PetscCall(PetscOptionDeviceView(PetscOptionsObject, &initView.first, &initView.second)); 196 PetscFunctionReturn(0); 197 } 198 199 template <typename D> 200 inline PetscErrorCode DeviceBase<D>::PetscOptionDeviceAll(MPI_Comm comm, std::pair<PetscDeviceInitType, PetscBool> &initType, std::pair<PetscInt, PetscBool> &initId, std::pair<PetscBool, PetscBool> &initView) noexcept { 201 constexpr char optname[] = "PetscDevice %s Options"; 202 constexpr auto dtype = PETSC_DEVICE_IMPL(); 203 const auto implname = PetscDeviceTypes[dtype]; 204 auto buf = std::array<char, 128>{}; 205 constexpr auto buflen = buf.size() - 1; // -1 to leave room for null 206 207 PetscFunctionBegin; 208 if (PetscDefined(USE_DEBUG)) { 209 // -3 since '%s' is replaced and dont count null char for optname 210 const auto len = std::strlen(implname) + PETSC_STATIC_ARRAY_LENGTH(optname) - 3; 211 212 PetscCheck(len < buflen, comm, PETSC_ERR_PLIB, "char buffer is not large enough to hold 'PetscDevice %s Options'; have %zu need %zu", implname, buflen, len); 213 } 214 PetscCall(PetscSNPrintf(buf.data(), buflen, optname, implname)); 215 PetscOptionsBegin(comm, nullptr, buf.data(), "Sys"); 216 PetscCall(PetscOptionDeviceBasic(PetscOptionsObject, initType, initId, initView)); 217 PetscOptionsEnd(); 218 PetscFunctionReturn(0); 219 } 220 221 } // namespace impl 222 223 } // namespace device 224 225 } // namespace Petsc 226 227 #define PETSC_DEVICE_IMPL_BASE_CLASS_HEADER(base_name, T) \ 228 using base_name = ::Petsc::device::impl::DeviceBase<T>; \ 229 friend base_name; \ 230 using base_name::base_name 231 232 #endif // __cplusplus 233 234 #endif // IMPLDEVICEBASE_HPP 235