1 #ifndef PETSCDEVICETYPES_H 2 #define PETSCDEVICETYPES_H 3 4 #include <petscmacros.h> /*I <petscdevicetypes.h> I*/ 5 6 /* SUBMANSEC = Sys */ 7 8 /*E 9 PetscMemType - Memory type of a pointer 10 11 Developer Note: 12 Encoding of the bitmask in binary: xxxxyyyz 13 14 $ z = 0 - Host memory 15 $ z = 1 - Device memory 16 $ yyy = 000 - CUDA-related memory 17 $ yyy = 001 - HIP-related memory 18 $ yyy = 010 - SYCL-related memory 19 $ xxxxyyy1 = 0000,0001 - CUDA memory 20 $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory 21 $ xxxxyyy1 = 0000,0011 - HIP memory 22 $ xxxxyyy1 = 0000,0101 - SYCL memory 23 24 Other types of memory, e.g., CUDA managed memory, can be added when needed. 25 26 Level: beginner 27 28 Notes: 29 `PETSC_MEMTYPE_KOKKOS` depends on the Kokkos backend configuration 30 31 .seealso: `VecGetArrayAndMemType()`, `PetscSFBcastWithMemTypeBegin()`, `PetscSFReduceWithMemTypeBegin()` 32 E*/ 33 typedef enum { 34 PETSC_MEMTYPE_HOST = 0, 35 PETSC_MEMTYPE_DEVICE = 0x01, 36 PETSC_MEMTYPE_CUDA = 0x01, 37 PETSC_MEMTYPE_NVSHMEM = 0x11, 38 PETSC_MEMTYPE_HIP = 0x03, 39 PETSC_MEMTYPE_SYCL = 0x05, 40 } PetscMemType; 41 #if PetscDefined(HAVE_CUDA) 42 #define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_CUDA 43 #elif PetscDefined(HAVE_HIP) 44 #define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HIP 45 #elif PetscDefined(HAVE_SYCL) 46 #define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_SYCL 47 #else 48 #define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HOST 49 #endif 50 51 #define PetscMemTypeHost(m) (((m)&0x1) == PETSC_MEMTYPE_HOST) 52 #define PetscMemTypeDevice(m) (((m)&0x1) == PETSC_MEMTYPE_DEVICE) 53 #define PetscMemTypeCUDA(m) (((m)&0xF) == PETSC_MEMTYPE_CUDA) 54 #define PetscMemTypeHIP(m) (((m)&0xF) == PETSC_MEMTYPE_HIP) 55 #define PetscMemTypeSYCL(m) (((m)&0xF) == PETSC_MEMTYPE_SYCL) 56 #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM) 57 58 #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)") 59 /*E 60 PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data 61 62 $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors 63 $ PETSC_OFFLOAD_GPU - GPU has valid vector/matrix entries 64 $ PETSC_OFFLOAD_CPU - CPU has valid vector/matrix entries 65 $ PETSC_OFFLOAD_BOTH - Both GPU and CPU have valid vector/matrix entries and they match 66 $ PETSC_OFFLOAD_KOKKOS - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is. 67 68 Level: developer 69 E*/ 70 typedef enum { 71 PETSC_OFFLOAD_UNALLOCATED = 0x0, 72 PETSC_OFFLOAD_CPU = 0x1, 73 PETSC_OFFLOAD_GPU = 0x2, 74 PETSC_OFFLOAD_BOTH = 0x3, 75 PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100, 76 PETSC_OFFLOAD_KOKKOS = 0x100 77 } PetscOffloadMask; 78 79 /*E 80 PetscDeviceInitType - Initialization strategy for `PetscDevice` 81 82 $ PETSC_DEVICE_INIT_NONE - PetscDevice is never initialized 83 $ PETSC_DEVICE_INIT_LAZY - PetscDevice is initialized on demand 84 $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible 85 86 Notes: 87 PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and 88 doing so results in an error. Useful to ensure that no accelerator is used in a program. 89 90 Level: beginner 91 92 .seealso: `PetscDevice`, `PetscDeviceType`, `PetscDeviceInitialize()`, `PetscDeviceInitialized()`, `PetscDeviceCreate()` 93 E*/ 94 typedef enum { 95 PETSC_DEVICE_INIT_NONE, 96 PETSC_DEVICE_INIT_LAZY, 97 PETSC_DEVICE_INIT_EAGER 98 } PetscDeviceInitType; 99 PETSC_EXTERN const char *const PetscDeviceInitTypes[]; 100 101 /*E 102 PetscDeviceType - Kind of accelerator device backend 103 104 $ PETSC_DEVICE_INVALID - Invalid type, do not use 105 $ PETSC_DEVICE_CUDA - CUDA enabled GPU 106 $ PETSC_DEVICE_HIP - ROCM/HIP enabled GPU 107 $ PETSC_DEVICE_SYCL - SYCL enabled device 108 $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability 109 $ PETSC_DEVICE_MAX - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use 110 111 Notes: 112 PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID. 113 114 Level: beginner 115 116 .seealso: `PetscDevice`, `PetscDeviceInitType`, `PetscDeviceCreate()` 117 E*/ 118 typedef enum { 119 PETSC_DEVICE_INVALID, 120 PETSC_DEVICE_CUDA, 121 PETSC_DEVICE_HIP, 122 PETSC_DEVICE_SYCL, 123 PETSC_DEVICE_MAX 124 } PetscDeviceType; 125 PETSC_EXTERN const char *const PetscDeviceTypes[]; 126 #if defined(PETSC_HAVE_HIP) 127 #define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP 128 #elif defined(PETSC_HAVE_CUDA) 129 #define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA 130 #elif PetscDefined(HAVE_SYCL) 131 #define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL 132 #else 133 #define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID 134 #endif 135 136 /*S 137 PetscDevice - Object to manage an accelerator "device" (usually a GPU) 138 139 Note: 140 This object is used to house configuration and state of a device, but does not offer any ability to interact with or 141 drive device computation. This functionality is facilitated instead by the `PetscDeviceContext` object. 142 143 Level: beginner 144 145 .seealso: `PetscDeviceType`, `PetscDeviceInitType`, `PetscDeviceCreate()`, `PetscDeviceConfigure()`, `PetscDeviceDestroy()`, `PetscDeviceContext`, `PetscDeviceContextSetDevice()`, `PetscDeviceContextGetDevice()` 146 S*/ 147 typedef struct _n_PetscDevice *PetscDevice; 148 149 /*E 150 PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL" 151 stream, which is usually blocking. 152 153 $ PETSC_STREAM_GLOBAL_BLOCKING - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations. 154 $ PETSC_STREAM_DEFAULT_BLOCKING - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream. 155 $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream. 156 $ PETSC_STREAM_MAX - Always 1 greater than the largest PetscStreamType, do not use 157 158 Level: intermediate 159 160 .seealso: `PetscDeviceContextSetStreamType()`, `PetscDeviceContextGetStreamType()` 161 E*/ 162 typedef enum { 163 PETSC_STREAM_GLOBAL_BLOCKING, 164 PETSC_STREAM_DEFAULT_BLOCKING, 165 PETSC_STREAM_GLOBAL_NONBLOCKING, 166 PETSC_STREAM_MAX 167 } PetscStreamType; 168 PETSC_EXTERN const char *const PetscStreamTypes[]; 169 170 /*E 171 PetscDeviceContextJoinMode - Describes the type of join operation to perform in `PetscDeviceContextJoin()` 172 173 $ PETSC_DEVICE_CONTEXT_JOIN_DESTROY - Destroy all incoming sub-contexts after join. 174 $ PETSC_DEVICE_CONTEXT_JOIN_SYNC - Synchronize incoming sub-contexts after join. 175 $ PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join. 176 177 Level: beginner 178 179 .seealso: `PetscDeviceContext`, `PetscDeviceContextFork()`, `PetscDeviceContextJoin()` 180 E*/ 181 typedef enum { 182 PETSC_DEVICE_CONTEXT_JOIN_DESTROY, 183 PETSC_DEVICE_CONTEXT_JOIN_SYNC, 184 PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC 185 } PetscDeviceContextJoinMode; 186 PETSC_EXTERN const char *const PetscDeviceContextJoinModes[]; 187 188 /*S 189 PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute. 190 191 Level: beginner 192 193 .seealso: `PetscDevice`, `PetscDeviceContextCreate()`, `PetscDeviceContextSetDevice()`, `PetscDeviceContextDestroy()`, 194 `PetscDeviceContextFork()`, `PetscDeviceContextJoin()` 195 S*/ 196 typedef struct _n_PetscDeviceContext *PetscDeviceContext; 197 #endif /* PETSCDEVICETYPES_H */ 198