1 #ifndef PETSCDEVICETYPES_H 2 #define PETSCDEVICETYPES_H 3 4 #include <petscmacros.h> /*I <petscdevicetypes.h> I*/ 5 6 /*E 7 PetscMemType - Memory type of a pointer 8 9 Developer Note: 10 Encoding of the bitmask in binary: xxxxyyyz 11 12 $ z = 0 - Host memory 13 $ z = 1 - Device memory 14 $ yyy = 000 - CUDA-related memory 15 $ yyy = 001 - HIP-related memory 16 $ yyy = 010 - SYCL-related memory 17 $ xxxxyyy1 = 0000,0001 - CUDA memory 18 $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory 19 $ xxxxyyy1 = 0000,0011 - HIP memory 20 $ xxxxyyy1 = 0000,0101 - SYCL memory 21 22 Other types of memory, e.g., CUDA managed memory, can be added when needed. 23 24 Level: beginner 25 26 Notes: 27 PETSC_MEMTYPE_KOKKOS depends on the KOKKOS backend configuration 28 29 .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin() 30 E*/ 31 typedef enum { 32 PETSC_MEMTYPE_HOST = 0, 33 PETSC_MEMTYPE_DEVICE = 0x01, 34 PETSC_MEMTYPE_CUDA = 0x01, 35 PETSC_MEMTYPE_NVSHMEM = 0x11, 36 PETSC_MEMTYPE_HIP = 0x03, 37 PETSC_MEMTYPE_SYCL = 0x05, 38 } PetscMemType; 39 #if PetscDefined(HAVE_CUDA) 40 # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_CUDA 41 #elif PetscDefined(HAVE_HIP) 42 # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HIP 43 #elif PetscDefined(HAVE_SYCL) 44 # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_SYCL 45 #else 46 # define PETSC_MEMTYPE_KOKKOS PETSC_MEMTYPE_HOST 47 #endif 48 49 #define PetscMemTypeHost(m) (((m) & 0x1) == PETSC_MEMTYPE_HOST) 50 #define PetscMemTypeDevice(m) (((m) & 0x1) == PETSC_MEMTYPE_DEVICE) 51 #define PetscMemTypeCUDA(m) (((m) & 0xF) == PETSC_MEMTYPE_CUDA) 52 #define PetscMemTypeHIP(m) (((m) & 0xF) == PETSC_MEMTYPE_HIP) 53 #define PetscMemTypeSYCL(m) (((m) & 0xF) == PETSC_MEMTYPE_SYCL) 54 #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM) 55 56 #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)") 57 /*E 58 PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data 59 60 $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors 61 $ PETSC_OFFLOAD_GPU - GPU has valid vector/matrix entries 62 $ PETSC_OFFLOAD_CPU - CPU has valid vector/matrix entries 63 $ PETSC_OFFLOAD_BOTH - Both GPU and CPU have valid vector/matrix entries and they match 64 $ PETSC_OFFLOAD_KOKKOS - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is. 65 66 Level: developer 67 E*/ 68 typedef enum { 69 PETSC_OFFLOAD_UNALLOCATED = 0x0, 70 PETSC_OFFLOAD_CPU = 0x1, 71 PETSC_OFFLOAD_GPU = 0x2, 72 PETSC_OFFLOAD_BOTH = 0x3, 73 PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100, 74 PETSC_OFFLOAD_KOKKOS = 0x100 75 } PetscOffloadMask; 76 77 /*E 78 PetscDeviceInitType - Initialization strategy for PetscDevice 79 80 $ PETSC_DEVICE_INIT_NONE - PetscDevice is never initialized 81 $ PETSC_DEVICE_INIT_LAZY - PetscDevice is initialized on demand 82 $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible 83 84 Notes: 85 PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and 86 doing so results in an error. Useful to ensure that no accelerator is used in a program. 87 88 Level: beginner 89 90 .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate() 91 E*/ 92 typedef enum { 93 PETSC_DEVICE_INIT_NONE, 94 PETSC_DEVICE_INIT_LAZY, 95 PETSC_DEVICE_INIT_EAGER 96 } PetscDeviceInitType; 97 PETSC_EXTERN const char *const PetscDeviceInitTypes[]; 98 99 /*E 100 PetscDeviceType - Kind of accelerator device backend 101 102 $ PETSC_DEVICE_INVALID - Invalid type, do not use 103 $ PETSC_DEVICE_CUDA - CUDA enabled GPU 104 $ PETSC_DEVICE_HIP - ROCM/HIP enabled GPU 105 $ PETSC_DEVICE_SYCL - SYCL enabled device 106 $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability 107 $ PETSC_DEVICE_MAX - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use 108 109 Notes: 110 PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID. 111 112 Level: beginner 113 114 .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate() 115 E*/ 116 typedef enum { 117 PETSC_DEVICE_INVALID, 118 PETSC_DEVICE_CUDA, 119 PETSC_DEVICE_HIP, 120 PETSC_DEVICE_SYCL, 121 PETSC_DEVICE_MAX 122 } PetscDeviceType; 123 PETSC_EXTERN const char *const PetscDeviceTypes[]; 124 #if defined(PETSC_HAVE_HIP) 125 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP 126 #elif defined(PETSC_HAVE_CUDA) 127 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA 128 #elif PetscDefined(HAVE_SYCL) 129 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL 130 #else 131 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID 132 #endif 133 134 /*S 135 PetscDevice - Handle to an accelerator "device" (usually a GPU) 136 137 Notes: 138 This object is used to house configuration and state of a device, but does not offer any ability to interact with or 139 drive device computation. This functionality is facilitated instead by the PetscDeviceContext object. 140 141 Level: beginner 142 143 .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice() 144 S*/ 145 typedef struct _n_PetscDevice *PetscDevice; 146 147 /*E 148 PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL" 149 stream, which is usually blocking. 150 151 $ PETSC_STREAM_GLOBAL_BLOCKING - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations. 152 $ PETSC_STREAM_DEFAULT_BLOCKING - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream. 153 $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream. 154 $ PETSC_STREAM_MAX - Always 1 greater than the largest PetscStreamType, do not use 155 156 Level: intermediate 157 158 .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType() 159 E*/ 160 typedef enum { 161 PETSC_STREAM_GLOBAL_BLOCKING, 162 PETSC_STREAM_DEFAULT_BLOCKING, 163 PETSC_STREAM_GLOBAL_NONBLOCKING, 164 PETSC_STREAM_MAX 165 } PetscStreamType; 166 PETSC_EXTERN const char *const PetscStreamTypes[]; 167 168 /*E 169 PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin() 170 171 $ PETSC_DEVICE_CONTEXT_JOIN_DESTROY - Destroy all incoming sub-contexts after join. 172 $ PETSC_DEVICE_CONTEXT_JOIN_SYNC - Synchronize incoming sub-contexts after join. 173 $ PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join. 174 175 Level: beginner 176 177 .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin() 178 E*/ 179 typedef enum { 180 PETSC_DEVICE_CONTEXT_JOIN_DESTROY, 181 PETSC_DEVICE_CONTEXT_JOIN_SYNC, 182 PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC 183 } PetscDeviceContextJoinMode; 184 PETSC_EXTERN const char *const PetscDeviceContextJoinModes[]; 185 186 /*S 187 PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute. 188 189 Level: beginner 190 191 .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(), 192 PetscDeviceContextFork(), PetscDeviceContextJoin() 193 S*/ 194 typedef struct _n_PetscDeviceContext *PetscDeviceContext; 195 #endif /* PETSCDEVICETYPES_H */ 196