1 #ifndef PETSCDEVICETYPES_H 2 #define PETSCDEVICETYPES_H 3 4 /* for PETSC_HAVE_CUDA/HIP/KOKKOS etc */ 5 #include <petscsys.h> /*I petscsys.h I*/ 6 7 /*E 8 PetscMemType - Memory type of a pointer 9 10 Developer Note: 11 Encoding of the bitmask in binary: xxxxyyyz 12 13 $ z = 0 - Host memory 14 $ z = 1 - Device memory 15 $ yyy = 000 - CUDA-related memory 16 $ yyy = 001 - HIP-related memory 17 $ yyy = 010 - SYCL-related memory 18 $ xxxxyyy1 = 0000,0001 - CUDA memory 19 $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory 20 $ xxxxyyy1 = 0000,0011 - HIP memory 21 $ xxxxyyy1 = 0000,0101 - SYCL memory 22 23 Other types of memory, e.g., CUDA managed memory, can be added when needed. 24 25 Level: beginner 26 27 Notes: 28 PETSC_MEMTYPE_KOKKOS depends on the KOKKOS backend configuration 29 30 .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin() 31 E*/ 32 typedef enum { 33 PETSC_MEMTYPE_HOST = 0, 34 PETSC_MEMTYPE_DEVICE = 0x01, 35 PETSC_MEMTYPE_CUDA = 0x01, 36 PETSC_MEMTYPE_NVSHMEM = 0x11, 37 PETSC_MEMTYPE_HIP = 0x03, 38 PETSC_MEMTYPE_SYCL = 0x05, 39 #if PetscDefined(HAVE_CUDA) 40 PETSC_MEMTYPE_KOKKOS = PETSC_MEMTYPE_CUDA 41 #elif PetscDefined(HAVE_HIP) 42 PETSC_MEMTYPE_KOKKOS = PETSC_MEMTYPE_HIP 43 #elif PetscDefined(HAVE_SYCL) 44 PETSC_MEMTYPE_KOKKOS = PETSC_MEMTYPE_SYCL 45 #else 46 PETSC_MEMTYPE_KOKKOS = PETSC_MEMTYPE_HOST 47 #endif 48 } PetscMemType; 49 50 #define PetscMemTypeHost(m) (((m) & 0x1) == PETSC_MEMTYPE_HOST) 51 #define PetscMemTypeDevice(m) (((m) & 0x1) == PETSC_MEMTYPE_DEVICE) 52 #define PetscMemTypeCUDA(m) (((m) & 0xF) == PETSC_MEMTYPE_CUDA) 53 #define PetscMemTypeHIP(m) (((m) & 0xF) == PETSC_MEMTYPE_HIP) 54 #define PetscMemTypeSYCL(m) (((m) & 0xF) == PETSC_MEMTYPE_SYCL) 55 #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM) 56 57 #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)") 58 /*E 59 PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data 60 61 $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors 62 $ PETSC_OFFLOAD_GPU - GPU has valid vector/matrix entries 63 $ PETSC_OFFLOAD_CPU - CPU has valid vector/matrix entries 64 $ PETSC_OFFLOAD_BOTH - Both GPU and CPU have valid vector/matrix entries and they match 65 $ PETSC_OFFLOAD_KOKKOS - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is. 66 67 Level: developer 68 E*/ 69 typedef enum { 70 PETSC_OFFLOAD_UNALLOCATED = 0x0, 71 PETSC_OFFLOAD_CPU = 0x1, 72 PETSC_OFFLOAD_GPU = 0x2, 73 PETSC_OFFLOAD_BOTH = 0x3, 74 PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100, 75 PETSC_OFFLOAD_KOKKOS = 0x100 76 } PetscOffloadMask; 77 78 /*E 79 PetscDeviceInitType - Initialization strategy for PetscDevice 80 81 $ PETSC_DEVICE_INIT_NONE - PetscDevice is never initialized 82 $ PETSC_DEVICE_INIT_LAZY - PetscDevice is initialized on demand 83 $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible 84 85 Notes: 86 PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and 87 doing so results in an error. Useful to ensure that no accelerator is used in a program. 88 89 Level: beginner 90 91 .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate() 92 E*/ 93 typedef enum { 94 PETSC_DEVICE_INIT_NONE, 95 PETSC_DEVICE_INIT_LAZY, 96 PETSC_DEVICE_INIT_EAGER 97 } PetscDeviceInitType; 98 PETSC_EXTERN const char *const PetscDeviceInitTypes[]; 99 100 /*E 101 PetscDeviceType - Kind of accelerator device backend 102 103 $ PETSC_DEVICE_INVALID - Invalid type, do not use 104 $ PETSC_DEVICE_CUDA - CUDA enabled GPU 105 $ PETSC_DEVICE_HIP - ROCM/HIP enabled GPU 106 $ PETSC_DEVICE_SYCL - SYCL enabled device 107 $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability 108 $ PETSC_DEVICE_MAX - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use 109 110 Notes: 111 PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID. 112 113 Level: beginner 114 115 .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate() 116 E*/ 117 typedef enum { 118 PETSC_DEVICE_INVALID, 119 PETSC_DEVICE_CUDA, 120 PETSC_DEVICE_HIP, 121 PETSC_DEVICE_SYCL, 122 PETSC_DEVICE_MAX 123 } PetscDeviceType; 124 PETSC_EXTERN const char *const PetscDeviceTypes[]; 125 #if defined(PETSC_HAVE_HIP) 126 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP 127 #elif defined(PETSC_HAVE_CUDA) 128 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA 129 #elif PetscDefined(HAVE_SYCL) 130 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL 131 #else 132 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID 133 #endif 134 135 /*S 136 PetscDevice - Handle to an accelerator "device" (usually a GPU) 137 138 Notes: 139 This object is used to house configuration and state of a device, but does not offer any ability to interact with or 140 drive device computation. This functionality is facilitated instead by the PetscDeviceContext object. 141 142 Level: beginner 143 144 .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice() 145 S*/ 146 typedef struct _n_PetscDevice *PetscDevice; 147 148 /*E 149 PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL" 150 stream, which is usually blocking. 151 152 $ PETSC_STREAM_GLOBAL_BLOCKING - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations. 153 $ PETSC_STREAM_DEFAULT_BLOCKING - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream. 154 $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream. 155 $ PETSC_STREAM_MAX - Always 1 greater than the largest PetscStreamType, do not use 156 157 Level: intermediate 158 159 .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType() 160 E*/ 161 typedef enum { 162 PETSC_STREAM_GLOBAL_BLOCKING, 163 PETSC_STREAM_DEFAULT_BLOCKING, 164 PETSC_STREAM_GLOBAL_NONBLOCKING, 165 PETSC_STREAM_MAX 166 } PetscStreamType; 167 PETSC_EXTERN const char *const PetscStreamTypes[]; 168 169 /*E 170 PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin() 171 172 $ PETSC_DEVICE_CONTEXT_DESTROY - Destroy all incoming sub-contexts after join. 173 $ PETSC_CONTEXT_JOIN_SYNC - Synchronize incoming sub-contexts after join. 174 $ PETSC_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join. 175 176 Level: beginner 177 178 .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin() 179 E*/ 180 typedef enum { 181 PETSC_DEVICE_CONTEXT_JOIN_DESTROY, 182 PETSC_DEVICE_CONTEXT_JOIN_SYNC, 183 PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC 184 } PetscDeviceContextJoinMode; 185 PETSC_EXTERN const char *const PetscDeviceContextJoinModes[]; 186 187 /*S 188 PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute. 189 190 Level: beginner 191 192 .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(), 193 PetscDeviceContextFork(), PetscDeviceContextJoin() 194 S*/ 195 typedef struct _n_PetscDeviceContext *PetscDeviceContext; 196 #endif /* PETSCDEVICETYPES_H */ 197