1 #if !defined(PETSCDEVICETYPES_H) 2 #define PETSCDEVICETYPES_H 3 4 /* for PETSC_HAVE_CUDA/HIP/KOKKOS etc */ 5 #include <petscsys.h> /*I petscsys.h I*/ 6 7 /*E 8 PetscMemType - Memory type of a pointer 9 10 Developer Note: 11 Encoding of the bitmask in binary: xxxxyyyz 12 13 $ z = 0 - Host memory 14 $ z = 1 - Device memory 15 $ yyy = 000 - CUDA-related memory 16 $ yyy = 001 - HIP-related memory 17 $ yyy = 010 - SYCL-related memory 18 $ xxxxyyy1 = 0000,0001 - CUDA memory 19 $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory 20 $ xxxxyyy1 = 0000,0011 - HIP memory 21 $ xxxxyyy1 = 0000,0101 - SYCL memory 22 23 Other types of memory, e.g., CUDA managed memory, can be added when needed. 24 25 Level: beginner 26 27 .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin() 28 E*/ 29 typedef enum { 30 PETSC_MEMTYPE_HOST = 0, 31 PETSC_MEMTYPE_DEVICE = 0x01, 32 PETSC_MEMTYPE_CUDA = 0x01, 33 PETSC_MEMTYPE_NVSHMEM = 0x11, 34 PETSC_MEMTYPE_HIP = 0x03, 35 PETSC_MEMTYPE_SYCL = 0x05 36 } PetscMemType; 37 38 #define PetscMemTypeHost(m) (((m) & 0x1) == PETSC_MEMTYPE_HOST) 39 #define PetscMemTypeDevice(m) (((m) & 0x1) == PETSC_MEMTYPE_DEVICE) 40 #define PetscMemTypeCUDA(m) (((m) & 0xF) == PETSC_MEMTYPE_CUDA) 41 #define PetscMemTypeHIP(m) (((m) & 0xF) == PETSC_MEMTYPE_HIP) 42 #define PetscMemTypeSYCL(m) (((m) & 0xF) == PETSC_MEMTYPE_SYCL) 43 #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM) 44 45 /*E 46 PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data 47 48 $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors 49 $ PETSC_OFFLOAD_GPU - GPU has valid vector/matrix entries 50 $ PETSC_OFFLOAD_CPU - CPU has valid vector/matrix entries 51 $ PETSC_OFFLOAD_BOTH - Both GPU and CPU have valid vector/matrix entries and they match 52 $ PETSC_OFFLOAD_KOKKOS - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is. 53 54 Level: developer 55 E*/ 56 #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)") 57 typedef enum { 58 PETSC_OFFLOAD_UNALLOCATED = 0x0, 59 PETSC_OFFLOAD_CPU = 0x1, 60 PETSC_OFFLOAD_GPU = 0x2, 61 PETSC_OFFLOAD_BOTH = 0x3, 62 PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100, 63 PETSC_OFFLOAD_KOKKOS = 0x100 64 } PetscOffloadMask; 65 66 /*E 67 PetscDeviceInitType - Initialization strategy for PetscDevice 68 69 $ PETSC_DEVICE_INIT_NONE - PetscDevice is never initialized 70 $ PETSC_DEVICE_INIT_LAZY - PetscDevice is initialized on demand 71 $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible 72 73 Notes: 74 PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and 75 doing so results in an error. Useful to ensure that no accelerator is used in a program. 76 77 Level: beginner 78 79 .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate() 80 E*/ 81 typedef enum { 82 PETSC_DEVICE_INIT_NONE, 83 PETSC_DEVICE_INIT_LAZY, 84 PETSC_DEVICE_INIT_EAGER 85 } PetscDeviceInitType; 86 PETSC_EXTERN const char *const PetscDeviceInitTypes[]; 87 88 /*E 89 PetscDeviceType - Kind of accelerator device backend 90 91 $ PETSC_DEVICE_INVALID - Invalid type, do not use 92 $ PETSC_DEVICE_CUDA - CUDA enabled GPU 93 $ PETSC_DEVICE_HIP - ROCM/HIP enabled GPU 94 $ PETSC_DEVICE_SYCL - SYCL enabled device 95 $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability 96 $ PETSC_DEVICE_MAX - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use 97 98 Notes: 99 PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_SYCL, PETSC_DEVICE_INVALID. 100 101 Level: beginner 102 103 .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate() 104 E*/ 105 typedef enum { 106 PETSC_DEVICE_INVALID, 107 PETSC_DEVICE_CUDA, 108 PETSC_DEVICE_HIP, 109 PETSC_DEVICE_SYCL, 110 PETSC_DEVICE_MAX 111 } PetscDeviceType; 112 PETSC_EXTERN const char *const PetscDeviceTypes[]; 113 #if PetscDefined(HAVE_HIP) 114 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP 115 #elif PetscDefined(HAVE_CUDA) 116 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA 117 #elif PetscDefined(HAVE_SYCL) 118 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_SYCL 119 #else 120 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID 121 #endif 122 123 /*S 124 PetscDevice - Handle to an accelerator "device" (usually a GPU) 125 126 Notes: 127 This object is used to house configuration and state of a device, but does not offer any ability to interact with or 128 drive device computation. This functionality is facilitated instead by the PetscDeviceContext object. 129 130 Level: beginner 131 132 .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice() 133 S*/ 134 typedef struct _n_PetscDevice *PetscDevice; 135 136 /*E 137 PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL" 138 stream, which is usually blocking. 139 140 $ PETSC_STREAM_GLOBAL_BLOCKING - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations. 141 $ PETSC_STREAM_DEFAULT_BLOCKING - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream. 142 $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream. 143 $ PETSC_STREAM_MAX - Always 1 greater than the largest PetscStreamType, do not use 144 145 Level: intermediate 146 147 .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType() 148 E*/ 149 typedef enum { 150 PETSC_STREAM_GLOBAL_BLOCKING, 151 PETSC_STREAM_DEFAULT_BLOCKING, 152 PETSC_STREAM_GLOBAL_NONBLOCKING, 153 PETSC_STREAM_MAX 154 } PetscStreamType; 155 PETSC_EXTERN const char *const PetscStreamTypes[]; 156 157 /*E 158 PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin() 159 160 $ PETSC_DEVICE_CONTEXT_DESTROY - Destroy all incoming sub-contexts after join. 161 $ PETSC_CONTEXT_JOIN_SYNC - Synchronize incoming sub-contexts after join. 162 $ PETSC_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join. 163 164 Level: beginner 165 166 .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin() 167 E*/ 168 typedef enum { 169 PETSC_DEVICE_CONTEXT_JOIN_DESTROY, 170 PETSC_DEVICE_CONTEXT_JOIN_SYNC, 171 PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC 172 } PetscDeviceContextJoinMode; 173 PETSC_EXTERN const char *const PetscDeviceContextJoinModes[]; 174 175 /*S 176 PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute. 177 178 Level: beginner 179 180 .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(), 181 PetscDeviceContextFork(), PetscDeviceContextJoin() 182 S*/ 183 typedef struct _n_PetscDeviceContext *PetscDeviceContext; 184 #endif /* PETSCDEVICETYPES_H */ 185