1 #if !defined(PETSCDEVICETYPES_H) 2 #define PETSCDEVICETYPES_H 3 4 /* for PETSC_HAVE_CUDA/HIP/KOKKOS etc */ 5 #include <petscsys.h> /*I petscsys.h I*/ 6 7 /*E 8 PetscMemType - Memory type of a pointer 9 10 Developer Note: 11 Encoding of the bitmask in binary: xxxxyyyz 12 13 $ z = 0 - Host memory 14 $ z = 1 - Device memory 15 $ yyy = 000 - CUDA-related memory 16 $ yyy = 001 - HIP-related memory 17 $ xxxxyyy1 = 0000,0001 - CUDA memory 18 $ xxxxyyy1 = 0001,0001 - CUDA NVSHMEM memory 19 $ xxxxyyy1 = 0000,0011 - HIP memory 20 21 Other types of memory, e.g., CUDA managed memory, can be added when needed. 22 23 Level: beginner 24 25 .seealso: VecGetArrayAndMemType(), PetscSFBcastWithMemTypeBegin(), PetscSFReduceWithMemTypeBegin() 26 E*/ 27 typedef enum { 28 PETSC_MEMTYPE_HOST = 0, 29 PETSC_MEMTYPE_DEVICE = 0x01, 30 PETSC_MEMTYPE_CUDA = 0x01, 31 PETSC_MEMTYPE_NVSHMEM = 0x11, 32 PETSC_MEMTYPE_HIP = 0x03 33 } PetscMemType; 34 35 #define PetscMemTypeHost(m) (((m) & 0x1) == PETSC_MEMTYPE_HOST) 36 #define PetscMemTypeDevice(m) (((m) & 0x1) == PETSC_MEMTYPE_DEVICE) 37 #define PetscMemTypeCUDA(m) (((m) & 0xF) == PETSC_MEMTYPE_CUDA) 38 #define PetscMemTypeHIP(m) (((m) & 0xF) == PETSC_MEMTYPE_HIP) 39 #define PetscMemTypeNVSHMEM(m) ((m) == PETSC_MEMTYPE_NVSHMEM) 40 41 /*E 42 PetscOffloadMask - indicates which memory (CPU, GPU, or none) contains valid data 43 44 $ PETSC_OFFLOAD_UNALLOCATED - no memory contains valid matrix entries; NEVER used for vectors 45 $ PETSC_OFFLOAD_GPU - GPU has valid vector/matrix entries 46 $ PETSC_OFFLOAD_CPU - CPU has valid vector/matrix entries 47 $ PETSC_OFFLOAD_BOTH - Both GPU and CPU have valid vector/matrix entries and they match 48 $ PETSC_OFFLOAD_KOKKOS - Reserved for Kokkos matrix and vector. It means the offload is managed by Kokkos, thus this flag itself cannot tell you where the valid data is. 49 50 Level: developer 51 E*/ 52 #define PETSC_OFFLOAD_VECKOKKOS_DEPRECATED PETSC_OFFLOAD_VECKOKKOS PETSC_DEPRECATED_ENUM("Use PETSC_OFFLOAD_KOKKOS (since version 3.17.0)") 53 typedef enum { 54 PETSC_OFFLOAD_UNALLOCATED = 0x0, 55 PETSC_OFFLOAD_CPU = 0x1, 56 PETSC_OFFLOAD_GPU = 0x2, 57 PETSC_OFFLOAD_BOTH = 0x3, 58 PETSC_OFFLOAD_VECKOKKOS_DEPRECATED = 0x100, 59 PETSC_OFFLOAD_KOKKOS = 0x100 60 } PetscOffloadMask; 61 62 /*E 63 PetscDeviceInitType - Initialization strategy for PetscDevice 64 65 $ PETSC_DEVICE_INIT_NONE - PetscDevice is never initialized 66 $ PETSC_DEVICE_INIT_LAZY - PetscDevice is initialized on demand 67 $ PETSC_DEVICE_INIT_EAGER - PetscDevice is initialized as soon as possible 68 69 Notes: 70 PETSC_DEVICE_INIT_NONE implies that any initialization of PetscDevice is disallowed and 71 doing so results in an error. Useful to ensure that no accelerator is used in a program. 72 73 Level: beginner 74 75 .seealso: PetscDevice, PetscDeviceType, PetscDeviceInitialize(), PetscDeviceInitialized(), PetscDeviceCreate() 76 E*/ 77 typedef enum { 78 PETSC_DEVICE_INIT_NONE, 79 PETSC_DEVICE_INIT_LAZY, 80 PETSC_DEVICE_INIT_EAGER 81 } PetscDeviceInitType; 82 PETSC_EXTERN const char *const PetscDeviceInitTypes[]; 83 84 /*E 85 PetscDeviceType - Kind of accelerator device backend 86 87 $ PETSC_DEVICE_INVALID - Invalid type, do not use 88 $ PETSC_DEVICE_CUDA - CUDA enabled GPU 89 $ PETSC_DEVICE_HIP - ROCM/HIP enabled GPU 90 $ PETSC_DEVICE_DEFAULT - Automatically select backend based on availability 91 $ PETSC_DEVICE_MAX - Always 1 greater than the largest valid PetscDeviceType, invalid type, do not use 92 93 Notes: 94 PETSC_DEVICE_DEFAULT is selected in the following order: PETSC_DEVICE_HIP, PETSC_DEVICE_CUDA, PETSC_DEVICE_INVALID. 95 96 Level: beginner 97 98 .seealso: PetscDevice, PetscDeviceInitType, PetscDeviceCreate() 99 E*/ 100 typedef enum { 101 PETSC_DEVICE_INVALID, 102 PETSC_DEVICE_CUDA, 103 PETSC_DEVICE_HIP, 104 PETSC_DEVICE_MAX 105 } PetscDeviceType; 106 PETSC_EXTERN const char *const PetscDeviceTypes[]; 107 #if PetscDefined(HAVE_HIP) 108 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_HIP 109 #elif PetscDefined(HAVE_CUDA) 110 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_CUDA 111 #else 112 # define PETSC_DEVICE_DEFAULT PETSC_DEVICE_INVALID 113 #endif 114 115 /*S 116 PetscDevice - Handle to an accelerator "device" (usually a GPU) 117 118 Notes: 119 This object is used to house configuration and state of a device, but does not offer any ability to interact with or 120 drive device computation. This functionality is facilitated instead by the PetscDeviceContext object. 121 122 Level: beginner 123 124 .seealso: PetscDeviceType, PetscDeviceInitType, PetscDeviceCreate(), PetscDeviceConfigure(), PetscDeviceDestroy(), PetscDeviceContext, PetscDeviceContextSetDevice(), PetscDeviceContextGetDevice() 125 S*/ 126 typedef struct _n_PetscDevice *PetscDevice; 127 128 /*E 129 PetscStreamType - Stream blocking mode, indicates how a stream implementation will interact with the default "NULL" 130 stream, which is usually blocking. 131 132 $ PETSC_STREAM_GLOBAL_BLOCKING - Alias for NULL stream. Any stream of this type will block the host for all other streams to finish work before starting its operations. 133 $ PETSC_STREAM_DEFAULT_BLOCKING - Stream will act independent of other streams, but will still be blocked by actions on the NULL stream. 134 $ PETSC_STREAM_GLOBAL_NONBLOCKING - Stream is truly asynchronous, and is blocked by nothing, not even the NULL stream. 135 $ PETSC_STREAM_MAX - Always 1 greater than the largest PetscStreamType, do not use 136 137 Level: intermediate 138 139 .seealso: PetscDeviceContextSetStreamType(), PetscDeviceContextGetStreamType() 140 E*/ 141 typedef enum { 142 PETSC_STREAM_GLOBAL_BLOCKING, 143 PETSC_STREAM_DEFAULT_BLOCKING, 144 PETSC_STREAM_GLOBAL_NONBLOCKING, 145 PETSC_STREAM_MAX 146 } PetscStreamType; 147 PETSC_EXTERN const char *const PetscStreamTypes[]; 148 149 /*E 150 PetscDeviceContextJoinMode - Describes the type of join operation to perform in PetscDeviceContextJoin() 151 152 $ PETSC_DEVICE_CONTEXT_DESTROY - Destroy all incoming sub-contexts after join. 153 $ PETSC_CONTEXT_JOIN_SYNC - Synchronize incoming sub-contexts after join. 154 $ PETSC_CONTEXT_JOIN_NO_SYNC - Do not synchronize incoming sub-contexts after join. 155 156 Level: beginner 157 158 .seealso: PetscDeviceContext, PetscDeviceContextFork(), PetscDeviceContextJoin() 159 E*/ 160 typedef enum { 161 PETSC_DEVICE_CONTEXT_JOIN_DESTROY, 162 PETSC_DEVICE_CONTEXT_JOIN_SYNC, 163 PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC 164 } PetscDeviceContextJoinMode; 165 PETSC_EXTERN const char *const PetscDeviceContextJoinModes[]; 166 167 /*S 168 PetscDeviceContext - Container to manage stream dependencies and the various solver handles for asynchronous device compute. 169 170 Level: beginner 171 172 .seealso: PetscDevice, PetscDeviceContextCreate(), PetscDeviceContextSetDevice(), PetscDeviceContextDestroy(), 173 PetscDeviceContextFork(), PetscDeviceContextJoin() 174 S*/ 175 typedef struct _n_PetscDeviceContext *PetscDeviceContext; 176 #endif /* PETSCDEVICETYPES_H */ 177