1 #pragma once
2
3 /*
4 This private file should not be included in users' code. Defines the fields shared by all
5 vector implementations.
6 */
7
8 #include <petscvec.h>
9 #include <petsc/private/petscimpl.h>
10
11 PETSC_INTERN PetscBool VecRegisterAllCalled;
12 PETSC_EXTERN MPI_Op MPIU_MAXLOC;
13 PETSC_EXTERN MPI_Op MPIU_MINLOC;
14
15 /* ----------------------------------------------------------------------------*/
16
17 typedef struct _VecOps *VecOps;
18 struct _VecOps {
19 PetscErrorCode (*duplicate)(Vec, Vec *); /* get single vector */
20 PetscErrorCode (*duplicatevecs)(Vec, PetscInt, Vec **); /* get array of vectors */
21 PetscErrorCode (*destroyvecs)(PetscInt, Vec[]); /* free array of vectors */
22 PetscErrorCode (*dot)(Vec, Vec, PetscScalar *); /* z = x^H * y */
23 PetscErrorCode (*mdot)(Vec, PetscInt, const Vec[], PetscScalar *); /* z[j] = x dot y[j] */
24 PetscErrorCode (*norm)(Vec, NormType, PetscReal *); /* z = sqrt(x^H * x) */
25 PetscErrorCode (*tdot)(Vec, Vec, PetscScalar *); /* x'*y */
26 PetscErrorCode (*mtdot)(Vec, PetscInt, const Vec[], PetscScalar *); /* z[j] = x dot y[j] */
27 PetscErrorCode (*scale)(Vec, PetscScalar); /* x = alpha * x */
28 PetscErrorCode (*copy)(Vec, Vec); /* y = x */
29 PetscErrorCode (*set)(Vec, PetscScalar); /* y = alpha */
30 PetscErrorCode (*swap)(Vec, Vec); /* exchange x and y */
31 PetscErrorCode (*axpy)(Vec, PetscScalar, Vec); /* y = y + alpha * x */
32 PetscErrorCode (*axpby)(Vec, PetscScalar, PetscScalar, Vec); /* y = alpha * x + beta * y*/
33 PetscErrorCode (*maxpy)(Vec, PetscInt, const PetscScalar *, Vec *); /* y = y + alpha[j] x[j] */
34 PetscErrorCode (*aypx)(Vec, PetscScalar, Vec); /* y = x + alpha * y */
35 PetscErrorCode (*waxpy)(Vec, PetscScalar, Vec, Vec); /* w = y + alpha * x */
36 PetscErrorCode (*axpbypcz)(Vec, PetscScalar, PetscScalar, PetscScalar, Vec, Vec); /* z = alpha * x + beta *y + gamma *z*/
37 PetscErrorCode (*pointwisemult)(Vec, Vec, Vec); /* w = x .* y */
38 PetscErrorCode (*pointwisedivide)(Vec, Vec, Vec); /* w = x ./ y */
39 PetscErrorCode (*setvalues)(Vec, PetscInt, const PetscInt[], const PetscScalar[], InsertMode);
40 PetscErrorCode (*assemblybegin)(Vec); /* start global assembly */
41 PetscErrorCode (*assemblyend)(Vec); /* end global assembly */
42 PetscErrorCode (*getarray)(Vec, PetscScalar **); /* get data array */
43 PetscErrorCode (*getsize)(Vec, PetscInt *);
44 PetscErrorCode (*getlocalsize)(Vec, PetscInt *);
45 PetscErrorCode (*restorearray)(Vec, PetscScalar **); /* restore data array */
46 PetscErrorCode (*max)(Vec, PetscInt *, PetscReal *); /* z = max(x); idx=index of max(x) */
47 PetscErrorCode (*min)(Vec, PetscInt *, PetscReal *); /* z = min(x); idx=index of min(x) */
48 PetscErrorCode (*setrandom)(Vec, PetscRandom); /* set y[j] = random numbers */
49 PetscErrorCode (*setoption)(Vec, VecOption, PetscBool);
50 PetscErrorCode (*setvaluesblocked)(Vec, PetscInt, const PetscInt[], const PetscScalar[], InsertMode);
51 PetscErrorCode (*destroy)(Vec);
52 PetscErrorCode (*view)(Vec, PetscViewer);
53 PetscErrorCode (*placearray)(Vec, const PetscScalar *); /* place data array */
54 PetscErrorCode (*replacearray)(Vec, const PetscScalar *); /* replace data array */
55 PetscErrorCode (*dot_local)(Vec, Vec, PetscScalar *);
56 PetscErrorCode (*tdot_local)(Vec, Vec, PetscScalar *);
57 PetscErrorCode (*norm_local)(Vec, NormType, PetscReal *);
58 PetscErrorCode (*mdot_local)(Vec, PetscInt, const Vec[], PetscScalar *);
59 PetscErrorCode (*mtdot_local)(Vec, PetscInt, const Vec[], PetscScalar *);
60 PetscErrorCode (*load)(Vec, PetscViewer);
61 PetscErrorCode (*reciprocal)(Vec);
62 PetscErrorCode (*conjugate)(Vec);
63 PetscErrorCode (*setlocaltoglobalmapping)(Vec, ISLocalToGlobalMapping);
64 PetscErrorCode (*getlocaltoglobalmapping)(Vec, ISLocalToGlobalMapping *);
65 PetscErrorCode (*resetarray)(Vec); /* vector points to its original array, i.e. undoes any VecPlaceArray() */
66 PetscErrorCode (*setfromoptions)(Vec, PetscOptionItems);
67 PetscErrorCode (*maxpointwisedivide)(Vec, Vec, PetscReal *); /* m = max abs(x ./ y) */
68 PetscErrorCode (*pointwisemax)(Vec, Vec, Vec);
69 PetscErrorCode (*pointwisemaxabs)(Vec, Vec, Vec);
70 PetscErrorCode (*pointwisemin)(Vec, Vec, Vec);
71 PetscErrorCode (*getvalues)(Vec, PetscInt, const PetscInt[], PetscScalar[]);
72 PetscErrorCode (*sqrt)(Vec);
73 PetscErrorCode (*abs)(Vec);
74 PetscErrorCode (*exp)(Vec);
75 PetscErrorCode (*log)(Vec);
76 PetscErrorCode (*shift)(Vec, PetscScalar);
77 PetscErrorCode (*create)(Vec);
78 PetscErrorCode (*stridegather)(Vec, PetscInt, Vec, InsertMode);
79 PetscErrorCode (*stridescatter)(Vec, PetscInt, Vec, InsertMode);
80 PetscErrorCode (*dotnorm2)(Vec, Vec, PetscScalar *, PetscScalar *);
81 PetscErrorCode (*getsubvector)(Vec, IS, Vec *);
82 PetscErrorCode (*restoresubvector)(Vec, IS, Vec *);
83 PetscErrorCode (*getarrayread)(Vec, const PetscScalar **);
84 PetscErrorCode (*restorearrayread)(Vec, const PetscScalar **);
85 PetscErrorCode (*stridesubsetgather)(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
86 PetscErrorCode (*stridesubsetscatter)(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
87 PetscErrorCode (*viewnative)(Vec, PetscViewer);
88 PetscErrorCode (*loadnative)(Vec, PetscViewer);
89 PetscErrorCode (*createlocalvector)(Vec, Vec *);
90 PetscErrorCode (*getlocalvector)(Vec, Vec);
91 PetscErrorCode (*restorelocalvector)(Vec, Vec);
92 PetscErrorCode (*getlocalvectorread)(Vec, Vec);
93 PetscErrorCode (*restorelocalvectorread)(Vec, Vec);
94 PetscErrorCode (*bindtocpu)(Vec, PetscBool);
95 PetscErrorCode (*getarraywrite)(Vec, PetscScalar **);
96 PetscErrorCode (*restorearraywrite)(Vec, PetscScalar **);
97 PetscErrorCode (*getarrayandmemtype)(Vec, PetscScalar **, PetscMemType *);
98 PetscErrorCode (*restorearrayandmemtype)(Vec, PetscScalar **);
99 PetscErrorCode (*getarrayreadandmemtype)(Vec, const PetscScalar **, PetscMemType *);
100 PetscErrorCode (*restorearrayreadandmemtype)(Vec, const PetscScalar **);
101 PetscErrorCode (*getarraywriteandmemtype)(Vec, PetscScalar **, PetscMemType *);
102 PetscErrorCode (*restorearraywriteandmemtype)(Vec, PetscScalar **, PetscMemType *);
103 PetscErrorCode (*concatenate)(PetscInt, const Vec[], Vec *, IS *[]);
104 PetscErrorCode (*sum)(Vec, PetscScalar *);
105 PetscErrorCode (*setpreallocationcoo)(Vec, PetscCount, const PetscInt[]);
106 PetscErrorCode (*setvaluescoo)(Vec, const PetscScalar[], InsertMode);
107 PetscErrorCode (*errorwnorm)(Vec, Vec, Vec, NormType, PetscReal, Vec, PetscReal, Vec, PetscReal, PetscReal *, PetscInt *, PetscReal *, PetscInt *, PetscReal *, PetscInt *);
108 PetscErrorCode (*maxpby)(Vec, PetscInt, const PetscScalar *, PetscScalar, Vec *); /* y = beta y + alpha[j] x[j] */
109 };
110
111 #if defined(offsetof) && (defined(__cplusplus) || (PETSC_C_VERSION >= 23))
112 static_assert(offsetof(struct _VecOps, duplicate) == sizeof(PetscErrorCodeFn *) * VECOP_DUPLICATE, "");
113 static_assert(offsetof(struct _VecOps, set) == sizeof(PetscErrorCodeFn *) * VECOP_SET, "");
114 static_assert(offsetof(struct _VecOps, view) == sizeof(PetscErrorCodeFn *) * VECOP_VIEW, "");
115 static_assert(offsetof(struct _VecOps, load) == sizeof(PetscErrorCodeFn *) * VECOP_LOAD, "");
116 static_assert(offsetof(struct _VecOps, viewnative) == sizeof(PetscErrorCodeFn *) * VECOP_VIEWNATIVE, "");
117 static_assert(offsetof(struct _VecOps, loadnative) == sizeof(PetscErrorCodeFn *) * VECOP_LOADNATIVE, "");
118 #endif
119
120 /*
121 The stash is used to temporarily store inserted vec values that
122 belong to another processor. During the assembly phase the stashed
123 values are moved to the correct processor and
124 */
125
126 typedef struct {
127 PetscInt nmax; /* maximum stash size */
128 PetscInt umax; /* max stash size user wants */
129 PetscInt oldnmax; /* the nmax value used previously */
130 PetscInt n; /* stash size */
131 PetscInt bs; /* block size of the stash */
132 PetscInt reallocs; /* preserve the no of mallocs invoked */
133 PetscInt *idx; /* global row numbers in stash */
134 PetscScalar *array; /* array to hold stashed values */
135 /* The following variables are used for communication */
136 MPI_Comm comm;
137 PetscMPIInt size, rank;
138 PetscMPIInt tag1, tag2;
139 MPI_Request *send_waits; /* array of send requests */
140 MPI_Request *recv_waits; /* array of receive requests */
141 MPI_Status *send_status; /* array of send status */
142 PetscMPIInt nsends, nrecvs; /* numbers of sends and receives */
143 PetscScalar *svalues, *rvalues; /* sending and receiving data */
144 PetscInt *sindices, *rindices;
145 PetscInt rmax; /* maximum message length */
146 PetscInt *nprocs; /* tmp data used both during scatterbegin and end */
147 PetscMPIInt nprocessed; /* number of messages already processed */
148 PetscBool donotstash;
149 PetscBool ignorenegidx; /* ignore negative indices passed into VecSetValues/VetGetValues */
150 InsertMode insertmode;
151 PetscInt *bowners;
152 } VecStash;
153
154 struct _p_Vec {
155 PETSCHEADER(struct _VecOps);
156 PetscLayout map;
157 void *data; /* implementation-specific data */
158 PetscBool array_gotten;
159 VecStash stash, bstash; /* used for storing off-proc values during assembly */
160 PetscBool petscnative; /* means the ->data starts with VECHEADER and can use VecGetArrayFast()*/
161 PetscInt lock; /* lock state. vector can be free (=0), locked for read (>0) or locked for write(<0) */
162 #if PetscDefined(USE_DEBUG)
163 PetscStack lockstack; /* the file,func,line of where locks are added */
164 #endif
165 PetscOffloadMask offloadmask; /* a mask which indicates where the valid vector data is (GPU, CPU or both) */
166 #if defined(PETSC_HAVE_DEVICE)
167 void *spptr; /* this is the special pointer to the array on the GPU */
168 PetscBool boundtocpu;
169 PetscBool bindingpropagates;
170 size_t minimum_bytes_pinned_memory; /* minimum data size in bytes for which pinned memory will be allocated */
171 PetscBool pinned_memory; /* PETSC_TRUE if the current host allocation has been made from pinned memory. */
172 #endif
173 char *defaultrandtype;
174 };
175
176 PETSC_EXTERN PetscLogEvent VEC_SetRandom;
177 PETSC_EXTERN PetscLogEvent VEC_View;
178 PETSC_EXTERN PetscLogEvent VEC_Max;
179 PETSC_EXTERN PetscLogEvent VEC_Min;
180 PETSC_EXTERN PetscLogEvent VEC_Dot;
181 PETSC_EXTERN PetscLogEvent VEC_MDot;
182 PETSC_EXTERN PetscLogEvent VEC_TDot;
183 PETSC_EXTERN PetscLogEvent VEC_MTDot;
184 PETSC_EXTERN PetscLogEvent VEC_Norm;
185 PETSC_EXTERN PetscLogEvent VEC_Normalize;
186 PETSC_EXTERN PetscLogEvent VEC_Scale;
187 PETSC_EXTERN PetscLogEvent VEC_Shift;
188 PETSC_EXTERN PetscLogEvent VEC_Copy;
189 PETSC_EXTERN PetscLogEvent VEC_Set;
190 PETSC_EXTERN PetscLogEvent VEC_AXPY;
191 PETSC_EXTERN PetscLogEvent VEC_AYPX;
192 PETSC_EXTERN PetscLogEvent VEC_WAXPY;
193 PETSC_EXTERN PetscLogEvent VEC_MAXPY;
194 PETSC_EXTERN PetscLogEvent VEC_AssemblyEnd;
195 PETSC_EXTERN PetscLogEvent VEC_PointwiseMult;
196 PETSC_EXTERN PetscLogEvent VEC_PointwiseDivide;
197 PETSC_EXTERN PetscLogEvent VEC_Reciprocal;
198 PETSC_EXTERN PetscLogEvent VEC_SetValues;
199 PETSC_EXTERN PetscLogEvent VEC_SetPreallocateCOO;
200 PETSC_EXTERN PetscLogEvent VEC_SetValuesCOO;
201 PETSC_EXTERN PetscLogEvent VEC_Load;
202 PETSC_EXTERN PetscLogEvent VEC_ScatterBegin;
203 PETSC_EXTERN PetscLogEvent VEC_ScatterEnd;
204 PETSC_EXTERN PetscLogEvent VEC_ReduceArithmetic;
205 PETSC_EXTERN PetscLogEvent VEC_ReduceCommunication;
206 PETSC_EXTERN PetscLogEvent VEC_ReduceBegin;
207 PETSC_EXTERN PetscLogEvent VEC_ReduceEnd;
208 PETSC_EXTERN PetscLogEvent VEC_Swap;
209 PETSC_EXTERN PetscLogEvent VEC_AssemblyBegin;
210 PETSC_EXTERN PetscLogEvent VEC_DotNorm2;
211 PETSC_EXTERN PetscLogEvent VEC_AXPBYPCZ;
212 PETSC_EXTERN PetscLogEvent VEC_Ops;
213 PETSC_EXTERN PetscLogEvent VEC_ViennaCLCopyToGPU;
214 PETSC_EXTERN PetscLogEvent VEC_ViennaCLCopyFromGPU;
215 PETSC_EXTERN PetscLogEvent VEC_CUDACopyToGPU;
216 PETSC_EXTERN PetscLogEvent VEC_CUDACopyFromGPU;
217 PETSC_EXTERN PetscLogEvent VEC_HIPCopyToGPU;
218 PETSC_EXTERN PetscLogEvent VEC_HIPCopyFromGPU;
219
220 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
221 #if defined(PETSC_HAVE_VIENNACL)
222 PETSC_EXTERN PetscErrorCode VecViennaCLAllocateCheckHost(Vec v);
223 PETSC_EXTERN PetscErrorCode VecViennaCLCopyFromGPU(Vec v);
224 #endif
225
226 /*
227 Common header shared by array based vectors,
228 currently Vec_Seq and Vec_MPI
229 */
230 #define VECHEADER \
231 PetscScalar *array; \
232 PetscScalar *array_allocated; /* if the array was allocated by PETSc this is its pointer */ \
233 PetscScalar *unplacedarray; /* if one called VecPlaceArray(), this is where it stashed the original */
234
235 /* Get Root type of vector. e.g. VECSEQ -> VECSTANDARD, VECMPICUDA -> VECCUDA */
236 PETSC_EXTERN PetscErrorCode VecGetRootType_Private(Vec, VecType *);
237
238 /* Default obtain and release vectors; can be used by any implementation */
239 PETSC_INTERN PetscErrorCode VecDuplicateVecs_Default(Vec, PetscInt, Vec *[]);
240 PETSC_INTERN PetscErrorCode VecDestroyVecs_Default(PetscInt, Vec[]);
241 PETSC_INTERN PetscErrorCode VecView_Binary(Vec, PetscViewer);
242
243 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecLoad_Default(Vec, PetscViewer);
244
245 PETSC_INTERN PetscInt NormIds[4]; /* map from NormType to IDs used to cache/retrieve values of norms, 1_AND_2 is excluded */
246
247 PETSC_INTERN PetscErrorCode VecStashCreate_Private(MPI_Comm, PetscInt, VecStash *);
248 PETSC_INTERN PetscErrorCode VecStashDestroy_Private(VecStash *);
249 PETSC_INTERN PetscErrorCode VecStashScatterEnd_Private(VecStash *);
250 PETSC_INTERN PetscErrorCode VecStashSetInitialSize_Private(VecStash *, PetscInt);
251 PETSC_INTERN PetscErrorCode VecStashGetInfo_Private(VecStash *, PetscInt *, PetscInt *);
252 PETSC_INTERN PetscErrorCode VecStashScatterBegin_Private(VecStash *, const PetscInt *);
253 PETSC_INTERN PetscErrorCode VecStashScatterGetMesg_Private(VecStash *, PetscMPIInt *, PetscInt **, PetscScalar **, PetscInt *);
254 PETSC_INTERN PetscErrorCode VecStashSortCompress_Private(VecStash *);
255 PETSC_INTERN PetscErrorCode VecStashGetOwnerList_Private(VecStash *, PetscLayout, PetscMPIInt *, PetscMPIInt **);
256
257 PETSC_INTERN PetscErrorCode VecStashExpand_Private(VecStash *, PetscInt);
258
259 /*
260 VecStashValue_Private - inserts a single value into the stash.
261
262 Input Parameters:
263 stash - the stash
264 idx - the global of the inserted value
265 values - the value inserted
266 */
VecStashValue_Private(VecStash * stash,PetscInt row,PetscScalar value)267 static inline PetscErrorCode VecStashValue_Private(VecStash *stash, PetscInt row, PetscScalar value)
268 {
269 /* Check and see if we have sufficient memory */
270 PetscFunctionBegin;
271 if (((stash)->n + 1) > (stash)->nmax) PetscCall(VecStashExpand_Private(stash, 1));
272 (stash)->idx[(stash)->n] = row;
273 (stash)->array[(stash)->n] = value;
274 (stash)->n++;
275 PetscFunctionReturn(PETSC_SUCCESS);
276 }
277
278 /*
279 VecStashValuesBlocked_Private - inserts 1 block of values into the stash.
280
281 Input Parameters:
282 stash - the stash
283 idx - the global block index
284 values - the values inserted
285 */
VecStashValuesBlocked_Private(VecStash * stash,PetscInt row,PetscScalar * values)286 static inline PetscErrorCode VecStashValuesBlocked_Private(VecStash *stash, PetscInt row, PetscScalar *values)
287 {
288 PetscInt stash_bs = (stash)->bs;
289 PetscScalar *array;
290
291 PetscFunctionBegin;
292 if (((stash)->n + 1) > (stash)->nmax) PetscCall(VecStashExpand_Private(stash, 1));
293 array = (stash)->array + stash_bs * (stash)->n;
294 (stash)->idx[(stash)->n] = row;
295 if (values) PetscCall(PetscArraycpy(array, values, stash_bs));
296 else PetscCall(PetscArrayzero(array, stash_bs));
297 (stash)->n++;
298 PetscFunctionReturn(PETSC_SUCCESS);
299 }
300
301 PETSC_INTERN PetscErrorCode VecStrideGather_Default(Vec, PetscInt, Vec, InsertMode);
302 PETSC_INTERN PetscErrorCode VecStrideScatter_Default(Vec, PetscInt, Vec, InsertMode);
303 PETSC_INTERN PetscErrorCode VecStrideSubSetGather_Default(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
304 PETSC_INTERN PetscErrorCode VecStrideSubSetScatter_Default(Vec, PetscInt, const PetscInt[], const PetscInt[], Vec, InsertMode);
305
306 PETSC_INTERN PetscErrorCode VecReciprocal_Default(Vec);
307 #if defined(PETSC_HAVE_MATLAB)
308 PETSC_EXTERN PetscErrorCode VecMatlabEnginePut_Default(PetscObject, void *);
309 PETSC_EXTERN PetscErrorCode VecMatlabEngineGet_Default(PetscObject, void *);
310 #endif
311
312 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode PetscSectionGetField_Internal(PetscSection, PetscSection, Vec, PetscInt, PetscInt, PetscInt, IS *, Vec *);
313 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode PetscSectionRestoreField_Internal(PetscSection, PetscSection, Vec, PetscInt, PetscInt, PetscInt, IS *, Vec *);
314
315 #define VecCheckSameLocalSize(x, ar1, y, ar2) \
316 do { \
317 PetscCheck((x)->map->n == (y)->map->n, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Incompatible vector local lengths parameter # %d local size %" PetscInt_FMT " != parameter # %d local size %" PetscInt_FMT, ar1, (x)->map->n, ar2, (y)->map->n); \
318 } while (0)
319
320 #define VecCheckSameSize(x, ar1, y, ar2) \
321 do { \
322 PetscCheck((x)->map->N == (y)->map->N, PetscObjectComm((PetscObject)x), PETSC_ERR_ARG_INCOMP, "Incompatible vector global lengths parameter # %d global size %" PetscInt_FMT " != parameter # %d global size %" PetscInt_FMT, ar1, (x)->map->N, ar2, \
323 (y)->map->N); \
324 VecCheckSameLocalSize(x, ar1, y, ar2); \
325 } while (0)
326
327 #define VecCheckLocalSize(x, ar1, n) \
328 do { \
329 PetscCheck((x)->map->n == (n), PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Incorrect vector local size: parameter # %d local size %" PetscInt_FMT " != %" PetscInt_FMT, ar1, (x)->map->n, n); \
330 } while (0)
331
332 #define VecCheckSize(x, ar1, n, N) \
333 do { \
334 PetscCheck((x)->map->N == (N), PetscObjectComm((PetscObject)x), PETSC_ERR_ARG_INCOMP, "Incorrect vector global size: parameter # %d global size %" PetscInt_FMT " != %" PetscInt_FMT, ar1, (x)->map->N, N); \
335 VecCheckLocalSize(x, ar1, n); \
336 } while (0)
337
338 typedef struct _VecTaggerOps *VecTaggerOps;
339 struct _VecTaggerOps {
340 PetscErrorCode (*create)(VecTagger);
341 PetscErrorCode (*destroy)(VecTagger);
342 PetscErrorCode (*setfromoptions)(VecTagger, PetscOptionItems);
343 PetscErrorCode (*setup)(VecTagger);
344 PetscErrorCode (*view)(VecTagger, PetscViewer);
345 PetscErrorCode (*computeboxes)(VecTagger, Vec, PetscInt *, VecTaggerBox **, PetscBool *);
346 PetscErrorCode (*computeis)(VecTagger, Vec, IS *, PetscBool *);
347 };
348 struct _p_VecTagger {
349 PETSCHEADER(struct _VecTaggerOps);
350 void *data;
351 PetscInt blocksize;
352 PetscBool invert;
353 PetscBool setupcalled;
354 };
355
356 PETSC_INTERN PetscBool VecTaggerRegisterAllCalled;
357 PETSC_INTERN PetscErrorCode VecTaggerComputeIS_FromBoxes(VecTagger, Vec, IS *, PetscBool *);
358 PETSC_INTERN PetscMPIInt Petsc_Reduction_keyval;
359
360 PETSC_INTERN PetscInt VecGetSubVectorSavedStateId;
361 PETSC_INTERN PetscErrorCode VecGetSubVectorContiguityAndBS_Private(Vec, IS, PetscBool *, PetscInt *, PetscInt *);
362 PETSC_INTERN PetscErrorCode VecGetSubVectorThroughVecScatter_Private(Vec, IS, PetscInt, Vec *);
363
364 #if PetscDefined(HAVE_CUDA)
365 PETSC_INTERN PetscErrorCode VecCreate_CUDA(Vec);
366 PETSC_INTERN PetscErrorCode VecCreate_SeqCUDA(Vec);
367 PETSC_INTERN PetscErrorCode VecCreate_MPICUDA(Vec);
368 PETSC_INTERN PetscErrorCode VecCUDAGetArrays_Private(Vec, const PetscScalar **, const PetscScalar **, PetscOffloadMask *);
369 PETSC_INTERN PetscErrorCode VecConvert_Seq_SeqCUDA_inplace(Vec);
370 PETSC_INTERN PetscErrorCode VecConvert_MPI_MPICUDA_inplace(Vec);
371 #endif
372
373 #if PetscDefined(HAVE_HIP)
374 PETSC_INTERN PetscErrorCode VecCreate_HIP(Vec);
375 PETSC_INTERN PetscErrorCode VecCreate_SeqHIP(Vec);
376 PETSC_INTERN PetscErrorCode VecCreate_MPIHIP(Vec);
377 PETSC_INTERN PetscErrorCode VecHIPGetArrays_Private(Vec, const PetscScalar **, const PetscScalar **, PetscOffloadMask *);
378 PETSC_INTERN PetscErrorCode VecConvert_Seq_SeqHIP_inplace(Vec);
379 PETSC_INTERN PetscErrorCode VecConvert_MPI_MPIHIP_inplace(Vec);
380 #endif
381
382 #if defined(PETSC_HAVE_KOKKOS)
383 PETSC_INTERN PetscErrorCode VecCreateMPIKokkosWithArrays_Private(MPI_Comm, PetscInt, PetscInt, PetscInt, const PetscScalar *, const PetscScalar *, Vec *);
384 PETSC_INTERN PetscErrorCode VecConvert_Seq_SeqKokkos_inplace(Vec);
385 PETSC_INTERN PetscErrorCode VecConvert_MPI_MPIKokkos_inplace(Vec);
386 #endif
387
388 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecCreateWithLayout_Private(PetscLayout, Vec *);
389 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecCreateSeqWithLayoutAndArray_Private(PetscLayout, const PetscScalar *, Vec *);
390 PETSC_SINGLE_LIBRARY_INTERN PetscErrorCode VecCreateMPIWithLayoutAndArray_Private(PetscLayout, const PetscScalar *, Vec *);
391
392 /* std::upper_bound(): Given a sorted array, return index of the first element in range [first,last) whose value
393 is greater than value, or last if there is no such element.
394 */
PetscSortedIntUpperBound(const PetscInt * array,PetscCount first,PetscCount last,PetscInt value,PetscCount * upper)395 static inline PetscErrorCode PetscSortedIntUpperBound(const PetscInt *array, PetscCount first, PetscCount last, PetscInt value, PetscCount *upper)
396 {
397 PetscCount count = last - first;
398
399 PetscFunctionBegin;
400 while (count > 0) {
401 const PetscCount step = count / 2;
402 PetscCount it = first + step;
403
404 if (value < array[it]) {
405 count = step;
406 } else {
407 first = it + 1;
408 count -= step + 1;
409 }
410 }
411 *upper = first;
412 PetscFunctionReturn(PETSC_SUCCESS);
413 }
414
415 #define VEC_ASYNC_FN_NAME(Base) "Vec" Base "Async_Private_C"
416 #define VecAsyncFnName(Base) VEC_##Base##_ASYNC_FN_NAME
417
418 #define VEC_Abs_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Abs")
419 #define VEC_AXPBY_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("AXPBY")
420 #define VEC_AXPBYPCZ_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("AXPBYPCZ")
421 #define VEC_AXPY_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("AXPY")
422 #define VEC_AYPX_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("AYPX")
423 #define VEC_Conjugate_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Conjugate")
424 #define VEC_Copy_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Copy")
425 #define VEC_Exp_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Exp")
426 #define VEC_Log_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Log")
427 #define VEC_MAXPY_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("MAXPY")
428 #define VEC_PointwiseDivide_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseDivide")
429 #define VEC_PointwiseMax_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseMax")
430 #define VEC_PointwiseMaxAbs_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseMaxAbs")
431 #define VEC_PointwiseMin_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseMin")
432 #define VEC_PointwiseMult_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("PointwiseMult")
433 #define VEC_Reciprocal_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Reciprocal")
434 #define VEC_Scale_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Scale")
435 #define VEC_Set_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Set")
436 #define VEC_Shift_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Shift")
437 #define VEC_SqrtAbs_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("SqrtAbs")
438 #define VEC_Swap_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("Swap")
439 #define VEC_WAXPY_ASYNC_FN_NAME VEC_ASYNC_FN_NAME("WAXPY")
440
441 PETSC_INTERN PetscErrorCode VecAbsAsync_Private(Vec, PetscDeviceContext);
442 PETSC_INTERN PetscErrorCode VecAXPBYAsync_Private(Vec, PetscScalar, PetscScalar, Vec, PetscDeviceContext);
443 PETSC_INTERN PetscErrorCode VecAXPBYPCZAsync_Private(Vec, PetscScalar, PetscScalar, PetscScalar, Vec, Vec, PetscDeviceContext);
444 PETSC_INTERN PetscErrorCode VecAXPYAsync_Private(Vec, PetscScalar, Vec, PetscDeviceContext);
445 PETSC_INTERN PetscErrorCode VecAYPXAsync_Private(Vec, PetscScalar, Vec, PetscDeviceContext);
446 PETSC_INTERN PetscErrorCode VecConjugateAsync_Private(Vec, PetscDeviceContext);
447 PETSC_INTERN PetscErrorCode VecCopyAsync_Private(Vec, Vec, PetscDeviceContext);
448 PETSC_INTERN PetscErrorCode VecExpAsync_Private(Vec, PetscDeviceContext);
449 PETSC_INTERN PetscErrorCode VecLogAsync_Private(Vec, PetscDeviceContext);
450 PETSC_INTERN PetscErrorCode VecMAXPYAsync_Private(Vec, PetscInt, const PetscScalar *, Vec[], PetscDeviceContext);
451 PETSC_INTERN PetscErrorCode VecPointwiseDivideAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
452 PETSC_INTERN PetscErrorCode VecPointwiseMaxAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
453 PETSC_INTERN PetscErrorCode VecPointwiseMaxAbsAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
454 PETSC_INTERN PetscErrorCode VecPointwiseMinAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
455 PETSC_INTERN PetscErrorCode VecPointwiseMultAsync_Private(Vec, Vec, Vec, PetscDeviceContext);
456 PETSC_INTERN PetscErrorCode VecReciprocalAsync_Private(Vec, PetscDeviceContext);
457 PETSC_INTERN PetscErrorCode VecScaleAsync_Private(Vec, PetscScalar, PetscDeviceContext);
458 PETSC_INTERN PetscErrorCode VecSetAsync_Private(Vec, PetscScalar, PetscDeviceContext);
459 PETSC_INTERN PetscErrorCode VecShiftAsync_Private(Vec, PetscScalar, PetscDeviceContext);
460 PETSC_INTERN PetscErrorCode VecSqrtAbsAsync_Private(Vec, PetscDeviceContext);
461 PETSC_INTERN PetscErrorCode VecSwapAsync_Private(Vec, Vec, PetscDeviceContext);
462 PETSC_INTERN PetscErrorCode VecWAXPYAsync_Private(Vec, PetscScalar, Vec, Vec, PetscDeviceContext);
463
464 #define VecMethodDispatch(v, dctx, async_name, name, async_arg_types, ...) \
465 do { \
466 PetscErrorCode(*_8_f) async_arg_types = NULL; \
467 if (dctx) PetscCall(PetscObjectQueryFunction((PetscObject)(v), async_name, &_8_f)); \
468 if (_8_f) { \
469 PetscCall((*_8_f)(v, __VA_ARGS__, dctx)); \
470 } else { \
471 PetscUseTypeMethod(v, name, __VA_ARGS__); \
472 } \
473 } while (0)
474
475 // return in lda the vector's local size aligned to <alignment> bytes, where lda is an integer pointer
476 #define VecGetLocalSizeAligned(v, alignment, lda) \
477 do { \
478 PetscInt n = (v)->map->n; \
479 const size_t s = (alignment) / sizeof(PetscScalar); \
480 *(lda) = ((n + s - 1) / s) * s; \
481 } while (0)
482