1 // Copyright (c) 2017-2026, Lawrence Livermore National Security, LLC and other CEED contributors.
2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
3 //
4 // SPDX-License-Identifier: BSD-2-Clause
5 //
6 // This file is part of CEED: http://github.com/ceed
7
8 #include <ceed.h>
9 #include <ceed/backend.h>
10 #include <assert.h>
11 #include <math.h>
12 #include <stdbool.h>
13 #include <string.h>
14 #include <valgrind/memcheck.h>
15
16 #include "ceed-memcheck.h"
17
18 //------------------------------------------------------------------------------
19 // Has Valid Array
20 //------------------------------------------------------------------------------
CeedVectorHasValidArray_Memcheck(CeedVector vec,bool * has_valid_array)21 static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_array) {
22 CeedVector_Memcheck *impl;
23
24 CeedCallBackend(CeedVectorGetData(vec, &impl));
25 *has_valid_array = !!impl->array_allocated;
26 return CEED_ERROR_SUCCESS;
27 }
28
29 //------------------------------------------------------------------------------
30 // Check if has borrowed array of given type
31 //------------------------------------------------------------------------------
CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec,CeedMemType mem_type,bool * has_borrowed_array_of_type)32 static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) {
33 CeedVector_Memcheck *impl;
34
35 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend");
36
37 CeedCallBackend(CeedVectorGetData(vec, &impl));
38 *has_borrowed_array_of_type = !!impl->array_borrowed;
39 return CEED_ERROR_SUCCESS;
40 }
41
42 //------------------------------------------------------------------------------
43 // Vector Set Array
44 //------------------------------------------------------------------------------
CeedVectorSetArray_Memcheck(CeedVector vec,CeedMemType mem_type,CeedCopyMode copy_mode,CeedScalar * array)45 static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedCopyMode copy_mode, CeedScalar *array) {
46 CeedSize length;
47 CeedVector_Memcheck *impl;
48
49 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend");
50
51 CeedCallBackend(CeedVectorGetData(vec, &impl));
52 CeedCallBackend(CeedVectorGetLength(vec, &length));
53
54 // Clear previous owned arrays
55 if (impl->array_allocated) {
56 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN;
57 VALGRIND_DISCARD(impl->allocated_block_id);
58 }
59 CeedCallBackend(CeedFree(&impl->array_allocated));
60 if (copy_mode != CEED_COPY_VALUES) {
61 if (impl->array_owned) {
62 for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN;
63 VALGRIND_DISCARD(impl->owned_block_id);
64 }
65 CeedCallBackend(CeedFree(&impl->array_owned));
66 }
67
68 // Clear borrowed block id, if present
69 if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id);
70
71 // Set internal pointers to external arrays
72 switch (copy_mode) {
73 case CEED_COPY_VALUES:
74 // Nothing to update
75 break;
76 case CEED_OWN_POINTER:
77 impl->array_owned = array;
78 impl->array_borrowed = NULL;
79 impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer");
80 break;
81 case CEED_USE_POINTER:
82 impl->array_owned = NULL;
83 impl->array_borrowed = array;
84 impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer");
85 break;
86 }
87
88 // Create internal array data buffer
89 CeedCallBackend(CeedCalloc(length, &impl->array_allocated));
90 impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer");
91 if (array) {
92 memcpy(impl->array_allocated, array, length * sizeof(CeedScalar));
93 } else {
94 for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN;
95 }
96 return CEED_ERROR_SUCCESS;
97 }
98
99 //------------------------------------------------------------------------------
100 // Set internal array to value
101 //------------------------------------------------------------------------------
CeedVectorSetValue_Memcheck(CeedVector vec,CeedScalar value)102 static int CeedVectorSetValue_Memcheck(CeedVector vec, CeedScalar value) {
103 CeedSize length;
104 CeedVector_Memcheck *impl;
105
106 CeedCallBackend(CeedVectorGetData(vec, &impl));
107 CeedCallBackend(CeedVectorGetLength(vec, &length));
108
109 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL));
110 assert(impl->array_allocated);
111 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = value;
112 return CEED_ERROR_SUCCESS;
113 }
114
115 //------------------------------------------------------------------------------
116 // Set internal array to value strided
117 //------------------------------------------------------------------------------
CeedVectorSetValueStrided_Memcheck(CeedVector vec,CeedSize start,CeedSize stop,CeedSize step,CeedScalar val)118 static int CeedVectorSetValueStrided_Memcheck(CeedVector vec, CeedSize start, CeedSize stop, CeedSize step, CeedScalar val) {
119 CeedSize length;
120 CeedVector_Memcheck *impl;
121
122 CeedCallBackend(CeedVectorGetData(vec, &impl));
123 CeedCallBackend(CeedVectorGetLength(vec, &length));
124
125 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL));
126 assert(impl->array_allocated);
127 if (stop == -1) stop = length;
128 for (CeedSize i = start; i < stop; i += step) impl->array_allocated[i] = val;
129 return CEED_ERROR_SUCCESS;
130 }
131
132 //------------------------------------------------------------------------------
133 // Sync arrays
134 //------------------------------------------------------------------------------
CeedVectorSyncArray_Memcheck(const CeedVector vec,CeedMemType mem_type)135 static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) {
136 CeedSize length;
137 CeedVector_Memcheck *impl;
138
139 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend");
140
141 CeedCallBackend(CeedVectorGetData(vec, &impl));
142 CeedCallBackend(CeedVectorGetLength(vec, &length));
143
144 // Copy internal buffer back to owned or borrowed array
145 if (impl->array_owned) {
146 memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar));
147 }
148 if (impl->array_borrowed) {
149 memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar));
150 }
151 return CEED_ERROR_SUCCESS;
152 }
153
154 //------------------------------------------------------------------------------
155 // Vector Take Array
156 //------------------------------------------------------------------------------
CeedVectorTakeArray_Memcheck(CeedVector vec,CeedMemType mem_type,CeedScalar ** array)157 static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) {
158 CeedSize length;
159 CeedVector_Memcheck *impl;
160
161 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend");
162
163 CeedCallBackend(CeedVectorGetData(vec, &impl));
164 CeedCallBackend(CeedVectorGetLength(vec, &length));
165
166 // Synchronize memory
167 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST));
168
169 // Return borrowed array
170 (*array) = impl->array_borrowed;
171 impl->array_borrowed = NULL;
172 VALGRIND_DISCARD(impl->borrowed_block_id);
173
174 // De-allocate internal memory
175 if (impl->array_allocated) {
176 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN;
177 VALGRIND_DISCARD(impl->allocated_block_id);
178 }
179 CeedCallBackend(CeedFree(&impl->array_allocated));
180 return CEED_ERROR_SUCCESS;
181 }
182
183 //------------------------------------------------------------------------------
184 // Vector Get Array
185 //------------------------------------------------------------------------------
CeedVectorGetArray_Memcheck(CeedVector vec,CeedMemType mem_type,CeedScalar ** array)186 static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) {
187 CeedSize length;
188 CeedVector_Memcheck *impl;
189
190 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend");
191
192 CeedCallBackend(CeedVectorGetData(vec, &impl));
193 CeedCallBackend(CeedVectorGetLength(vec, &length));
194
195 // Create and return writable buffer
196 CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy));
197 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy");
198 memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar));
199 *array = impl->array_writable_copy;
200 return CEED_ERROR_SUCCESS;
201 }
202
203 //------------------------------------------------------------------------------
204 // Vector Get Array Read
205 //------------------------------------------------------------------------------
CeedVectorGetArrayRead_Memcheck(CeedVector vec,CeedMemType mem_type,const CeedScalar ** array)206 static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, const CeedScalar **array) {
207 CeedSize length;
208 CeedVector_Memcheck *impl;
209
210 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend");
211
212 CeedCallBackend(CeedVectorGetData(vec, &impl));
213 CeedCallBackend(CeedVectorGetLength(vec, &length));
214
215 // Create and return read-only buffer
216 if (!impl->array_read_only_copy) {
217 CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy));
218 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy");
219 memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar));
220 }
221 *array = impl->array_read_only_copy;
222 return CEED_ERROR_SUCCESS;
223 }
224
225 //------------------------------------------------------------------------------
226 // Vector Get Array Write
227 //------------------------------------------------------------------------------
CeedVectorGetArrayWrite_Memcheck(CeedVector vec,CeedMemType mem_type,CeedScalar ** array)228 static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) {
229 CeedSize length;
230 CeedVector_Memcheck *impl;
231
232 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend");
233
234 CeedCallBackend(CeedVectorGetData(vec, &impl));
235 CeedCallBackend(CeedVectorGetLength(vec, &length));
236
237 // Allocate buffer if necessary
238 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL));
239
240 // Get writable buffer
241 CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array));
242
243 // Invalidate array data to prevent accidental reads
244 for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN;
245 impl->is_write_only_access = true;
246 return CEED_ERROR_SUCCESS;
247 }
248
249 //------------------------------------------------------------------------------
250 // Vector Restore Array
251 //------------------------------------------------------------------------------
CeedVectorRestoreArray_Memcheck(CeedVector vec)252 static int CeedVectorRestoreArray_Memcheck(CeedVector vec) {
253 CeedSize length;
254 CeedVector_Memcheck *impl;
255
256 CeedCallBackend(CeedVectorGetData(vec, &impl));
257 CeedCallBackend(CeedVectorGetLength(vec, &length));
258
259 // Check for unset entries after write-only access
260 if (impl->is_write_only_access) {
261 for (CeedSize i = 0; i < length; i++) {
262 if (isnan(impl->array_writable_copy[i])) {
263 CeedDebug256(CeedVectorReturnCeed(vec), CEED_DEBUG_COLOR_WARNING,
264 "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i);
265 }
266 }
267 impl->is_write_only_access = false;
268 }
269
270 // Copy back to internal buffer and sync
271 memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar));
272 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST));
273
274 // Invalidate writable buffer
275 for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN;
276 CeedCallBackend(CeedFree(&impl->array_writable_copy));
277 VALGRIND_DISCARD(impl->writable_block_id);
278 return CEED_ERROR_SUCCESS;
279 }
280
281 //------------------------------------------------------------------------------
282 // Vector Restore Array Read-Only
283 //------------------------------------------------------------------------------
CeedVectorRestoreArrayRead_Memcheck(CeedVector vec)284 static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) {
285 CeedSize length;
286 CeedVector_Memcheck *impl;
287
288 CeedCallBackend(CeedVectorGetData(vec, &impl));
289 CeedCallBackend(CeedVectorGetLength(vec, &length));
290
291 // Verify no changes made during read-only access
292 bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar));
293
294 CeedCheck(!is_changed, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode");
295
296 // Invalidate read-only buffer
297 for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN;
298 CeedCallBackend(CeedFree(&impl->array_read_only_copy));
299 VALGRIND_DISCARD(impl->read_only_block_id);
300 return CEED_ERROR_SUCCESS;
301 }
302
303 //------------------------------------------------------------------------------
304 // Take reciprocal of a vector
305 //------------------------------------------------------------------------------
CeedVectorReciprocal_Memcheck(CeedVector vec)306 static int CeedVectorReciprocal_Memcheck(CeedVector vec) {
307 CeedSize length;
308 CeedVector_Memcheck *impl;
309
310 CeedCallBackend(CeedVectorGetData(vec, &impl));
311 CeedCallBackend(CeedVectorGetLength(vec, &length));
312
313 for (CeedSize i = 0; i < length; i++) {
314 if (fabs(impl->array_allocated[i]) > CEED_EPSILON) impl->array_allocated[i] = 1. / impl->array_allocated[i];
315 }
316 return CEED_ERROR_SUCCESS;
317 }
318
319 //------------------------------------------------------------------------------
320 // Compute x = alpha x
321 //------------------------------------------------------------------------------
CeedVectorScale_Memcheck(CeedVector x,CeedScalar alpha)322 static int CeedVectorScale_Memcheck(CeedVector x, CeedScalar alpha) {
323 CeedSize length;
324 CeedVector_Memcheck *impl;
325
326 CeedCallBackend(CeedVectorGetData(x, &impl));
327 CeedCallBackend(CeedVectorGetLength(x, &length));
328
329 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] *= alpha;
330 return CEED_ERROR_SUCCESS;
331 }
332
333 //------------------------------------------------------------------------------
334 // Compute y = alpha x + y
335 //------------------------------------------------------------------------------
CeedVectorAXPY_Memcheck(CeedVector y,CeedScalar alpha,CeedVector x)336 static int CeedVectorAXPY_Memcheck(CeedVector y, CeedScalar alpha, CeedVector x) {
337 CeedSize length;
338 CeedVector_Memcheck *impl_x, *impl_y;
339
340 CeedCallBackend(CeedVectorGetData(x, &impl_x));
341 CeedCallBackend(CeedVectorGetData(y, &impl_y));
342 CeedCallBackend(CeedVectorGetLength(y, &length));
343
344 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] += alpha * impl_x->array_allocated[i];
345 return CEED_ERROR_SUCCESS;
346 }
347
348 //------------------------------------------------------------------------------
349 // Compute y = alpha x + beta y
350 //------------------------------------------------------------------------------
CeedVectorAXPBY_Memcheck(CeedVector y,CeedScalar alpha,CeedScalar beta,CeedVector x)351 static int CeedVectorAXPBY_Memcheck(CeedVector y, CeedScalar alpha, CeedScalar beta, CeedVector x) {
352 CeedSize length;
353 CeedVector_Memcheck *impl_x, *impl_y;
354
355 CeedCallBackend(CeedVectorGetData(x, &impl_x));
356 CeedCallBackend(CeedVectorGetData(y, &impl_y));
357 CeedCallBackend(CeedVectorGetLength(y, &length));
358
359 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] = alpha * impl_x->array_allocated[i] + beta * impl_y->array_allocated[i];
360 return CEED_ERROR_SUCCESS;
361 }
362
363 //------------------------------------------------------------------------------
364 // Compute the pointwise multiplication w = x .* y
365 //------------------------------------------------------------------------------
CeedVectorPointwiseMult_Memcheck(CeedVector w,CeedVector x,CeedVector y)366 static int CeedVectorPointwiseMult_Memcheck(CeedVector w, CeedVector x, CeedVector y) {
367 CeedSize length;
368 CeedVector_Memcheck *impl_x, *impl_y, *impl_w;
369
370 CeedCallBackend(CeedVectorGetData(x, &impl_x));
371 CeedCallBackend(CeedVectorGetData(y, &impl_y));
372 CeedCallBackend(CeedVectorGetData(w, &impl_w));
373 CeedCallBackend(CeedVectorGetLength(w, &length));
374
375 if (!impl_w->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(w, CEED_MEM_HOST, CEED_COPY_VALUES, NULL));
376 assert(impl_w->array_allocated);
377 for (CeedSize i = 0; i < length; i++) impl_w->array_allocated[i] = impl_x->array_allocated[i] * impl_y->array_allocated[i];
378 return CEED_ERROR_SUCCESS;
379 }
380
381 //------------------------------------------------------------------------------
382 // Vector Destroy
383 //------------------------------------------------------------------------------
CeedVectorDestroy_Memcheck(CeedVector vec)384 static int CeedVectorDestroy_Memcheck(CeedVector vec) {
385 CeedVector_Memcheck *impl;
386
387 // Free allocations and discard block ids
388 CeedCallBackend(CeedVectorGetData(vec, &impl));
389 if (impl->array_allocated) {
390 CeedCallBackend(CeedFree(&impl->array_allocated));
391 VALGRIND_DISCARD(impl->allocated_block_id);
392 }
393 if (impl->array_owned) {
394 CeedCallBackend(CeedFree(&impl->array_owned));
395 VALGRIND_DISCARD(impl->owned_block_id);
396 }
397 if (impl->array_borrowed) {
398 VALGRIND_DISCARD(impl->borrowed_block_id);
399 }
400 CeedCallBackend(CeedFree(&impl));
401 return CEED_ERROR_SUCCESS;
402 }
403
404 //------------------------------------------------------------------------------
405 // Vector Create
406 //------------------------------------------------------------------------------
CeedVectorCreate_Memcheck(CeedSize n,CeedVector vec)407 int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) {
408 Ceed ceed;
409 CeedVector_Memcheck *impl;
410
411 CeedCallBackend(CeedVectorGetCeed(vec, &ceed));
412 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck));
413 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck));
414 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck));
415 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValue", CeedVectorSetValue_Memcheck));
416 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValueStrided", CeedVectorSetValueStrided_Memcheck));
417 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck));
418 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck));
419 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck));
420 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck));
421 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck));
422 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck));
423 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck));
424 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Reciprocal", CeedVectorReciprocal_Memcheck));
425 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Scale", CeedVectorScale_Memcheck));
426 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPY", CeedVectorAXPY_Memcheck));
427 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPBY", CeedVectorAXPBY_Memcheck));
428 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "PointwiseMult", CeedVectorPointwiseMult_Memcheck));
429 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck));
430 CeedCallBackend(CeedDestroy(&ceed));
431 CeedCallBackend(CeedCalloc(1, &impl));
432 CeedCallBackend(CeedVectorSetData(vec, impl));
433 return CEED_ERROR_SUCCESS;
434 }
435
436 //------------------------------------------------------------------------------
437