1 // Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed.h> 9 #include <ceed/backend.h> 10 #include <assert.h> 11 #include <math.h> 12 #include <stdbool.h> 13 #include <string.h> 14 #include <valgrind/memcheck.h> 15 16 #include "ceed-memcheck.h" 17 18 //------------------------------------------------------------------------------ 19 // Has Valid Array 20 //------------------------------------------------------------------------------ 21 static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_array) { 22 CeedVector_Memcheck *impl; 23 24 CeedCallBackend(CeedVectorGetData(vec, &impl)); 25 *has_valid_array = !!impl->array_allocated; 26 return CEED_ERROR_SUCCESS; 27 } 28 29 //------------------------------------------------------------------------------ 30 // Check if has borrowed array of given type 31 //------------------------------------------------------------------------------ 32 static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) { 33 CeedVector_Memcheck *impl; 34 35 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 36 37 CeedCallBackend(CeedVectorGetData(vec, &impl)); 38 *has_borrowed_array_of_type = !!impl->array_borrowed; 39 return CEED_ERROR_SUCCESS; 40 } 41 42 //------------------------------------------------------------------------------ 43 // Vector Set Array 44 //------------------------------------------------------------------------------ 45 static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedCopyMode copy_mode, CeedScalar *array) { 46 CeedSize length; 47 CeedVector_Memcheck *impl; 48 49 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 50 51 CeedCallBackend(CeedVectorGetData(vec, &impl)); 52 CeedCallBackend(CeedVectorGetLength(vec, &length)); 53 54 // Clear previous owned arrays 55 if (impl->array_allocated) { 56 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 57 VALGRIND_DISCARD(impl->allocated_block_id); 58 } 59 CeedCallBackend(CeedFree(&impl->array_allocated)); 60 if (copy_mode != CEED_COPY_VALUES) { 61 if (impl->array_owned) { 62 for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN; 63 VALGRIND_DISCARD(impl->owned_block_id); 64 } 65 CeedCallBackend(CeedFree(&impl->array_owned)); 66 } 67 68 // Clear borrowed block id, if present 69 if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); 70 71 // Set internal pointers to external arrays 72 switch (copy_mode) { 73 case CEED_COPY_VALUES: 74 // Nothing to update 75 break; 76 case CEED_OWN_POINTER: 77 impl->array_owned = array; 78 impl->array_borrowed = NULL; 79 impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer"); 80 break; 81 case CEED_USE_POINTER: 82 impl->array_owned = NULL; 83 impl->array_borrowed = array; 84 impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer"); 85 break; 86 } 87 88 // Create internal array data buffer 89 CeedCallBackend(CeedCalloc(length, &impl->array_allocated)); 90 impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer"); 91 if (array) { 92 memcpy(impl->array_allocated, array, length * sizeof(CeedScalar)); 93 } else { 94 for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN; 95 } 96 return CEED_ERROR_SUCCESS; 97 } 98 99 //------------------------------------------------------------------------------ 100 // Set internal array to value 101 //------------------------------------------------------------------------------ 102 static int CeedVectorSetValue_Memcheck(CeedVector vec, CeedScalar value) { 103 CeedSize length; 104 CeedVector_Memcheck *impl; 105 106 CeedCallBackend(CeedVectorGetData(vec, &impl)); 107 CeedCallBackend(CeedVectorGetLength(vec, &length)); 108 109 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 110 assert(impl->array_allocated); 111 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = value; 112 return CEED_ERROR_SUCCESS; 113 } 114 115 //------------------------------------------------------------------------------ 116 // Set internal array to value strided 117 //------------------------------------------------------------------------------ 118 static int CeedVectorSetValueStrided_Memcheck(CeedVector vec, CeedSize start, CeedSize stop, CeedSize step, CeedScalar val) { 119 CeedSize length; 120 CeedVector_Memcheck *impl; 121 122 CeedCallBackend(CeedVectorGetData(vec, &impl)); 123 CeedCallBackend(CeedVectorGetLength(vec, &length)); 124 125 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 126 assert(impl->array_allocated); 127 if (stop == -1) stop = length; 128 for (CeedSize i = start; i < stop; i += step) impl->array_allocated[i] = val; 129 return CEED_ERROR_SUCCESS; 130 } 131 132 //------------------------------------------------------------------------------ 133 // Sync arrays 134 //------------------------------------------------------------------------------ 135 static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) { 136 CeedSize length; 137 CeedVector_Memcheck *impl; 138 139 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 140 141 CeedCallBackend(CeedVectorGetData(vec, &impl)); 142 CeedCallBackend(CeedVectorGetLength(vec, &length)); 143 144 // Copy internal buffer back to owned or borrowed array 145 if (impl->array_owned) { 146 memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar)); 147 } 148 if (impl->array_borrowed) { 149 memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar)); 150 } 151 return CEED_ERROR_SUCCESS; 152 } 153 154 //------------------------------------------------------------------------------ 155 // Vector Take Array 156 //------------------------------------------------------------------------------ 157 static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 158 CeedSize length; 159 CeedVector_Memcheck *impl; 160 161 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 162 163 CeedCallBackend(CeedVectorGetData(vec, &impl)); 164 CeedCallBackend(CeedVectorGetLength(vec, &length)); 165 166 // Synchronize memory 167 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 168 169 // Return borrowed array 170 (*array) = impl->array_borrowed; 171 impl->array_borrowed = NULL; 172 VALGRIND_DISCARD(impl->borrowed_block_id); 173 174 // De-allocate internal memory 175 if (impl->array_allocated) { 176 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 177 VALGRIND_DISCARD(impl->allocated_block_id); 178 } 179 CeedCallBackend(CeedFree(&impl->array_allocated)); 180 return CEED_ERROR_SUCCESS; 181 } 182 183 //------------------------------------------------------------------------------ 184 // Vector Get Array 185 //------------------------------------------------------------------------------ 186 static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 187 CeedSize length; 188 CeedVector_Memcheck *impl; 189 190 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 191 192 CeedCallBackend(CeedVectorGetData(vec, &impl)); 193 CeedCallBackend(CeedVectorGetLength(vec, &length)); 194 195 // Create and return writable buffer 196 CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy)); 197 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy"); 198 memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar)); 199 *array = impl->array_writable_copy; 200 return CEED_ERROR_SUCCESS; 201 } 202 203 //------------------------------------------------------------------------------ 204 // Vector Get Array Read 205 //------------------------------------------------------------------------------ 206 static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, const CeedScalar **array) { 207 CeedSize length; 208 CeedVector_Memcheck *impl; 209 210 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 211 212 CeedCallBackend(CeedVectorGetData(vec, &impl)); 213 CeedCallBackend(CeedVectorGetLength(vec, &length)); 214 215 // Create and return read-only buffer 216 if (!impl->array_read_only_copy) { 217 CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy)); 218 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy"); 219 memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar)); 220 } 221 *array = impl->array_read_only_copy; 222 return CEED_ERROR_SUCCESS; 223 } 224 225 //------------------------------------------------------------------------------ 226 // Vector Get Array Write 227 //------------------------------------------------------------------------------ 228 static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 229 CeedSize length; 230 CeedVector_Memcheck *impl; 231 232 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 233 234 CeedCallBackend(CeedVectorGetData(vec, &impl)); 235 CeedCallBackend(CeedVectorGetLength(vec, &length)); 236 237 // Allocate buffer if necessary 238 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); 239 240 // Get writable buffer 241 CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array)); 242 243 // Invalidate array data to prevent accidental reads 244 for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN; 245 impl->is_write_only_access = true; 246 return CEED_ERROR_SUCCESS; 247 } 248 249 //------------------------------------------------------------------------------ 250 // Vector Restore Array 251 //------------------------------------------------------------------------------ 252 static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { 253 CeedSize length; 254 CeedVector_Memcheck *impl; 255 256 CeedCallBackend(CeedVectorGetData(vec, &impl)); 257 CeedCallBackend(CeedVectorGetLength(vec, &length)); 258 259 // Check for unset entries after write-only access 260 if (impl->is_write_only_access) { 261 for (CeedSize i = 0; i < length; i++) { 262 if (isnan(impl->array_writable_copy[i])) { 263 CeedDebug256(CeedVectorReturnCeed(vec), CEED_DEBUG_COLOR_WARNING, 264 "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i); 265 } 266 } 267 impl->is_write_only_access = false; 268 } 269 270 // Copy back to internal buffer and sync 271 memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar)); 272 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 273 274 // Invalidate writable buffer 275 for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; 276 CeedCallBackend(CeedFree(&impl->array_writable_copy)); 277 VALGRIND_DISCARD(impl->writable_block_id); 278 return CEED_ERROR_SUCCESS; 279 } 280 281 //------------------------------------------------------------------------------ 282 // Vector Restore Array Read-Only 283 //------------------------------------------------------------------------------ 284 static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { 285 CeedSize length; 286 CeedVector_Memcheck *impl; 287 288 CeedCallBackend(CeedVectorGetData(vec, &impl)); 289 CeedCallBackend(CeedVectorGetLength(vec, &length)); 290 291 // Verify no changes made during read-only access 292 bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar)); 293 294 CeedCheck(!is_changed, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode"); 295 296 // Invalidate read-only buffer 297 for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN; 298 CeedCallBackend(CeedFree(&impl->array_read_only_copy)); 299 VALGRIND_DISCARD(impl->read_only_block_id); 300 return CEED_ERROR_SUCCESS; 301 } 302 303 //------------------------------------------------------------------------------ 304 // Take reciprocal of a vector 305 //------------------------------------------------------------------------------ 306 static int CeedVectorReciprocal_Memcheck(CeedVector vec) { 307 CeedSize length; 308 CeedVector_Memcheck *impl; 309 310 CeedCallBackend(CeedVectorGetData(vec, &impl)); 311 CeedCallBackend(CeedVectorGetLength(vec, &length)); 312 313 for (CeedSize i = 0; i < length; i++) { 314 if (fabs(impl->array_allocated[i]) > CEED_EPSILON) impl->array_allocated[i] = 1. / impl->array_allocated[i]; 315 } 316 return CEED_ERROR_SUCCESS; 317 } 318 319 //------------------------------------------------------------------------------ 320 // Compute x = alpha x 321 //------------------------------------------------------------------------------ 322 static int CeedVectorScale_Memcheck(CeedVector x, CeedScalar alpha) { 323 CeedSize length; 324 CeedVector_Memcheck *impl; 325 326 CeedCallBackend(CeedVectorGetData(x, &impl)); 327 CeedCallBackend(CeedVectorGetLength(x, &length)); 328 329 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] *= alpha; 330 return CEED_ERROR_SUCCESS; 331 } 332 333 //------------------------------------------------------------------------------ 334 // Compute y = alpha x + y 335 //------------------------------------------------------------------------------ 336 static int CeedVectorAXPY_Memcheck(CeedVector y, CeedScalar alpha, CeedVector x) { 337 CeedSize length; 338 CeedVector_Memcheck *impl_x, *impl_y; 339 340 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 341 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 342 CeedCallBackend(CeedVectorGetLength(y, &length)); 343 344 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] += alpha * impl_x->array_allocated[i]; 345 return CEED_ERROR_SUCCESS; 346 } 347 348 //------------------------------------------------------------------------------ 349 // Compute y = alpha x + beta y 350 //------------------------------------------------------------------------------ 351 static int CeedVectorAXPBY_Memcheck(CeedVector y, CeedScalar alpha, CeedScalar beta, CeedVector x) { 352 CeedSize length; 353 CeedVector_Memcheck *impl_x, *impl_y; 354 355 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 356 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 357 CeedCallBackend(CeedVectorGetLength(y, &length)); 358 359 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] = alpha * impl_x->array_allocated[i] + beta * impl_y->array_allocated[i]; 360 return CEED_ERROR_SUCCESS; 361 } 362 363 //------------------------------------------------------------------------------ 364 // Compute the pointwise multiplication w = x .* y 365 //------------------------------------------------------------------------------ 366 static int CeedVectorPointwiseMult_Memcheck(CeedVector w, CeedVector x, CeedVector y) { 367 CeedSize length; 368 CeedVector_Memcheck *impl_x, *impl_y, *impl_w; 369 370 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 371 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 372 CeedCallBackend(CeedVectorGetData(w, &impl_w)); 373 CeedCallBackend(CeedVectorGetLength(w, &length)); 374 375 if (!impl_w->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(w, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 376 assert(impl_w->array_allocated); 377 for (CeedSize i = 0; i < length; i++) impl_w->array_allocated[i] = impl_x->array_allocated[i] * impl_y->array_allocated[i]; 378 return CEED_ERROR_SUCCESS; 379 } 380 381 //------------------------------------------------------------------------------ 382 // Vector Destroy 383 //------------------------------------------------------------------------------ 384 static int CeedVectorDestroy_Memcheck(CeedVector vec) { 385 CeedVector_Memcheck *impl; 386 387 // Free allocations and discard block ids 388 CeedCallBackend(CeedVectorGetData(vec, &impl)); 389 if (impl->array_allocated) { 390 CeedCallBackend(CeedFree(&impl->array_allocated)); 391 VALGRIND_DISCARD(impl->allocated_block_id); 392 } 393 if (impl->array_owned) { 394 CeedCallBackend(CeedFree(&impl->array_owned)); 395 VALGRIND_DISCARD(impl->owned_block_id); 396 } 397 if (impl->array_borrowed) { 398 VALGRIND_DISCARD(impl->borrowed_block_id); 399 } 400 CeedCallBackend(CeedFree(&impl)); 401 return CEED_ERROR_SUCCESS; 402 } 403 404 //------------------------------------------------------------------------------ 405 // Vector Create 406 //------------------------------------------------------------------------------ 407 int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) { 408 Ceed ceed; 409 CeedVector_Memcheck *impl; 410 411 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 412 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck)); 413 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck)); 414 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck)); 415 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValue", CeedVectorSetValue_Memcheck)); 416 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValueStrided", CeedVectorSetValueStrided_Memcheck)); 417 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck)); 418 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck)); 419 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck)); 420 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck)); 421 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck)); 422 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck)); 423 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck)); 424 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Reciprocal", CeedVectorReciprocal_Memcheck)); 425 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Scale", CeedVectorScale_Memcheck)); 426 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPY", CeedVectorAXPY_Memcheck)); 427 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPBY", CeedVectorAXPBY_Memcheck)); 428 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "PointwiseMult", CeedVectorPointwiseMult_Memcheck)); 429 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck)); 430 CeedCallBackend(CeedDestroy(&ceed)); 431 CeedCallBackend(CeedCalloc(1, &impl)); 432 CeedCallBackend(CeedVectorSetData(vec, impl)); 433 return CEED_ERROR_SUCCESS; 434 } 435 436 //------------------------------------------------------------------------------ 437