1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed.h> 9 #include <ceed/backend.h> 10 #include <assert.h> 11 #include <math.h> 12 #include <stdbool.h> 13 #include <string.h> 14 #include <valgrind/memcheck.h> 15 16 #include "ceed-memcheck.h" 17 18 //------------------------------------------------------------------------------ 19 // Has Valid Array 20 //------------------------------------------------------------------------------ 21 static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_array) { 22 CeedVector_Memcheck *impl; 23 24 CeedCallBackend(CeedVectorGetData(vec, &impl)); 25 *has_valid_array = !!impl->array_allocated; 26 return CEED_ERROR_SUCCESS; 27 } 28 29 //------------------------------------------------------------------------------ 30 // Check if has borrowed array of given type 31 //------------------------------------------------------------------------------ 32 static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) { 33 CeedVector_Memcheck *impl; 34 35 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 36 37 CeedCallBackend(CeedVectorGetData(vec, &impl)); 38 *has_borrowed_array_of_type = !!impl->array_borrowed; 39 return CEED_ERROR_SUCCESS; 40 } 41 42 //------------------------------------------------------------------------------ 43 // Vector Set Array 44 //------------------------------------------------------------------------------ 45 static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedCopyMode copy_mode, CeedScalar *array) { 46 CeedSize length; 47 CeedVector_Memcheck *impl; 48 49 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 50 51 CeedCallBackend(CeedVectorGetData(vec, &impl)); 52 CeedCallBackend(CeedVectorGetLength(vec, &length)); 53 54 // Clear previous owned arrays 55 if (impl->array_allocated) { 56 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 57 VALGRIND_DISCARD(impl->allocated_block_id); 58 } 59 CeedCallBackend(CeedFree(&impl->array_allocated)); 60 if (copy_mode != CEED_COPY_VALUES) { 61 if (impl->array_owned) { 62 for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN; 63 VALGRIND_DISCARD(impl->owned_block_id); 64 } 65 CeedCallBackend(CeedFree(&impl->array_owned)); 66 } 67 68 // Clear borrowed block id, if present 69 if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); 70 71 // Set internal pointers to external arrays 72 switch (copy_mode) { 73 case CEED_COPY_VALUES: 74 // Nothing to update 75 break; 76 case CEED_OWN_POINTER: 77 impl->array_owned = array; 78 impl->array_borrowed = NULL; 79 impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer"); 80 break; 81 case CEED_USE_POINTER: 82 impl->array_owned = NULL; 83 impl->array_borrowed = array; 84 impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer"); 85 break; 86 } 87 88 // Create internal array data buffer 89 CeedCallBackend(CeedCalloc(length, &impl->array_allocated)); 90 impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer"); 91 if (array) { 92 memcpy(impl->array_allocated, array, length * sizeof(CeedScalar)); 93 } else { 94 for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN; 95 } 96 return CEED_ERROR_SUCCESS; 97 } 98 99 //------------------------------------------------------------------------------ 100 // Set internal array to value 101 //------------------------------------------------------------------------------ 102 static int CeedVectorSetValue_Memcheck(CeedVector vec, CeedScalar value) { 103 CeedSize length; 104 CeedVector_Memcheck *impl; 105 106 CeedCallBackend(CeedVectorGetData(vec, &impl)); 107 CeedCallBackend(CeedVectorGetLength(vec, &length)); 108 109 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 110 assert(impl->array_allocated); 111 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = value; 112 return CEED_ERROR_SUCCESS; 113 } 114 115 //------------------------------------------------------------------------------ 116 // Set internal array to value strided 117 //------------------------------------------------------------------------------ 118 static int CeedVectorSetValueStrided_Memcheck(CeedVector vec, CeedSize start, CeedSize step, CeedScalar val) { 119 CeedSize length; 120 CeedVector_Memcheck *impl; 121 122 CeedCallBackend(CeedVectorGetData(vec, &impl)); 123 CeedCallBackend(CeedVectorGetLength(vec, &length)); 124 125 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 126 assert(impl->array_allocated); 127 for (CeedSize i = start; i < length; i += step) impl->array_allocated[i] = val; 128 return CEED_ERROR_SUCCESS; 129 } 130 131 //------------------------------------------------------------------------------ 132 // Sync arrays 133 //------------------------------------------------------------------------------ 134 static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) { 135 CeedSize length; 136 CeedVector_Memcheck *impl; 137 138 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 139 140 CeedCallBackend(CeedVectorGetData(vec, &impl)); 141 CeedCallBackend(CeedVectorGetLength(vec, &length)); 142 143 // Copy internal buffer back to owned or borrowed array 144 if (impl->array_owned) { 145 memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar)); 146 } 147 if (impl->array_borrowed) { 148 memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar)); 149 } 150 return CEED_ERROR_SUCCESS; 151 } 152 153 //------------------------------------------------------------------------------ 154 // Vector Take Array 155 //------------------------------------------------------------------------------ 156 static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 157 CeedSize length; 158 CeedVector_Memcheck *impl; 159 160 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 161 162 CeedCallBackend(CeedVectorGetData(vec, &impl)); 163 CeedCallBackend(CeedVectorGetLength(vec, &length)); 164 165 // Synchronize memory 166 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 167 168 // Return borrowed array 169 (*array) = impl->array_borrowed; 170 impl->array_borrowed = NULL; 171 VALGRIND_DISCARD(impl->borrowed_block_id); 172 173 // De-allocate internal memory 174 if (impl->array_allocated) { 175 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 176 VALGRIND_DISCARD(impl->allocated_block_id); 177 } 178 CeedCallBackend(CeedFree(&impl->array_allocated)); 179 return CEED_ERROR_SUCCESS; 180 } 181 182 //------------------------------------------------------------------------------ 183 // Vector Get Array 184 //------------------------------------------------------------------------------ 185 static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 186 CeedSize length; 187 CeedVector_Memcheck *impl; 188 189 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 190 191 CeedCallBackend(CeedVectorGetData(vec, &impl)); 192 CeedCallBackend(CeedVectorGetLength(vec, &length)); 193 194 // Create and return writable buffer 195 CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy)); 196 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy"); 197 memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar)); 198 *array = impl->array_writable_copy; 199 return CEED_ERROR_SUCCESS; 200 } 201 202 //------------------------------------------------------------------------------ 203 // Vector Get Array Read 204 //------------------------------------------------------------------------------ 205 static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, const CeedScalar **array) { 206 CeedSize length; 207 CeedVector_Memcheck *impl; 208 209 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 210 211 CeedCallBackend(CeedVectorGetData(vec, &impl)); 212 CeedCallBackend(CeedVectorGetLength(vec, &length)); 213 214 // Create and return read-only buffer 215 if (!impl->array_read_only_copy) { 216 CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy)); 217 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy"); 218 memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar)); 219 } 220 *array = impl->array_read_only_copy; 221 return CEED_ERROR_SUCCESS; 222 } 223 224 //------------------------------------------------------------------------------ 225 // Vector Get Array Write 226 //------------------------------------------------------------------------------ 227 static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 228 CeedSize length; 229 CeedVector_Memcheck *impl; 230 231 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 232 233 CeedCallBackend(CeedVectorGetData(vec, &impl)); 234 CeedCallBackend(CeedVectorGetLength(vec, &length)); 235 236 // Allocate buffer if necessary 237 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); 238 239 // Get writable buffer 240 CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array)); 241 242 // Invalidate array data to prevent accidental reads 243 for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN; 244 impl->is_write_only_access = true; 245 return CEED_ERROR_SUCCESS; 246 } 247 248 //------------------------------------------------------------------------------ 249 // Vector Restore Array 250 //------------------------------------------------------------------------------ 251 static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { 252 CeedSize length; 253 CeedVector_Memcheck *impl; 254 255 CeedCallBackend(CeedVectorGetData(vec, &impl)); 256 CeedCallBackend(CeedVectorGetLength(vec, &length)); 257 258 // Check for unset entries after write-only access 259 if (impl->is_write_only_access) { 260 for (CeedSize i = 0; i < length; i++) { 261 if (isnan(impl->array_writable_copy[i])) { 262 CeedDebug256(CeedVectorReturnCeed(vec), CEED_DEBUG_COLOR_WARNING, 263 "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i); 264 } 265 } 266 impl->is_write_only_access = false; 267 } 268 269 // Copy back to internal buffer and sync 270 memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar)); 271 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 272 273 // Invalidate writable buffer 274 for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; 275 CeedCallBackend(CeedFree(&impl->array_writable_copy)); 276 VALGRIND_DISCARD(impl->writable_block_id); 277 return CEED_ERROR_SUCCESS; 278 } 279 280 //------------------------------------------------------------------------------ 281 // Vector Restore Array Read-Only 282 //------------------------------------------------------------------------------ 283 static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { 284 CeedSize length; 285 CeedVector_Memcheck *impl; 286 287 CeedCallBackend(CeedVectorGetData(vec, &impl)); 288 CeedCallBackend(CeedVectorGetLength(vec, &length)); 289 290 // Verify no changes made during read-only access 291 bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar)); 292 293 CeedCheck(!is_changed, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode"); 294 295 // Invalidate read-only buffer 296 for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN; 297 CeedCallBackend(CeedFree(&impl->array_read_only_copy)); 298 VALGRIND_DISCARD(impl->read_only_block_id); 299 return CEED_ERROR_SUCCESS; 300 } 301 302 //------------------------------------------------------------------------------ 303 // Take reciprocal of a vector 304 //------------------------------------------------------------------------------ 305 static int CeedVectorReciprocal_Memcheck(CeedVector vec) { 306 CeedSize length; 307 CeedVector_Memcheck *impl; 308 309 CeedCallBackend(CeedVectorGetData(vec, &impl)); 310 CeedCallBackend(CeedVectorGetLength(vec, &length)); 311 312 for (CeedSize i = 0; i < length; i++) { 313 if (fabs(impl->array_allocated[i]) > CEED_EPSILON) impl->array_allocated[i] = 1. / impl->array_allocated[i]; 314 } 315 return CEED_ERROR_SUCCESS; 316 } 317 318 //------------------------------------------------------------------------------ 319 // Compute x = alpha x 320 //------------------------------------------------------------------------------ 321 static int CeedVectorScale_Memcheck(CeedVector x, CeedScalar alpha) { 322 CeedSize length; 323 CeedVector_Memcheck *impl; 324 325 CeedCallBackend(CeedVectorGetData(x, &impl)); 326 CeedCallBackend(CeedVectorGetLength(x, &length)); 327 328 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] *= alpha; 329 return CEED_ERROR_SUCCESS; 330 } 331 332 //------------------------------------------------------------------------------ 333 // Compute y = alpha x + y 334 //------------------------------------------------------------------------------ 335 static int CeedVectorAXPY_Memcheck(CeedVector y, CeedScalar alpha, CeedVector x) { 336 CeedSize length; 337 CeedVector_Memcheck *impl_x, *impl_y; 338 339 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 340 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 341 CeedCallBackend(CeedVectorGetLength(y, &length)); 342 343 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] += alpha * impl_x->array_allocated[i]; 344 return CEED_ERROR_SUCCESS; 345 } 346 347 //------------------------------------------------------------------------------ 348 // Compute y = alpha x + beta y 349 //------------------------------------------------------------------------------ 350 static int CeedVectorAXPBY_Memcheck(CeedVector y, CeedScalar alpha, CeedScalar beta, CeedVector x) { 351 CeedSize length; 352 CeedVector_Memcheck *impl_x, *impl_y; 353 354 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 355 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 356 CeedCallBackend(CeedVectorGetLength(y, &length)); 357 358 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] = alpha * impl_x->array_allocated[i] + beta * impl_y->array_allocated[i]; 359 return CEED_ERROR_SUCCESS; 360 } 361 362 //------------------------------------------------------------------------------ 363 // Compute the pointwise multiplication w = x .* y 364 //------------------------------------------------------------------------------ 365 static int CeedVectorPointwiseMult_Memcheck(CeedVector w, CeedVector x, CeedVector y) { 366 CeedSize length; 367 CeedVector_Memcheck *impl_x, *impl_y, *impl_w; 368 369 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 370 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 371 CeedCallBackend(CeedVectorGetData(w, &impl_w)); 372 CeedCallBackend(CeedVectorGetLength(w, &length)); 373 374 if (!impl_w->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(w, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 375 assert(impl_w->array_allocated); 376 for (CeedSize i = 0; i < length; i++) impl_w->array_allocated[i] = impl_x->array_allocated[i] * impl_y->array_allocated[i]; 377 return CEED_ERROR_SUCCESS; 378 } 379 380 //------------------------------------------------------------------------------ 381 // Vector Destroy 382 //------------------------------------------------------------------------------ 383 static int CeedVectorDestroy_Memcheck(CeedVector vec) { 384 CeedVector_Memcheck *impl; 385 386 // Free allocations and discard block ids 387 CeedCallBackend(CeedVectorGetData(vec, &impl)); 388 if (impl->array_allocated) { 389 CeedCallBackend(CeedFree(&impl->array_allocated)); 390 VALGRIND_DISCARD(impl->allocated_block_id); 391 } 392 if (impl->array_owned) { 393 CeedCallBackend(CeedFree(&impl->array_owned)); 394 VALGRIND_DISCARD(impl->owned_block_id); 395 } 396 if (impl->array_borrowed) { 397 VALGRIND_DISCARD(impl->borrowed_block_id); 398 } 399 CeedCallBackend(CeedFree(&impl)); 400 return CEED_ERROR_SUCCESS; 401 } 402 403 //------------------------------------------------------------------------------ 404 // Vector Create 405 //------------------------------------------------------------------------------ 406 int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) { 407 Ceed ceed; 408 CeedVector_Memcheck *impl; 409 410 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 411 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck)); 412 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck)); 413 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck)); 414 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValue", CeedVectorSetValue_Memcheck)); 415 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValueStrided", CeedVectorSetValueStrided_Memcheck)); 416 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck)); 417 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck)); 418 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck)); 419 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck)); 420 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck)); 421 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck)); 422 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck)); 423 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Reciprocal", CeedVectorReciprocal_Memcheck)); 424 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Scale", CeedVectorScale_Memcheck)); 425 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPY", CeedVectorAXPY_Memcheck)); 426 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPBY", CeedVectorAXPBY_Memcheck)); 427 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "PointwiseMult", CeedVectorPointwiseMult_Memcheck)); 428 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck)); 429 CeedCallBackend(CeedDestroy(&ceed)); 430 CeedCallBackend(CeedCalloc(1, &impl)); 431 CeedCallBackend(CeedVectorSetData(vec, impl)); 432 return CEED_ERROR_SUCCESS; 433 } 434 435 //------------------------------------------------------------------------------ 436