1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed.h> 9 #include <ceed/backend.h> 10 #include <assert.h> 11 #include <math.h> 12 #include <stdbool.h> 13 #include <string.h> 14 #include <valgrind/memcheck.h> 15 16 #include "ceed-memcheck.h" 17 18 //------------------------------------------------------------------------------ 19 // Has Valid Array 20 //------------------------------------------------------------------------------ 21 static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_array) { 22 CeedVector_Memcheck *impl; 23 24 CeedCallBackend(CeedVectorGetData(vec, &impl)); 25 *has_valid_array = !!impl->array_allocated; 26 return CEED_ERROR_SUCCESS; 27 } 28 29 //------------------------------------------------------------------------------ 30 // Check if has borrowed array of given type 31 //------------------------------------------------------------------------------ 32 static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) { 33 CeedVector_Memcheck *impl; 34 35 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 36 37 CeedCallBackend(CeedVectorGetData(vec, &impl)); 38 *has_borrowed_array_of_type = !!impl->array_borrowed; 39 return CEED_ERROR_SUCCESS; 40 } 41 42 //------------------------------------------------------------------------------ 43 // Vector Set Array 44 //------------------------------------------------------------------------------ 45 static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedCopyMode copy_mode, CeedScalar *array) { 46 CeedSize length; 47 CeedVector_Memcheck *impl; 48 49 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 50 51 CeedCallBackend(CeedVectorGetData(vec, &impl)); 52 CeedCallBackend(CeedVectorGetLength(vec, &length)); 53 54 // Clear previous owned arrays 55 if (impl->array_allocated) { 56 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 57 VALGRIND_DISCARD(impl->allocated_block_id); 58 } 59 CeedCallBackend(CeedFree(&impl->array_allocated)); 60 if (impl->array_owned) { 61 for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN; 62 VALGRIND_DISCARD(impl->owned_block_id); 63 } 64 CeedCallBackend(CeedFree(&impl->array_owned)); 65 66 // Clear borrowed block id, if present 67 if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); 68 69 // Set internal pointers to external arrays 70 switch (copy_mode) { 71 case CEED_COPY_VALUES: 72 impl->array_owned = NULL; 73 impl->array_borrowed = NULL; 74 break; 75 case CEED_OWN_POINTER: 76 impl->array_owned = array; 77 impl->array_borrowed = NULL; 78 impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer"); 79 break; 80 case CEED_USE_POINTER: 81 impl->array_owned = NULL; 82 impl->array_borrowed = array; 83 impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer"); 84 break; 85 } 86 87 // Create internal array data buffer 88 CeedCallBackend(CeedCalloc(length, &impl->array_allocated)); 89 impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer"); 90 if (array) { 91 memcpy(impl->array_allocated, array, length * sizeof(CeedScalar)); 92 } else { 93 for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN; 94 } 95 return CEED_ERROR_SUCCESS; 96 } 97 98 //------------------------------------------------------------------------------ 99 // Set internal array to value 100 //------------------------------------------------------------------------------ 101 static int CeedVectorSetValue_Memcheck(CeedVector vec, CeedScalar value) { 102 CeedSize length; 103 CeedVector_Memcheck *impl; 104 105 CeedCallBackend(CeedVectorGetData(vec, &impl)); 106 CeedCallBackend(CeedVectorGetLength(vec, &length)); 107 108 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 109 assert(impl->array_allocated); 110 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = value; 111 return CEED_ERROR_SUCCESS; 112 } 113 114 //------------------------------------------------------------------------------ 115 // Set internal array to value strided 116 //------------------------------------------------------------------------------ 117 static int CeedVectorSetValueStrided_Memcheck(CeedVector vec, CeedSize start, CeedSize step, CeedScalar val) { 118 CeedSize length; 119 CeedVector_Memcheck *impl; 120 121 CeedCallBackend(CeedVectorGetData(vec, &impl)); 122 CeedCallBackend(CeedVectorGetLength(vec, &length)); 123 124 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 125 assert(impl->array_allocated); 126 for (CeedSize i = start; i < length; i += step) impl->array_allocated[i] = val; 127 return CEED_ERROR_SUCCESS; 128 } 129 130 //------------------------------------------------------------------------------ 131 // Sync arrays 132 //------------------------------------------------------------------------------ 133 static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) { 134 CeedSize length; 135 CeedVector_Memcheck *impl; 136 137 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 138 139 CeedCallBackend(CeedVectorGetData(vec, &impl)); 140 CeedCallBackend(CeedVectorGetLength(vec, &length)); 141 142 // Copy internal buffer back to owned or borrowed array 143 if (impl->array_owned) { 144 memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar)); 145 } 146 if (impl->array_borrowed) { 147 memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar)); 148 } 149 return CEED_ERROR_SUCCESS; 150 } 151 152 //------------------------------------------------------------------------------ 153 // Vector Take Array 154 //------------------------------------------------------------------------------ 155 static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 156 CeedSize length; 157 CeedVector_Memcheck *impl; 158 159 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 160 161 CeedCallBackend(CeedVectorGetData(vec, &impl)); 162 CeedCallBackend(CeedVectorGetLength(vec, &length)); 163 164 // Synchronize memory 165 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 166 167 // Return borrowed array 168 (*array) = impl->array_borrowed; 169 impl->array_borrowed = NULL; 170 VALGRIND_DISCARD(impl->borrowed_block_id); 171 172 // De-allocate internal memory 173 if (impl->array_allocated) { 174 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 175 VALGRIND_DISCARD(impl->allocated_block_id); 176 } 177 CeedCallBackend(CeedFree(&impl->array_allocated)); 178 return CEED_ERROR_SUCCESS; 179 } 180 181 //------------------------------------------------------------------------------ 182 // Vector Get Array 183 //------------------------------------------------------------------------------ 184 static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 185 CeedSize length; 186 CeedVector_Memcheck *impl; 187 188 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 189 190 CeedCallBackend(CeedVectorGetData(vec, &impl)); 191 CeedCallBackend(CeedVectorGetLength(vec, &length)); 192 193 // Create and return writable buffer 194 CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy)); 195 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy"); 196 memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar)); 197 *array = impl->array_writable_copy; 198 return CEED_ERROR_SUCCESS; 199 } 200 201 //------------------------------------------------------------------------------ 202 // Vector Get Array Read 203 //------------------------------------------------------------------------------ 204 static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, const CeedScalar **array) { 205 CeedSize length; 206 CeedVector_Memcheck *impl; 207 208 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 209 210 CeedCallBackend(CeedVectorGetData(vec, &impl)); 211 CeedCallBackend(CeedVectorGetLength(vec, &length)); 212 213 // Create and return read-only buffer 214 if (!impl->array_read_only_copy) { 215 CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy)); 216 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy"); 217 memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar)); 218 } 219 *array = impl->array_read_only_copy; 220 return CEED_ERROR_SUCCESS; 221 } 222 223 //------------------------------------------------------------------------------ 224 // Vector Get Array Write 225 //------------------------------------------------------------------------------ 226 static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 227 CeedSize length; 228 CeedVector_Memcheck *impl; 229 230 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 231 232 CeedCallBackend(CeedVectorGetData(vec, &impl)); 233 CeedCallBackend(CeedVectorGetLength(vec, &length)); 234 235 // Allocate buffer if necessary 236 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); 237 238 // Get writable buffer 239 CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array)); 240 241 // Invalidate array data to prevent accidental reads 242 for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN; 243 impl->is_write_only_access = true; 244 return CEED_ERROR_SUCCESS; 245 } 246 247 //------------------------------------------------------------------------------ 248 // Vector Restore Array 249 //------------------------------------------------------------------------------ 250 static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { 251 Ceed ceed; 252 CeedSize length; 253 CeedVector_Memcheck *impl; 254 255 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 256 CeedCallBackend(CeedVectorGetData(vec, &impl)); 257 CeedCallBackend(CeedVectorGetLength(vec, &length)); 258 259 // Check for unset entries after write-only access 260 if (impl->is_write_only_access) { 261 for (CeedSize i = 0; i < length; i++) { 262 if (isnan(impl->array_writable_copy[i])) { 263 CeedDebug256(ceed, CEED_DEBUG_COLOR_WARNING, "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i); 264 } 265 } 266 impl->is_write_only_access = false; 267 } 268 269 // Copy back to internal buffer and sync 270 memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar)); 271 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 272 273 // Invalidate writable buffer 274 for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; 275 CeedCallBackend(CeedFree(&impl->array_writable_copy)); 276 VALGRIND_DISCARD(impl->writable_block_id); 277 return CEED_ERROR_SUCCESS; 278 } 279 280 //------------------------------------------------------------------------------ 281 // Vector Restore Array Read-Only 282 //------------------------------------------------------------------------------ 283 static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { 284 Ceed ceed; 285 CeedSize length; 286 CeedVector_Memcheck *impl; 287 288 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 289 CeedCallBackend(CeedVectorGetData(vec, &impl)); 290 CeedCallBackend(CeedVectorGetLength(vec, &length)); 291 292 // Verify no changes made during read-only access 293 bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar)); 294 295 CeedCheck(!is_changed, ceed, CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode"); 296 297 // Invalidate read-only buffer 298 for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN; 299 CeedCallBackend(CeedFree(&impl->array_read_only_copy)); 300 VALGRIND_DISCARD(impl->read_only_block_id); 301 return CEED_ERROR_SUCCESS; 302 } 303 304 //------------------------------------------------------------------------------ 305 // Take reciprocal of a vector 306 //------------------------------------------------------------------------------ 307 static int CeedVectorReciprocal_Memcheck(CeedVector vec) { 308 CeedSize length; 309 CeedVector_Memcheck *impl; 310 311 CeedCallBackend(CeedVectorGetData(vec, &impl)); 312 CeedCallBackend(CeedVectorGetLength(vec, &length)); 313 314 for (CeedSize i = 0; i < length; i++) { 315 if (fabs(impl->array_allocated[i]) > CEED_EPSILON) impl->array_allocated[i] = 1. / impl->array_allocated[i]; 316 } 317 return CEED_ERROR_SUCCESS; 318 } 319 320 //------------------------------------------------------------------------------ 321 // Compute x = alpha x 322 //------------------------------------------------------------------------------ 323 static int CeedVectorScale_Memcheck(CeedVector x, CeedScalar alpha) { 324 CeedSize length; 325 CeedVector_Memcheck *impl; 326 327 CeedCallBackend(CeedVectorGetData(x, &impl)); 328 CeedCallBackend(CeedVectorGetLength(x, &length)); 329 330 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] *= alpha; 331 return CEED_ERROR_SUCCESS; 332 } 333 334 //------------------------------------------------------------------------------ 335 // Compute y = alpha x + y 336 //------------------------------------------------------------------------------ 337 static int CeedVectorAXPY_Memcheck(CeedVector y, CeedScalar alpha, CeedVector x) { 338 CeedSize length; 339 CeedVector_Memcheck *impl_x, *impl_y; 340 341 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 342 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 343 CeedCallBackend(CeedVectorGetLength(y, &length)); 344 345 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] += alpha * impl_x->array_allocated[i]; 346 return CEED_ERROR_SUCCESS; 347 } 348 349 //------------------------------------------------------------------------------ 350 // Compute y = alpha x + beta y 351 //------------------------------------------------------------------------------ 352 static int CeedVectorAXPBY_Memcheck(CeedVector y, CeedScalar alpha, CeedScalar beta, CeedVector x) { 353 CeedSize length; 354 CeedVector_Memcheck *impl_x, *impl_y; 355 356 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 357 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 358 CeedCallBackend(CeedVectorGetLength(y, &length)); 359 360 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] = alpha * impl_x->array_allocated[i] + beta * impl_y->array_allocated[i]; 361 return CEED_ERROR_SUCCESS; 362 } 363 364 //------------------------------------------------------------------------------ 365 // Compute the pointwise multiplication w = x .* y 366 //------------------------------------------------------------------------------ 367 static int CeedVectorPointwiseMult_Memcheck(CeedVector w, CeedVector x, CeedVector y) { 368 CeedSize length; 369 CeedVector_Memcheck *impl_x, *impl_y, *impl_w; 370 371 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 372 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 373 CeedCallBackend(CeedVectorGetData(w, &impl_w)); 374 CeedCallBackend(CeedVectorGetLength(w, &length)); 375 376 if (!impl_w->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(w, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 377 assert(impl_w->array_allocated); 378 for (CeedSize i = 0; i < length; i++) impl_w->array_allocated[i] = impl_x->array_allocated[i] * impl_y->array_allocated[i]; 379 return CEED_ERROR_SUCCESS; 380 } 381 382 //------------------------------------------------------------------------------ 383 // Vector Destroy 384 //------------------------------------------------------------------------------ 385 static int CeedVectorDestroy_Memcheck(CeedVector vec) { 386 CeedVector_Memcheck *impl; 387 388 // Free allocations and discard block ids 389 CeedCallBackend(CeedVectorGetData(vec, &impl)); 390 if (impl->array_allocated) { 391 CeedCallBackend(CeedFree(&impl->array_allocated)); 392 VALGRIND_DISCARD(impl->allocated_block_id); 393 } 394 if (impl->array_owned) { 395 CeedCallBackend(CeedFree(&impl->array_owned)); 396 VALGRIND_DISCARD(impl->owned_block_id); 397 } 398 if (impl->array_borrowed) { 399 VALGRIND_DISCARD(impl->borrowed_block_id); 400 } 401 CeedCallBackend(CeedFree(&impl)); 402 return CEED_ERROR_SUCCESS; 403 } 404 405 //------------------------------------------------------------------------------ 406 // Vector Create 407 //------------------------------------------------------------------------------ 408 int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) { 409 Ceed ceed; 410 CeedVector_Memcheck *impl; 411 412 CeedCallBackend(CeedCalloc(1, &impl)); 413 CeedCallBackend(CeedVectorSetData(vec, impl)); 414 415 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 416 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck)); 417 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck)); 418 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck)); 419 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValue", CeedVectorSetValue_Memcheck)); 420 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValueStrided", CeedVectorSetValueStrided_Memcheck)); 421 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck)); 422 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck)); 423 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck)); 424 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck)); 425 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck)); 426 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck)); 427 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck)); 428 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Reciprocal", CeedVectorReciprocal_Memcheck)); 429 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Scale", CeedVectorScale_Memcheck)); 430 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPY", CeedVectorAXPY_Memcheck)); 431 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPBY", CeedVectorAXPBY_Memcheck)); 432 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "PointwiseMult", CeedVectorPointwiseMult_Memcheck)); 433 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck)); 434 return CEED_ERROR_SUCCESS; 435 } 436 437 //------------------------------------------------------------------------------ 438