1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed.h> 9 #include <ceed/backend.h> 10 #include <assert.h> 11 #include <math.h> 12 #include <stdbool.h> 13 #include <string.h> 14 #include <valgrind/memcheck.h> 15 16 #include "ceed-memcheck.h" 17 18 //------------------------------------------------------------------------------ 19 // Has Valid Array 20 //------------------------------------------------------------------------------ 21 static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_array) { 22 CeedVector_Memcheck *impl; 23 24 CeedCallBackend(CeedVectorGetData(vec, &impl)); 25 *has_valid_array = !!impl->array_allocated; 26 return CEED_ERROR_SUCCESS; 27 } 28 29 //------------------------------------------------------------------------------ 30 // Check if has borrowed array of given type 31 //------------------------------------------------------------------------------ 32 static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) { 33 CeedVector_Memcheck *impl; 34 35 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 36 37 CeedCallBackend(CeedVectorGetData(vec, &impl)); 38 *has_borrowed_array_of_type = !!impl->array_borrowed; 39 return CEED_ERROR_SUCCESS; 40 } 41 42 //------------------------------------------------------------------------------ 43 // Vector Set Array 44 //------------------------------------------------------------------------------ 45 static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedCopyMode copy_mode, CeedScalar *array) { 46 CeedSize length; 47 CeedVector_Memcheck *impl; 48 49 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 50 51 CeedCallBackend(CeedVectorGetData(vec, &impl)); 52 CeedCallBackend(CeedVectorGetLength(vec, &length)); 53 54 // Clear previous owned arrays 55 if (impl->array_allocated) { 56 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 57 VALGRIND_DISCARD(impl->allocated_block_id); 58 } 59 CeedCallBackend(CeedFree(&impl->array_allocated)); 60 if (impl->array_owned) { 61 for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN; 62 VALGRIND_DISCARD(impl->owned_block_id); 63 } 64 CeedCallBackend(CeedFree(&impl->array_owned)); 65 66 // Clear borrowed block id, if present 67 if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); 68 69 // Set internal pointers to external arrays 70 switch (copy_mode) { 71 case CEED_COPY_VALUES: 72 impl->array_owned = NULL; 73 impl->array_borrowed = NULL; 74 break; 75 case CEED_OWN_POINTER: 76 impl->array_owned = array; 77 impl->array_borrowed = NULL; 78 impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer"); 79 break; 80 case CEED_USE_POINTER: 81 impl->array_owned = NULL; 82 impl->array_borrowed = array; 83 impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer"); 84 break; 85 } 86 87 // Create internal array data buffer 88 CeedCallBackend(CeedCalloc(length, &impl->array_allocated)); 89 impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer"); 90 if (array) { 91 memcpy(impl->array_allocated, array, length * sizeof(CeedScalar)); 92 } else { 93 for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN; 94 } 95 return CEED_ERROR_SUCCESS; 96 } 97 98 //------------------------------------------------------------------------------ 99 // Set internal array to value 100 //------------------------------------------------------------------------------ 101 static int CeedVectorSetValue_Memcheck(CeedVector vec, CeedScalar value) { 102 CeedSize length; 103 CeedVector_Memcheck *impl; 104 105 CeedCallBackend(CeedVectorGetData(vec, &impl)); 106 CeedCallBackend(CeedVectorGetLength(vec, &length)); 107 108 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 109 assert(impl->array_allocated); 110 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = value; 111 return CEED_ERROR_SUCCESS; 112 } 113 114 //------------------------------------------------------------------------------ 115 // Set internal array to value strided 116 //------------------------------------------------------------------------------ 117 static int CeedVectorSetValueStrided_Memcheck(CeedVector vec, CeedSize start, CeedSize step, CeedScalar val) { 118 CeedSize length; 119 CeedVector_Memcheck *impl; 120 121 CeedCallBackend(CeedVectorGetData(vec, &impl)); 122 CeedCallBackend(CeedVectorGetLength(vec, &length)); 123 124 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 125 assert(impl->array_allocated); 126 for (CeedSize i = start; i < length; i += step) impl->array_allocated[i] = val; 127 return CEED_ERROR_SUCCESS; 128 } 129 130 //------------------------------------------------------------------------------ 131 // Sync arrays 132 //------------------------------------------------------------------------------ 133 static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) { 134 CeedSize length; 135 CeedVector_Memcheck *impl; 136 137 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 138 139 CeedCallBackend(CeedVectorGetData(vec, &impl)); 140 CeedCallBackend(CeedVectorGetLength(vec, &length)); 141 142 // Copy internal buffer back to owned or borrowed array 143 if (impl->array_owned) { 144 memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar)); 145 } 146 if (impl->array_borrowed) { 147 memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar)); 148 } 149 return CEED_ERROR_SUCCESS; 150 } 151 152 //------------------------------------------------------------------------------ 153 // Vector Take Array 154 //------------------------------------------------------------------------------ 155 static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 156 CeedSize length; 157 CeedVector_Memcheck *impl; 158 159 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 160 161 CeedCallBackend(CeedVectorGetData(vec, &impl)); 162 CeedCallBackend(CeedVectorGetLength(vec, &length)); 163 164 // Synchronize memory 165 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 166 167 // Return borrowed array 168 (*array) = impl->array_borrowed; 169 impl->array_borrowed = NULL; 170 VALGRIND_DISCARD(impl->borrowed_block_id); 171 172 // De-allocate internal memory 173 if (impl->array_allocated) { 174 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 175 VALGRIND_DISCARD(impl->allocated_block_id); 176 } 177 CeedCallBackend(CeedFree(&impl->array_allocated)); 178 return CEED_ERROR_SUCCESS; 179 } 180 181 //------------------------------------------------------------------------------ 182 // Vector Get Array 183 //------------------------------------------------------------------------------ 184 static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 185 CeedSize length; 186 CeedVector_Memcheck *impl; 187 188 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 189 190 CeedCallBackend(CeedVectorGetData(vec, &impl)); 191 CeedCallBackend(CeedVectorGetLength(vec, &length)); 192 193 // Create and return writable buffer 194 CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy)); 195 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy"); 196 memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar)); 197 *array = impl->array_writable_copy; 198 return CEED_ERROR_SUCCESS; 199 } 200 201 //------------------------------------------------------------------------------ 202 // Vector Get Array Read 203 //------------------------------------------------------------------------------ 204 static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, const CeedScalar **array) { 205 CeedSize length; 206 CeedVector_Memcheck *impl; 207 208 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 209 210 CeedCallBackend(CeedVectorGetData(vec, &impl)); 211 CeedCallBackend(CeedVectorGetLength(vec, &length)); 212 213 // Create and return read-only buffer 214 if (!impl->array_read_only_copy) { 215 CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy)); 216 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy"); 217 memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar)); 218 } 219 *array = impl->array_read_only_copy; 220 return CEED_ERROR_SUCCESS; 221 } 222 223 //------------------------------------------------------------------------------ 224 // Vector Get Array Write 225 //------------------------------------------------------------------------------ 226 static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 227 CeedSize length; 228 CeedVector_Memcheck *impl; 229 230 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 231 232 CeedCallBackend(CeedVectorGetData(vec, &impl)); 233 CeedCallBackend(CeedVectorGetLength(vec, &length)); 234 235 // Allocate buffer if necessary 236 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); 237 238 // Get writable buffer 239 CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array)); 240 241 // Invalidate array data to prevent accidental reads 242 for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN; 243 impl->is_write_only_access = true; 244 return CEED_ERROR_SUCCESS; 245 } 246 247 //------------------------------------------------------------------------------ 248 // Vector Restore Array 249 //------------------------------------------------------------------------------ 250 static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { 251 CeedSize length; 252 CeedVector_Memcheck *impl; 253 254 CeedCallBackend(CeedVectorGetData(vec, &impl)); 255 CeedCallBackend(CeedVectorGetLength(vec, &length)); 256 257 // Check for unset entries after write-only access 258 if (impl->is_write_only_access) { 259 for (CeedSize i = 0; i < length; i++) { 260 if (isnan(impl->array_writable_copy[i])) { 261 CeedDebug256(CeedVectorReturnCeed(vec), CEED_DEBUG_COLOR_WARNING, 262 "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i); 263 } 264 } 265 impl->is_write_only_access = false; 266 } 267 268 // Copy back to internal buffer and sync 269 memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar)); 270 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 271 272 // Invalidate writable buffer 273 for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; 274 CeedCallBackend(CeedFree(&impl->array_writable_copy)); 275 VALGRIND_DISCARD(impl->writable_block_id); 276 return CEED_ERROR_SUCCESS; 277 } 278 279 //------------------------------------------------------------------------------ 280 // Vector Restore Array Read-Only 281 //------------------------------------------------------------------------------ 282 static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { 283 CeedSize length; 284 CeedVector_Memcheck *impl; 285 286 CeedCallBackend(CeedVectorGetData(vec, &impl)); 287 CeedCallBackend(CeedVectorGetLength(vec, &length)); 288 289 // Verify no changes made during read-only access 290 bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar)); 291 292 CeedCheck(!is_changed, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode"); 293 294 // Invalidate read-only buffer 295 for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN; 296 CeedCallBackend(CeedFree(&impl->array_read_only_copy)); 297 VALGRIND_DISCARD(impl->read_only_block_id); 298 return CEED_ERROR_SUCCESS; 299 } 300 301 //------------------------------------------------------------------------------ 302 // Take reciprocal of a vector 303 //------------------------------------------------------------------------------ 304 static int CeedVectorReciprocal_Memcheck(CeedVector vec) { 305 CeedSize length; 306 CeedVector_Memcheck *impl; 307 308 CeedCallBackend(CeedVectorGetData(vec, &impl)); 309 CeedCallBackend(CeedVectorGetLength(vec, &length)); 310 311 for (CeedSize i = 0; i < length; i++) { 312 if (fabs(impl->array_allocated[i]) > CEED_EPSILON) impl->array_allocated[i] = 1. / impl->array_allocated[i]; 313 } 314 return CEED_ERROR_SUCCESS; 315 } 316 317 //------------------------------------------------------------------------------ 318 // Compute x = alpha x 319 //------------------------------------------------------------------------------ 320 static int CeedVectorScale_Memcheck(CeedVector x, CeedScalar alpha) { 321 CeedSize length; 322 CeedVector_Memcheck *impl; 323 324 CeedCallBackend(CeedVectorGetData(x, &impl)); 325 CeedCallBackend(CeedVectorGetLength(x, &length)); 326 327 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] *= alpha; 328 return CEED_ERROR_SUCCESS; 329 } 330 331 //------------------------------------------------------------------------------ 332 // Compute y = alpha x + y 333 //------------------------------------------------------------------------------ 334 static int CeedVectorAXPY_Memcheck(CeedVector y, CeedScalar alpha, CeedVector x) { 335 CeedSize length; 336 CeedVector_Memcheck *impl_x, *impl_y; 337 338 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 339 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 340 CeedCallBackend(CeedVectorGetLength(y, &length)); 341 342 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] += alpha * impl_x->array_allocated[i]; 343 return CEED_ERROR_SUCCESS; 344 } 345 346 //------------------------------------------------------------------------------ 347 // Compute y = alpha x + beta y 348 //------------------------------------------------------------------------------ 349 static int CeedVectorAXPBY_Memcheck(CeedVector y, CeedScalar alpha, CeedScalar beta, CeedVector x) { 350 CeedSize length; 351 CeedVector_Memcheck *impl_x, *impl_y; 352 353 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 354 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 355 CeedCallBackend(CeedVectorGetLength(y, &length)); 356 357 for (CeedSize i = 0; i < length; i++) impl_y->array_allocated[i] = alpha * impl_x->array_allocated[i] + beta * impl_y->array_allocated[i]; 358 return CEED_ERROR_SUCCESS; 359 } 360 361 //------------------------------------------------------------------------------ 362 // Compute the pointwise multiplication w = x .* y 363 //------------------------------------------------------------------------------ 364 static int CeedVectorPointwiseMult_Memcheck(CeedVector w, CeedVector x, CeedVector y) { 365 CeedSize length; 366 CeedVector_Memcheck *impl_x, *impl_y, *impl_w; 367 368 CeedCallBackend(CeedVectorGetData(x, &impl_x)); 369 CeedCallBackend(CeedVectorGetData(y, &impl_y)); 370 CeedCallBackend(CeedVectorGetData(w, &impl_w)); 371 CeedCallBackend(CeedVectorGetLength(w, &length)); 372 373 if (!impl_w->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(w, CEED_MEM_HOST, CEED_COPY_VALUES, NULL)); 374 assert(impl_w->array_allocated); 375 for (CeedSize i = 0; i < length; i++) impl_w->array_allocated[i] = impl_x->array_allocated[i] * impl_y->array_allocated[i]; 376 return CEED_ERROR_SUCCESS; 377 } 378 379 //------------------------------------------------------------------------------ 380 // Vector Destroy 381 //------------------------------------------------------------------------------ 382 static int CeedVectorDestroy_Memcheck(CeedVector vec) { 383 CeedVector_Memcheck *impl; 384 385 // Free allocations and discard block ids 386 CeedCallBackend(CeedVectorGetData(vec, &impl)); 387 if (impl->array_allocated) { 388 CeedCallBackend(CeedFree(&impl->array_allocated)); 389 VALGRIND_DISCARD(impl->allocated_block_id); 390 } 391 if (impl->array_owned) { 392 CeedCallBackend(CeedFree(&impl->array_owned)); 393 VALGRIND_DISCARD(impl->owned_block_id); 394 } 395 if (impl->array_borrowed) { 396 VALGRIND_DISCARD(impl->borrowed_block_id); 397 } 398 CeedCallBackend(CeedFree(&impl)); 399 return CEED_ERROR_SUCCESS; 400 } 401 402 //------------------------------------------------------------------------------ 403 // Vector Create 404 //------------------------------------------------------------------------------ 405 int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) { 406 Ceed ceed; 407 CeedVector_Memcheck *impl; 408 409 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 410 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck)); 411 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck)); 412 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck)); 413 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValue", CeedVectorSetValue_Memcheck)); 414 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetValueStrided", CeedVectorSetValueStrided_Memcheck)); 415 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck)); 416 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck)); 417 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck)); 418 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck)); 419 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck)); 420 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck)); 421 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck)); 422 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Reciprocal", CeedVectorReciprocal_Memcheck)); 423 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Scale", CeedVectorScale_Memcheck)); 424 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPY", CeedVectorAXPY_Memcheck)); 425 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "AXPBY", CeedVectorAXPBY_Memcheck)); 426 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "PointwiseMult", CeedVectorPointwiseMult_Memcheck)); 427 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck)); 428 CeedCallBackend(CeedDestroy(&ceed)); 429 CeedCallBackend(CeedCalloc(1, &impl)); 430 CeedCallBackend(CeedVectorSetData(vec, impl)); 431 return CEED_ERROR_SUCCESS; 432 } 433 434 //------------------------------------------------------------------------------ 435