1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed.h> 9 #include <ceed/backend.h> 10 #include <math.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <valgrind/memcheck.h> 14 15 #include "ceed-memcheck.h" 16 17 //------------------------------------------------------------------------------ 18 // Has Valid Array 19 //------------------------------------------------------------------------------ 20 static int CeedVectorHasValidArray_Memcheck(CeedVector vec, bool *has_valid_array) { 21 CeedVector_Memcheck *impl; 22 23 CeedCallBackend(CeedVectorGetData(vec, &impl)); 24 *has_valid_array = !!impl->array_allocated; 25 return CEED_ERROR_SUCCESS; 26 } 27 28 //------------------------------------------------------------------------------ 29 // Check if has borrowed array of given type 30 //------------------------------------------------------------------------------ 31 static inline int CeedVectorHasBorrowedArrayOfType_Memcheck(const CeedVector vec, CeedMemType mem_type, bool *has_borrowed_array_of_type) { 32 CeedVector_Memcheck *impl; 33 34 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 35 36 CeedCallBackend(CeedVectorGetData(vec, &impl)); 37 *has_borrowed_array_of_type = !!impl->array_borrowed; 38 return CEED_ERROR_SUCCESS; 39 } 40 41 //------------------------------------------------------------------------------ 42 // Vector Set Array 43 //------------------------------------------------------------------------------ 44 static int CeedVectorSetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedCopyMode copy_mode, CeedScalar *array) { 45 CeedSize length; 46 CeedVector_Memcheck *impl; 47 48 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only set HOST memory for this backend"); 49 50 CeedCallBackend(CeedVectorGetData(vec, &impl)); 51 CeedCallBackend(CeedVectorGetLength(vec, &length)); 52 53 // Clear previous owned arrays 54 if (impl->array_allocated) { 55 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 56 } 57 CeedCallBackend(CeedFree(&impl->array_allocated)); 58 VALGRIND_DISCARD(impl->allocated_block_id); 59 if (impl->array_owned) { 60 for (CeedSize i = 0; i < length; i++) impl->array_owned[i] = NAN; 61 } 62 VALGRIND_DISCARD(impl->owned_block_id); 63 CeedCallBackend(CeedFree(&impl->array_owned)); 64 65 // Clear borrowed block id, if present 66 if (impl->array_borrowed) VALGRIND_DISCARD(impl->borrowed_block_id); 67 68 // Set internal pointers to external arrays 69 switch (copy_mode) { 70 case CEED_COPY_VALUES: 71 impl->array_owned = NULL; 72 impl->array_borrowed = NULL; 73 break; 74 case CEED_OWN_POINTER: 75 impl->array_owned = array; 76 impl->array_borrowed = NULL; 77 impl->owned_block_id = VALGRIND_CREATE_BLOCK(impl->array_owned, length * sizeof(CeedScalar), "Owned external array buffer"); 78 break; 79 case CEED_USE_POINTER: 80 impl->array_owned = NULL; 81 impl->array_borrowed = array; 82 impl->borrowed_block_id = VALGRIND_CREATE_BLOCK(impl->array_borrowed, length * sizeof(CeedScalar), "Borrowed external array buffer"); 83 break; 84 } 85 86 // Create internal array data buffer 87 CeedCallBackend(CeedCalloc(length, &impl->array_allocated)); 88 impl->allocated_block_id = VALGRIND_CREATE_BLOCK(impl->array_allocated, length * sizeof(CeedScalar), "Allocated internal array buffer"); 89 if (array) { 90 memcpy(impl->array_allocated, array, length * sizeof(CeedScalar)); 91 } else { 92 for (CeedInt i = 0; i < length; i++) impl->array_allocated[i] = NAN; 93 } 94 return CEED_ERROR_SUCCESS; 95 } 96 97 //------------------------------------------------------------------------------ 98 // Sync arrays 99 //------------------------------------------------------------------------------ 100 static int CeedVectorSyncArray_Memcheck(const CeedVector vec, CeedMemType mem_type) { 101 CeedSize length; 102 CeedVector_Memcheck *impl; 103 104 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 105 106 CeedCallBackend(CeedVectorGetData(vec, &impl)); 107 CeedCallBackend(CeedVectorGetLength(vec, &length)); 108 109 // Copy internal buffer back to owned or borrowed array 110 if (impl->array_owned) { 111 memcpy(impl->array_owned, impl->array_allocated, length * sizeof(CeedScalar)); 112 } 113 if (impl->array_borrowed) { 114 memcpy(impl->array_borrowed, impl->array_allocated, length * sizeof(CeedScalar)); 115 } 116 return CEED_ERROR_SUCCESS; 117 } 118 119 //------------------------------------------------------------------------------ 120 // Vector Take Array 121 //------------------------------------------------------------------------------ 122 static int CeedVectorTakeArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 123 CeedSize length; 124 CeedVector_Memcheck *impl; 125 126 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 127 128 CeedCallBackend(CeedVectorGetData(vec, &impl)); 129 CeedCallBackend(CeedVectorGetLength(vec, &length)); 130 131 // Synchronize memory 132 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 133 134 // Return borrowed array 135 (*array) = impl->array_borrowed; 136 impl->array_borrowed = NULL; 137 VALGRIND_DISCARD(impl->borrowed_block_id); 138 139 // De-allocate internal memory 140 if (impl->array_allocated) { 141 for (CeedSize i = 0; i < length; i++) impl->array_allocated[i] = NAN; 142 } 143 CeedCallBackend(CeedFree(&impl->array_allocated)); 144 VALGRIND_DISCARD(impl->allocated_block_id); 145 return CEED_ERROR_SUCCESS; 146 } 147 148 //------------------------------------------------------------------------------ 149 // Vector Get Array 150 //------------------------------------------------------------------------------ 151 static int CeedVectorGetArray_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 152 CeedSize length; 153 CeedVector_Memcheck *impl; 154 155 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 156 157 CeedCallBackend(CeedVectorGetData(vec, &impl)); 158 CeedCallBackend(CeedVectorGetLength(vec, &length)); 159 160 // Create and return writable buffer 161 CeedCallBackend(CeedCalloc(length, &impl->array_writable_copy)); 162 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_writable_copy, length * sizeof(CeedScalar), "Allocated writeable array buffer copy"); 163 memcpy(impl->array_writable_copy, impl->array_allocated, length * sizeof(CeedScalar)); 164 *array = impl->array_writable_copy; 165 return CEED_ERROR_SUCCESS; 166 } 167 168 //------------------------------------------------------------------------------ 169 // Vector Get Array Read 170 //------------------------------------------------------------------------------ 171 static int CeedVectorGetArrayRead_Memcheck(CeedVector vec, CeedMemType mem_type, const CeedScalar **array) { 172 CeedSize length; 173 CeedVector_Memcheck *impl; 174 175 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 176 177 CeedCallBackend(CeedVectorGetData(vec, &impl)); 178 CeedCallBackend(CeedVectorGetLength(vec, &length)); 179 180 // Create and return read-only buffer 181 if (!impl->array_read_only_copy) { 182 CeedCallBackend(CeedCalloc(length, &impl->array_read_only_copy)); 183 impl->writable_block_id = VALGRIND_CREATE_BLOCK(impl->array_read_only_copy, length * sizeof(CeedScalar), "Allocated read-only array buffer copy"); 184 memcpy(impl->array_read_only_copy, impl->array_allocated, length * sizeof(CeedScalar)); 185 } 186 *array = impl->array_read_only_copy; 187 return CEED_ERROR_SUCCESS; 188 } 189 190 //------------------------------------------------------------------------------ 191 // Vector Get Array Write 192 //------------------------------------------------------------------------------ 193 static int CeedVectorGetArrayWrite_Memcheck(CeedVector vec, CeedMemType mem_type, CeedScalar **array) { 194 CeedSize length; 195 CeedVector_Memcheck *impl; 196 197 CeedCheck(mem_type == CEED_MEM_HOST, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "Can only provide HOST memory for this backend"); 198 199 CeedCallBackend(CeedVectorGetData(vec, &impl)); 200 CeedCallBackend(CeedVectorGetLength(vec, &length)); 201 202 // Allocate buffer if necessary 203 if (!impl->array_allocated) CeedCallBackend(CeedVectorSetArray_Memcheck(vec, mem_type, CEED_COPY_VALUES, NULL)); 204 205 // Get writable buffer 206 CeedCallBackend(CeedVectorGetArray_Memcheck(vec, mem_type, array)); 207 208 // Invalidate array data to prevent accidental reads 209 for (CeedSize i = 0; i < length; i++) (*array)[i] = NAN; 210 impl->is_write_only_access = true; 211 return CEED_ERROR_SUCCESS; 212 } 213 214 //------------------------------------------------------------------------------ 215 // Vector Restore Array 216 //------------------------------------------------------------------------------ 217 static int CeedVectorRestoreArray_Memcheck(CeedVector vec) { 218 Ceed ceed; 219 CeedSize length; 220 CeedVector_Memcheck *impl; 221 222 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 223 CeedCallBackend(CeedVectorGetData(vec, &impl)); 224 CeedCallBackend(CeedVectorGetLength(vec, &length)); 225 226 // Check for unset entries after write-only access 227 if (impl->is_write_only_access) { 228 for (CeedSize i = 0; i < length; i++) { 229 if (isnan(impl->array_writable_copy[i])) 230 CeedDebug256(ceed, CEED_DEBUG_COLOR_WARNING, "WARNING: Vec entry %" CeedSize_FMT " is NaN after restoring write-only access", i); 231 } 232 impl->is_write_only_access = false; 233 } 234 235 // Copy back to internal buffer and sync 236 memcpy(impl->array_allocated, impl->array_writable_copy, length * sizeof(CeedScalar)); 237 CeedCallBackend(CeedVectorSyncArray_Memcheck(vec, CEED_MEM_HOST)); 238 239 // Invalidate writable buffer 240 for (CeedSize i = 0; i < length; i++) impl->array_writable_copy[i] = NAN; 241 CeedCallBackend(CeedFree(&impl->array_writable_copy)); 242 VALGRIND_DISCARD(impl->writable_block_id); 243 return CEED_ERROR_SUCCESS; 244 } 245 246 //------------------------------------------------------------------------------ 247 // Vector Restore Array Read-Only 248 //------------------------------------------------------------------------------ 249 static int CeedVectorRestoreArrayRead_Memcheck(CeedVector vec) { 250 Ceed ceed; 251 CeedSize length; 252 CeedVector_Memcheck *impl; 253 254 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 255 CeedCallBackend(CeedVectorGetData(vec, &impl)); 256 CeedCallBackend(CeedVectorGetLength(vec, &length)); 257 258 // Verify no changes made during read-only access 259 bool is_changed = memcmp(impl->array_allocated, impl->array_read_only_copy, length * sizeof(CeedScalar)); 260 261 CeedCheck(!is_changed, ceed, CEED_ERROR_BACKEND, "Array data changed while accessed in read-only mode"); 262 263 // Invalidate read-only buffer 264 for (CeedSize i = 0; i < length; i++) impl->array_read_only_copy[i] = NAN; 265 CeedCallBackend(CeedFree(&impl->array_read_only_copy)); 266 VALGRIND_DISCARD(impl->read_only_block_id); 267 return CEED_ERROR_SUCCESS; 268 } 269 270 //------------------------------------------------------------------------------ 271 // Vector Destroy 272 //------------------------------------------------------------------------------ 273 static int CeedVectorDestroy_Memcheck(CeedVector vec) { 274 CeedVector_Memcheck *impl; 275 276 // Free allocations and discard block ids 277 CeedCallBackend(CeedVectorGetData(vec, &impl)); 278 if (impl->array_allocated) { 279 CeedCallBackend(CeedFree(&impl->array_allocated)); 280 VALGRIND_DISCARD(impl->allocated_block_id); 281 } 282 if (impl->array_owned) { 283 CeedCallBackend(CeedFree(&impl->array_owned)); 284 VALGRIND_DISCARD(impl->owned_block_id); 285 } 286 if (impl->array_borrowed) { 287 VALGRIND_DISCARD(impl->borrowed_block_id); 288 } 289 CeedCallBackend(CeedFree(&impl)); 290 return CEED_ERROR_SUCCESS; 291 } 292 293 //------------------------------------------------------------------------------ 294 // Vector Create 295 //------------------------------------------------------------------------------ 296 int CeedVectorCreate_Memcheck(CeedSize n, CeedVector vec) { 297 Ceed ceed; 298 CeedVector_Memcheck *impl; 299 300 CeedCallBackend(CeedCalloc(1, &impl)); 301 CeedCallBackend(CeedVectorSetData(vec, impl)); 302 303 CeedCallBackend(CeedVectorGetCeed(vec, &ceed)); 304 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasValidArray", CeedVectorHasValidArray_Memcheck)); 305 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "HasBorrowedArrayOfType", CeedVectorHasBorrowedArrayOfType_Memcheck)); 306 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SetArray", CeedVectorSetArray_Memcheck)); 307 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "SyncArray", CeedVectorSyncArray_Memcheck)); 308 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "TakeArray", CeedVectorTakeArray_Memcheck)); 309 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArray", CeedVectorGetArray_Memcheck)); 310 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayRead", CeedVectorGetArrayRead_Memcheck)); 311 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "GetArrayWrite", CeedVectorGetArrayWrite_Memcheck)); 312 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArray", CeedVectorRestoreArray_Memcheck)); 313 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "RestoreArrayRead", CeedVectorRestoreArrayRead_Memcheck)); 314 CeedCallBackend(CeedSetBackendFunction(ceed, "Vector", vec, "Destroy", CeedVectorDestroy_Memcheck)); 315 return CEED_ERROR_SUCCESS; 316 } 317 318 //------------------------------------------------------------------------------ 319