Lines Matching full:-
1 // Copyright (c) 2017-2026, Lawrence Livermore National Security, LLC and other CEED contributors.
2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
4 // SPDX-License-Identifier: BSD-2-Clause
14 #include "../cuda/ceed-cuda-common.h"
15 #include "ceed-cuda-ref.h"
17 //------------------------------------------------------------------------------
19 //------------------------------------------------------------------------------
28 CeedCheck(impl->h_data, ceed, CEED_ERROR_BACKEND, "No valid host data to sync to device"); in CeedQFunctionContextSyncH2D_Cuda()
31 if (impl->d_data_borrowed) { in CeedQFunctionContextSyncH2D_Cuda()
32 impl->d_data = impl->d_data_borrowed; in CeedQFunctionContextSyncH2D_Cuda()
33 } else if (impl->d_data_owned) { in CeedQFunctionContextSyncH2D_Cuda()
34 impl->d_data = impl->d_data_owned; in CeedQFunctionContextSyncH2D_Cuda()
36 CeedCallCuda(ceed, cudaMalloc((void **)&impl->d_data_owned, ctx_size)); in CeedQFunctionContextSyncH2D_Cuda()
37 impl->d_data = impl->d_data_owned; in CeedQFunctionContextSyncH2D_Cuda()
39 CeedCallCuda(ceed, cudaMemcpy(impl->d_data, impl->h_data, ctx_size, cudaMemcpyHostToDevice)); in CeedQFunctionContextSyncH2D_Cuda()
44 //------------------------------------------------------------------------------
46 //------------------------------------------------------------------------------
55 CeedCheck(impl->d_data, ceed, CEED_ERROR_BACKEND, "No valid device data to sync to host"); in CeedQFunctionContextSyncD2H_Cuda()
59 if (impl->h_data_borrowed) { in CeedQFunctionContextSyncD2H_Cuda()
60 impl->h_data = impl->h_data_borrowed; in CeedQFunctionContextSyncD2H_Cuda()
61 } else if (impl->h_data_owned) { in CeedQFunctionContextSyncD2H_Cuda()
62 impl->h_data = impl->h_data_owned; in CeedQFunctionContextSyncD2H_Cuda()
64 CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->h_data_owned)); in CeedQFunctionContextSyncD2H_Cuda()
65 impl->h_data = impl->h_data_owned; in CeedQFunctionContextSyncD2H_Cuda()
67 CeedCallCuda(ceed, cudaMemcpy(impl->h_data, impl->d_data, ctx_size, cudaMemcpyDeviceToHost)); in CeedQFunctionContextSyncD2H_Cuda()
72 //------------------------------------------------------------------------------
74 //------------------------------------------------------------------------------
85 //------------------------------------------------------------------------------
87 //------------------------------------------------------------------------------
92 impl->h_data = NULL; in CeedQFunctionContextSetAllInvalid_Cuda()
93 impl->d_data = NULL; in CeedQFunctionContextSetAllInvalid_Cuda()
97 //------------------------------------------------------------------------------
99 //------------------------------------------------------------------------------
104 *has_valid_data = impl && (impl->h_data || impl->d_data); in CeedQFunctionContextHasValidData_Cuda()
108 //------------------------------------------------------------------------------
110 //------------------------------------------------------------------------------
118 *has_borrowed_data_of_type = impl->h_data_borrowed; in CeedQFunctionContextHasBorrowedDataOfType_Cuda()
121 *has_borrowed_data_of_type = impl->d_data_borrowed; in CeedQFunctionContextHasBorrowedDataOfType_Cuda()
127 //------------------------------------------------------------------------------
129 //------------------------------------------------------------------------------
138 *need_sync = has_valid_data && !impl->h_data; in CeedQFunctionContextNeedSync_Cuda()
141 *need_sync = has_valid_data && !impl->d_data; in CeedQFunctionContextNeedSync_Cuda()
147 //------------------------------------------------------------------------------
149 //------------------------------------------------------------------------------
155 CeedCallBackend(CeedFree(&impl->h_data_owned)); in CeedQFunctionContextSetDataHost_Cuda()
160 CeedCallBackend(CeedMallocArray(1, ctx_size, &impl->h_data_owned)); in CeedQFunctionContextSetDataHost_Cuda()
161 impl->h_data_borrowed = NULL; in CeedQFunctionContextSetDataHost_Cuda()
162 impl->h_data = impl->h_data_owned; in CeedQFunctionContextSetDataHost_Cuda()
163 memcpy(impl->h_data, data, ctx_size); in CeedQFunctionContextSetDataHost_Cuda()
166 impl->h_data_owned = data; in CeedQFunctionContextSetDataHost_Cuda()
167 impl->h_data_borrowed = NULL; in CeedQFunctionContextSetDataHost_Cuda()
168 impl->h_data = data; in CeedQFunctionContextSetDataHost_Cuda()
171 impl->h_data_borrowed = data; in CeedQFunctionContextSetDataHost_Cuda()
172 impl->h_data = data; in CeedQFunctionContextSetDataHost_Cuda()
178 //------------------------------------------------------------------------------
180 //------------------------------------------------------------------------------
188 CeedCallCuda(ceed, cudaFree(impl->d_data_owned)); in CeedQFunctionContextSetDataDevice_Cuda()
189 impl->d_data_owned = NULL; in CeedQFunctionContextSetDataDevice_Cuda()
194 CeedCallCuda(ceed, cudaMalloc((void **)&impl->d_data_owned, ctx_size)); in CeedQFunctionContextSetDataDevice_Cuda()
195 impl->d_data_borrowed = NULL; in CeedQFunctionContextSetDataDevice_Cuda()
196 impl->d_data = impl->d_data_owned; in CeedQFunctionContextSetDataDevice_Cuda()
197 CeedCallCuda(ceed, cudaMemcpy(impl->d_data, data, ctx_size, cudaMemcpyDeviceToDevice)); in CeedQFunctionContextSetDataDevice_Cuda()
200 impl->d_data_owned = data; in CeedQFunctionContextSetDataDevice_Cuda()
201 impl->d_data_borrowed = NULL; in CeedQFunctionContextSetDataDevice_Cuda()
202 impl->d_data = data; in CeedQFunctionContextSetDataDevice_Cuda()
205 impl->d_data_owned = NULL; in CeedQFunctionContextSetDataDevice_Cuda()
206 impl->d_data_borrowed = data; in CeedQFunctionContextSetDataDevice_Cuda()
207 impl->d_data = data; in CeedQFunctionContextSetDataDevice_Cuda()
214 //------------------------------------------------------------------------------
217 //------------------------------------------------------------------------------
229 //------------------------------------------------------------------------------
231 //------------------------------------------------------------------------------
245 *(void **)data = impl->h_data_borrowed; in CeedQFunctionContextTakeData_Cuda()
246 impl->h_data_borrowed = NULL; in CeedQFunctionContextTakeData_Cuda()
247 impl->h_data = NULL; in CeedQFunctionContextTakeData_Cuda()
250 *(void **)data = impl->d_data_borrowed; in CeedQFunctionContextTakeData_Cuda()
251 impl->d_data_borrowed = NULL; in CeedQFunctionContextTakeData_Cuda()
252 impl->d_data = NULL; in CeedQFunctionContextTakeData_Cuda()
258 //------------------------------------------------------------------------------
261 //------------------------------------------------------------------------------
275 *(void **)data = impl->h_data; in CeedQFunctionContextGetDataCore_Cuda()
278 *(void **)data = impl->d_data; in CeedQFunctionContextGetDataCore_Cuda()
284 //------------------------------------------------------------------------------
285 // Get read-only access to the data
286 //------------------------------------------------------------------------------
291 //------------------------------------------------------------------------------
293 //------------------------------------------------------------------------------
304 impl->h_data = *(void **)data; in CeedQFunctionContextGetData_Cuda()
307 impl->d_data = *(void **)data; in CeedQFunctionContextGetData_Cuda()
313 //------------------------------------------------------------------------------
315 //------------------------------------------------------------------------------
320 CeedCallCuda(CeedQFunctionContextReturnCeed(ctx), cudaFree(impl->d_data_owned)); in CeedQFunctionContextDestroy_Cuda()
321 CeedCallBackend(CeedFree(&impl->h_data_owned)); in CeedQFunctionContextDestroy_Cuda()
326 //------------------------------------------------------------------------------
328 //------------------------------------------------------------------------------
347 //------------------------------------------------------------------------------