Lines Matching refs:impl
23 CeedVector_Cuda *impl; in CeedVectorNeedSync_Cuda() local
25 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorNeedSync_Cuda()
29 *need_sync = has_valid_array && !impl->h_array; in CeedVectorNeedSync_Cuda()
32 *need_sync = has_valid_array && !impl->d_array; in CeedVectorNeedSync_Cuda()
44 CeedVector_Cuda *impl; in CeedVectorSyncH2D_Cuda() local
46 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSyncH2D_Cuda()
48 …CeedCheck(impl->h_array, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "No valid host data to syn… in CeedVectorSyncH2D_Cuda()
52 if (impl->d_array_borrowed) { in CeedVectorSyncH2D_Cuda()
53 impl->d_array = impl->d_array_borrowed; in CeedVectorSyncH2D_Cuda()
54 } else if (impl->d_array_owned) { in CeedVectorSyncH2D_Cuda()
55 impl->d_array = impl->d_array_owned; in CeedVectorSyncH2D_Cuda()
57 CeedCallCuda(CeedVectorReturnCeed(vec), cudaMalloc((void **)&impl->d_array_owned, bytes)); in CeedVectorSyncH2D_Cuda()
58 impl->d_array = impl->d_array_owned; in CeedVectorSyncH2D_Cuda()
60 …CeedCallCuda(CeedVectorReturnCeed(vec), cudaMemcpy(impl->d_array, impl->h_array, bytes, cudaMemcpy… in CeedVectorSyncH2D_Cuda()
69 CeedVector_Cuda *impl; in CeedVectorSyncD2H_Cuda() local
71 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSyncD2H_Cuda()
73 …CeedCheck(impl->d_array, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "No valid device data to s… in CeedVectorSyncD2H_Cuda()
75 if (impl->h_array_borrowed) { in CeedVectorSyncD2H_Cuda()
76 impl->h_array = impl->h_array_borrowed; in CeedVectorSyncD2H_Cuda()
77 } else if (impl->h_array_owned) { in CeedVectorSyncD2H_Cuda()
78 impl->h_array = impl->h_array_owned; in CeedVectorSyncD2H_Cuda()
83 CeedCallBackend(CeedCalloc(length, &impl->h_array_owned)); in CeedVectorSyncD2H_Cuda()
84 impl->h_array = impl->h_array_owned; in CeedVectorSyncD2H_Cuda()
90 …CeedCallCuda(CeedVectorReturnCeed(vec), cudaMemcpy(impl->h_array, impl->d_array, bytes, cudaMemcpy… in CeedVectorSyncD2H_Cuda()
117 CeedVector_Cuda *impl; in CeedVectorSetAllInvalid_Cuda() local
119 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetAllInvalid_Cuda()
120 impl->h_array = NULL; in CeedVectorSetAllInvalid_Cuda()
121 impl->d_array = NULL; in CeedVectorSetAllInvalid_Cuda()
129 CeedVector_Cuda *impl; in CeedVectorHasValidArray_Cuda() local
131 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorHasValidArray_Cuda()
132 *has_valid_array = impl->h_array || impl->d_array; in CeedVectorHasValidArray_Cuda()
140 CeedVector_Cuda *impl; in CeedVectorHasArrayOfType_Cuda() local
142 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorHasArrayOfType_Cuda()
145 *has_array_of_type = impl->h_array_borrowed || impl->h_array_owned; in CeedVectorHasArrayOfType_Cuda()
148 *has_array_of_type = impl->d_array_borrowed || impl->d_array_owned; in CeedVectorHasArrayOfType_Cuda()
158 CeedVector_Cuda *impl; in CeedVectorHasBorrowedArrayOfType_Cuda() local
160 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorHasBorrowedArrayOfType_Cuda()
163 *has_borrowed_array_of_type = impl->h_array_borrowed; in CeedVectorHasBorrowedArrayOfType_Cuda()
166 *has_borrowed_array_of_type = impl->d_array_borrowed; in CeedVectorHasBorrowedArrayOfType_Cuda()
177 CeedVector_Cuda *impl; in CeedVectorSetArrayHost_Cuda() local
179 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArrayHost_Cuda()
182 …CeedCallBackend(CeedSetHostCeedScalarArray(array, copy_mode, length, (const CeedScalar **)&impl->h… in CeedVectorSetArrayHost_Cuda()
183 … (const CeedScalar **)&impl->h_array_borrowed, (const CeedScalar **)&impl->h_array)); in CeedVectorSetArrayHost_Cuda()
193 CeedVector_Cuda *impl; in CeedVectorSetArrayDevice_Cuda() local
196 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArrayDevice_Cuda()
199 …viceCeedScalarArray_Cuda(ceed, array, copy_mode, length, (const CeedScalar **)&impl->d_array_owned, in CeedVectorSetArrayDevice_Cuda()
200 … (const CeedScalar **)&impl->d_array_borrowed, (const CeedScalar **)&impl->d_array)); in CeedVectorSetArrayDevice_Cuda()
210 CeedVector_Cuda *impl; in CeedVectorSetArray_Cuda() local
212 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArray_Cuda()
241 CeedVector_Cuda *impl; in CeedVectorCopyStrided_Cuda() local
243 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorCopyStrided_Cuda()
253 if (impl->d_array) { in CeedVectorCopyStrided_Cuda()
264 …CeedCallCublas(ceed, cublasScopy_64(handle, (int64_t)(stop - start), impl->d_array + start, (int64… in CeedVectorCopyStrided_Cuda()
266 …CeedCallCublas(ceed, cublasDcopy_64(handle, (int64_t)(stop - start), impl->d_array + start, (int64… in CeedVectorCopyStrided_Cuda()
270 CeedCallBackend(CeedDeviceCopyStrided_Cuda(impl->d_array, start, stop, step, copy_array)); in CeedVectorCopyStrided_Cuda()
273 impl->h_array = NULL; in CeedVectorCopyStrided_Cuda()
274 } else if (impl->h_array) { in CeedVectorCopyStrided_Cuda()
278 CeedCallBackend(CeedHostCopyStrided_Cuda(impl->h_array, start, stop, step, copy_array)); in CeedVectorCopyStrided_Cuda()
280 impl->d_array = NULL; in CeedVectorCopyStrided_Cuda()
305 CeedVector_Cuda *impl; in CeedVectorSetValue_Cuda() local
307 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetValue_Cuda()
310 if (!impl->d_array && !impl->h_array) { in CeedVectorSetValue_Cuda()
311 if (impl->d_array_borrowed) { in CeedVectorSetValue_Cuda()
312 impl->d_array = impl->d_array_borrowed; in CeedVectorSetValue_Cuda()
313 } else if (impl->h_array_borrowed) { in CeedVectorSetValue_Cuda()
314 impl->h_array = impl->h_array_borrowed; in CeedVectorSetValue_Cuda()
315 } else if (impl->d_array_owned) { in CeedVectorSetValue_Cuda()
316 impl->d_array = impl->d_array_owned; in CeedVectorSetValue_Cuda()
317 } else if (impl->h_array_owned) { in CeedVectorSetValue_Cuda()
318 impl->h_array = impl->h_array_owned; in CeedVectorSetValue_Cuda()
323 if (impl->d_array) { in CeedVectorSetValue_Cuda()
325 …CeedCallCuda(CeedVectorReturnCeed(vec), cudaMemset(impl->d_array, 0, length * sizeof(CeedScalar))); in CeedVectorSetValue_Cuda()
327 CeedCallBackend(CeedDeviceSetValue_Cuda(impl->d_array, length, val)); in CeedVectorSetValue_Cuda()
329 impl->h_array = NULL; in CeedVectorSetValue_Cuda()
330 } else if (impl->h_array) { in CeedVectorSetValue_Cuda()
331 CeedCallBackend(CeedHostSetValue_Cuda(impl->h_array, length, val)); in CeedVectorSetValue_Cuda()
332 impl->d_array = NULL; in CeedVectorSetValue_Cuda()
355 CeedVector_Cuda *impl; in CeedVectorSetValueStrided_Cuda() local
357 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetValueStrided_Cuda()
361 if (impl->d_array) { in CeedVectorSetValueStrided_Cuda()
362 CeedCallBackend(CeedDeviceSetValueStrided_Cuda(impl->d_array, start, stop, step, val)); in CeedVectorSetValueStrided_Cuda()
363 impl->h_array = NULL; in CeedVectorSetValueStrided_Cuda()
364 } else if (impl->h_array) { in CeedVectorSetValueStrided_Cuda()
365 CeedCallBackend(CeedHostSetValueStrided_Cuda(impl->h_array, start, stop, step, val)); in CeedVectorSetValueStrided_Cuda()
366 impl->d_array = NULL; in CeedVectorSetValueStrided_Cuda()
377 CeedVector_Cuda *impl; in CeedVectorTakeArray_Cuda() local
379 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorTakeArray_Cuda()
385 (*array) = impl->h_array_borrowed; in CeedVectorTakeArray_Cuda()
386 impl->h_array_borrowed = NULL; in CeedVectorTakeArray_Cuda()
387 impl->h_array = NULL; in CeedVectorTakeArray_Cuda()
390 (*array) = impl->d_array_borrowed; in CeedVectorTakeArray_Cuda()
391 impl->d_array_borrowed = NULL; in CeedVectorTakeArray_Cuda()
392 impl->d_array = NULL; in CeedVectorTakeArray_Cuda()
403 CeedVector_Cuda *impl; in CeedVectorGetArrayCore_Cuda() local
405 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorGetArrayCore_Cuda()
411 *array = impl->h_array; in CeedVectorGetArrayCore_Cuda()
414 *array = impl->d_array; in CeedVectorGetArrayCore_Cuda()
431 CeedVector_Cuda *impl; in CeedVectorGetArray_Cuda() local
433 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorGetArray_Cuda()
438 impl->h_array = *array; in CeedVectorGetArray_Cuda()
441 impl->d_array = *array; in CeedVectorGetArray_Cuda()
452 CeedVector_Cuda *impl; in CeedVectorGetArrayWrite_Cuda() local
454 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorGetArrayWrite_Cuda()
463 if (impl->h_array_borrowed) impl->h_array = impl->h_array_borrowed; in CeedVectorGetArrayWrite_Cuda()
464 else impl->h_array = impl->h_array_owned; in CeedVectorGetArrayWrite_Cuda()
467 if (impl->d_array_borrowed) impl->d_array = impl->d_array_borrowed; in CeedVectorGetArrayWrite_Cuda()
468 else impl->d_array = impl->d_array_owned; in CeedVectorGetArrayWrite_Cuda()
484 CeedVector_Cuda *impl; in CeedVectorNorm_Cuda() local
488 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorNorm_Cuda()
585 …CeedCallCuda(ceed, cudaMemcpy(&norm_no_abs, impl->d_array + index - 1, sizeof(CeedScalar), cudaMem… in CeedVectorNorm_Cuda()
609 …CeedCallCuda(ceed, cudaMemcpy(&norm_no_abs, impl->d_array + index - 1, sizeof(CeedScalar), cudaMem… in CeedVectorNorm_Cuda()
656 CeedVector_Cuda *impl; in CeedVectorReciprocal_Cuda() local
658 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorReciprocal_Cuda()
661 if (impl->d_array) CeedCallBackend(CeedDeviceReciprocal_Cuda(impl->d_array, length)); in CeedVectorReciprocal_Cuda()
662 if (impl->h_array) CeedCallBackend(CeedHostReciprocal_Cuda(impl->h_array, length)); in CeedVectorReciprocal_Cuda()
684 CeedVector_Cuda *impl; in CeedVectorScale_Cuda() local
686 CeedCallBackend(CeedVectorGetData(x, &impl)); in CeedVectorScale_Cuda()
689 if (impl->d_array) { in CeedVectorScale_Cuda()
695 …CeedCallCublas(CeedVectorReturnCeed(x), cublasSscal_64(handle, (int64_t)length, &alpha, impl->d_ar… in CeedVectorScale_Cuda()
697 …CeedCallCublas(CeedVectorReturnCeed(x), cublasDscal_64(handle, (int64_t)length, &alpha, impl->d_ar… in CeedVectorScale_Cuda()
700 CeedCallBackend(CeedDeviceScale_Cuda(impl->d_array, alpha, length)); in CeedVectorScale_Cuda()
702 impl->h_array = NULL; in CeedVectorScale_Cuda()
703 } else if (impl->h_array) { in CeedVectorScale_Cuda()
704 CeedCallBackend(CeedHostScale_Cuda(impl->h_array, alpha, length)); in CeedVectorScale_Cuda()
705 impl->d_array = NULL; in CeedVectorScale_Cuda()
837 CeedVector_Cuda *impl; in CeedVectorDestroy_Cuda() local
839 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorDestroy_Cuda()
840 CeedCallCuda(CeedVectorReturnCeed(vec), cudaFree(impl->d_array_owned)); in CeedVectorDestroy_Cuda()
841 CeedCallBackend(CeedFree(&impl->h_array_owned)); in CeedVectorDestroy_Cuda()
842 CeedCallBackend(CeedFree(&impl)); in CeedVectorDestroy_Cuda()
850 CeedVector_Cuda *impl; in CeedVectorCreate_Cuda() local
873 CeedCallBackend(CeedCalloc(1, &impl)); in CeedVectorCreate_Cuda()
874 CeedCallBackend(CeedVectorSetData(vec, impl)); in CeedVectorCreate_Cuda()