Lines Matching refs:impl
22 CeedVector_Hip *impl; in CeedVectorNeedSync_Hip() local
25 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorNeedSync_Hip()
29 *need_sync = has_valid_array && !impl->h_array; in CeedVectorNeedSync_Hip()
32 *need_sync = has_valid_array && !impl->d_array; in CeedVectorNeedSync_Hip()
44 CeedVector_Hip *impl; in CeedVectorSyncH2D_Hip() local
46 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSyncH2D_Hip()
48 …CeedCheck(impl->h_array, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "No valid host data to syn… in CeedVectorSyncH2D_Hip()
52 if (impl->d_array_borrowed) { in CeedVectorSyncH2D_Hip()
53 impl->d_array = impl->d_array_borrowed; in CeedVectorSyncH2D_Hip()
54 } else if (impl->d_array_owned) { in CeedVectorSyncH2D_Hip()
55 impl->d_array = impl->d_array_owned; in CeedVectorSyncH2D_Hip()
57 CeedCallHip(CeedVectorReturnCeed(vec), hipMalloc((void **)&impl->d_array_owned, bytes)); in CeedVectorSyncH2D_Hip()
58 impl->d_array = impl->d_array_owned; in CeedVectorSyncH2D_Hip()
60 …CeedCallHip(CeedVectorReturnCeed(vec), hipMemcpy(impl->d_array, impl->h_array, bytes, hipMemcpyHos… in CeedVectorSyncH2D_Hip()
70 CeedVector_Hip *impl; in CeedVectorSyncD2H_Hip() local
72 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSyncD2H_Hip()
74 …CeedCheck(impl->d_array, CeedVectorReturnCeed(vec), CEED_ERROR_BACKEND, "No valid device data to s… in CeedVectorSyncD2H_Hip()
76 if (impl->h_array_borrowed) { in CeedVectorSyncD2H_Hip()
77 impl->h_array = impl->h_array_borrowed; in CeedVectorSyncD2H_Hip()
78 } else if (impl->h_array_owned) { in CeedVectorSyncD2H_Hip()
79 impl->h_array = impl->h_array_owned; in CeedVectorSyncD2H_Hip()
84 CeedCallBackend(CeedCalloc(length, &impl->h_array_owned)); in CeedVectorSyncD2H_Hip()
85 impl->h_array = impl->h_array_owned; in CeedVectorSyncD2H_Hip()
90 …CeedCallHip(CeedVectorReturnCeed(vec), hipMemcpy(impl->h_array, impl->d_array, bytes, hipMemcpyDev… in CeedVectorSyncD2H_Hip()
99 CeedVector_Hip *impl; in CeedVectorSyncArray_Hip() local
102 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSyncArray_Hip()
103 if (impl->has_unified_addressing && !impl->h_array_borrowed) { in CeedVectorSyncArray_Hip()
125 CeedVector_Hip *impl; in CeedVectorSetAllInvalid_Hip() local
127 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetAllInvalid_Hip()
128 impl->h_array = NULL; in CeedVectorSetAllInvalid_Hip()
129 impl->d_array = NULL; in CeedVectorSetAllInvalid_Hip()
137 CeedVector_Hip *impl; in CeedVectorHasValidArray_Hip() local
139 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorHasValidArray_Hip()
140 *has_valid_array = impl->h_array || impl->d_array; in CeedVectorHasValidArray_Hip()
148 CeedVector_Hip *impl; in CeedVectorHasArrayOfType_Hip() local
150 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorHasArrayOfType_Hip()
153 *has_array_of_type = impl->h_array_borrowed || impl->h_array_owned; in CeedVectorHasArrayOfType_Hip()
156 *has_array_of_type = impl->d_array_borrowed || impl->d_array_owned; in CeedVectorHasArrayOfType_Hip()
166 CeedVector_Hip *impl; in CeedVectorHasBorrowedArrayOfType_Hip() local
168 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorHasBorrowedArrayOfType_Hip()
171 mem_type = impl->has_unified_addressing && !impl->h_array_borrowed ? CEED_MEM_DEVICE : mem_type; in CeedVectorHasBorrowedArrayOfType_Hip()
175 *has_borrowed_array_of_type = impl->h_array_borrowed; in CeedVectorHasBorrowedArrayOfType_Hip()
178 *has_borrowed_array_of_type = impl->d_array_borrowed; in CeedVectorHasBorrowedArrayOfType_Hip()
189 CeedVector_Hip *impl; in CeedVectorSetArrayHost_Hip() local
191 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArrayHost_Hip()
194 …CeedCallBackend(CeedSetHostCeedScalarArray(array, copy_mode, length, (const CeedScalar **)&impl->h… in CeedVectorSetArrayHost_Hip()
195 … (const CeedScalar **)&impl->h_array_borrowed, (const CeedScalar **)&impl->h_array)); in CeedVectorSetArrayHost_Hip()
205 CeedVector_Hip *impl; in CeedVectorSetArrayDevice_Hip() local
208 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArrayDevice_Hip()
211 …eviceCeedScalarArray_Hip(ceed, array, copy_mode, length, (const CeedScalar **)&impl->d_array_owned, in CeedVectorSetArrayDevice_Hip()
212 … (const CeedScalar **)&impl->d_array_borrowed, (const CeedScalar **)&impl->d_array)); in CeedVectorSetArrayDevice_Hip()
223 CeedVector_Hip *impl; in CeedVectorSetArrayUnifiedHostToDevice_Hip() local
226 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArrayUnifiedHostToDevice_Hip()
232 if (!impl->d_array) { in CeedVectorSetArrayUnifiedHostToDevice_Hip()
233 if (impl->d_array_borrowed) { in CeedVectorSetArrayUnifiedHostToDevice_Hip()
234 impl->d_array = impl->d_array_borrowed; in CeedVectorSetArrayUnifiedHostToDevice_Hip()
236 …if (!impl->d_array_owned) CeedCallHip(ceed, hipMalloc((void **)&impl->d_array_owned, sizeof(CeedSc… in CeedVectorSetArrayUnifiedHostToDevice_Hip()
237 impl->d_array = impl->d_array_owned; in CeedVectorSetArrayUnifiedHostToDevice_Hip()
240 …if (array) CeedCallHip(ceed, hipMemcpy(impl->d_array, array, sizeof(CeedScalar) * length, hipMemcp… in CeedVectorSetArrayUnifiedHostToDevice_Hip()
244 CeedCallHip(ceed, hipFree(impl->d_array_owned)); in CeedVectorSetArrayUnifiedHostToDevice_Hip()
245 CeedCallBackend(CeedFree(&impl->h_array_owned)); in CeedVectorSetArrayUnifiedHostToDevice_Hip()
246 impl->h_array_owned = NULL; in CeedVectorSetArrayUnifiedHostToDevice_Hip()
247 impl->h_array_borrowed = array; in CeedVectorSetArrayUnifiedHostToDevice_Hip()
248 impl->d_array = impl->h_array_borrowed; in CeedVectorSetArrayUnifiedHostToDevice_Hip()
259 CeedVector_Hip *impl; in CeedVectorSetArray_Hip() local
261 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetArray_Hip()
265 if (impl->has_unified_addressing) { in CeedVectorSetArray_Hip()
294 CeedVector_Hip *impl; in CeedVectorCopyStrided_Hip() local
296 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorCopyStrided_Hip()
306 if (impl->d_array) { in CeedVectorCopyStrided_Hip()
318 …CeedCallHipblas(ceed, hipblasScopy_64(handle, (int64_t)(stop - start), impl->d_array + start, (int… in CeedVectorCopyStrided_Hip()
320 …CeedCallHipblas(ceed, hipblasDcopy_64(handle, (int64_t)(stop - start), impl->d_array + start, (int… in CeedVectorCopyStrided_Hip()
324 CeedCallBackend(CeedDeviceCopyStrided_Hip(impl->d_array, start, stop, step, copy_array)); in CeedVectorCopyStrided_Hip()
327 impl->h_array = NULL; in CeedVectorCopyStrided_Hip()
329 } else if (impl->h_array) { in CeedVectorCopyStrided_Hip()
333 CeedCallBackend(CeedHostCopyStrided_Hip(impl->h_array, start, stop, step, copy_array)); in CeedVectorCopyStrided_Hip()
335 impl->d_array = NULL; in CeedVectorCopyStrided_Hip()
360 CeedVector_Hip *impl; in CeedVectorSetValue_Hip() local
363 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetValue_Hip()
367 if (!impl->d_array && !impl->h_array) { in CeedVectorSetValue_Hip()
368 if (impl->d_array_borrowed) { in CeedVectorSetValue_Hip()
369 impl->d_array = impl->d_array_borrowed; in CeedVectorSetValue_Hip()
370 } else if (impl->h_array_borrowed) { in CeedVectorSetValue_Hip()
371 impl->h_array = impl->h_array_borrowed; in CeedVectorSetValue_Hip()
372 } else if (impl->d_array_owned) { in CeedVectorSetValue_Hip()
373 impl->d_array = impl->d_array_owned; in CeedVectorSetValue_Hip()
374 } else if (impl->h_array_owned) { in CeedVectorSetValue_Hip()
375 impl->h_array = impl->h_array_owned; in CeedVectorSetValue_Hip()
380 if (impl->d_array) { in CeedVectorSetValue_Hip()
381 if (val == 0 && !impl->h_array_borrowed) { in CeedVectorSetValue_Hip()
382 … CeedCallHip(CeedVectorReturnCeed(vec), hipMemset(impl->d_array, 0, length * sizeof(CeedScalar))); in CeedVectorSetValue_Hip()
384 CeedCallBackend(CeedDeviceSetValue_Hip(impl->d_array, length, val)); in CeedVectorSetValue_Hip()
386 impl->h_array = NULL; in CeedVectorSetValue_Hip()
387 } else if (impl->h_array) { in CeedVectorSetValue_Hip()
388 CeedCallBackend(CeedHostSetValue_Hip(impl->h_array, length, val)); in CeedVectorSetValue_Hip()
389 impl->d_array = NULL; in CeedVectorSetValue_Hip()
412 CeedVector_Hip *impl; in CeedVectorSetValueStrided_Hip() local
414 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorSetValueStrided_Hip()
418 if (impl->d_array) { in CeedVectorSetValueStrided_Hip()
419 CeedCallBackend(CeedDeviceSetValueStrided_Hip(impl->d_array, start, stop, step, val)); in CeedVectorSetValueStrided_Hip()
420 impl->h_array = NULL; in CeedVectorSetValueStrided_Hip()
421 } else if (impl->h_array) { in CeedVectorSetValueStrided_Hip()
422 CeedCallBackend(CeedHostSetValueStrided_Hip(impl->h_array, start, stop, step, val)); in CeedVectorSetValueStrided_Hip()
423 impl->d_array = NULL; in CeedVectorSetValueStrided_Hip()
434 CeedVector_Hip *impl; in CeedVectorTakeArray_Hip() local
436 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorTakeArray_Hip()
444 (*array) = impl->h_array_borrowed; in CeedVectorTakeArray_Hip()
445 impl->h_array_borrowed = NULL; in CeedVectorTakeArray_Hip()
446 impl->h_array = NULL; in CeedVectorTakeArray_Hip()
449 (*array) = impl->d_array_borrowed; in CeedVectorTakeArray_Hip()
450 impl->d_array_borrowed = NULL; in CeedVectorTakeArray_Hip()
451 impl->d_array = NULL; in CeedVectorTakeArray_Hip()
462 CeedVector_Hip *impl; in CeedVectorGetArrayCore_Hip() local
464 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorGetArrayCore_Hip()
467 mem_type = impl->has_unified_addressing && !impl->h_array_borrowed ? CEED_MEM_DEVICE : mem_type; in CeedVectorGetArrayCore_Hip()
475 *array = impl->h_array; in CeedVectorGetArrayCore_Hip()
478 *array = impl->d_array; in CeedVectorGetArrayCore_Hip()
495 CeedVector_Hip *impl; in CeedVectorGetArray_Hip() local
497 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorGetArray_Hip()
500 mem_type = impl->has_unified_addressing && !impl->h_array_borrowed ? CEED_MEM_DEVICE : mem_type; in CeedVectorGetArray_Hip()
507 impl->h_array = *array; in CeedVectorGetArray_Hip()
508 if (impl->has_unified_addressing) impl->d_array = *array; in CeedVectorGetArray_Hip()
511 impl->d_array = *array; in CeedVectorGetArray_Hip()
522 CeedVector_Hip *impl; in CeedVectorGetArrayWrite_Hip() local
525 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorGetArrayWrite_Hip()
529 mem_type = impl->has_unified_addressing && !impl->h_array_borrowed ? CEED_MEM_DEVICE : mem_type; in CeedVectorGetArrayWrite_Hip()
539 if (impl->h_array_borrowed) impl->h_array = impl->h_array_borrowed; in CeedVectorGetArrayWrite_Hip()
540 else impl->h_array = impl->h_array_owned; in CeedVectorGetArrayWrite_Hip()
543 if (impl->d_array_borrowed) impl->d_array = impl->d_array_borrowed; in CeedVectorGetArrayWrite_Hip()
544 else impl->d_array = impl->d_array_owned; in CeedVectorGetArrayWrite_Hip()
560 CeedVector_Hip *impl; in CeedVectorNorm_Hip() local
567 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorNorm_Hip()
672 …CeedCallHip(ceed, hipMemcpyAsync(&norm_no_abs, impl->d_array + index - 1, sizeof(CeedScalar), hipM… in CeedVectorNorm_Hip()
707 …CeedCallHip(ceed, hipMemcpyAsync(&norm_no_abs, impl->d_array + index - 1, sizeof(CeedScalar), hipM… in CeedVectorNorm_Hip()
762 CeedVector_Hip *impl; in CeedVectorReciprocal_Hip() local
764 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorReciprocal_Hip()
767 if (impl->d_array) CeedCallBackend(CeedDeviceReciprocal_Hip(impl->d_array, length)); in CeedVectorReciprocal_Hip()
768 if (impl->h_array) CeedCallBackend(CeedHostReciprocal_Hip(impl->h_array, length)); in CeedVectorReciprocal_Hip()
790 CeedVector_Hip *impl; in CeedVectorScale_Hip() local
792 CeedCallBackend(CeedVectorGetData(x, &impl)); in CeedVectorScale_Hip()
795 if (impl->d_array) { in CeedVectorScale_Hip()
803 …CeedCallHipblas(CeedVectorReturnCeed(x), hipblasSscal_64(handle, (int64_t)length, &alpha, impl->d_… in CeedVectorScale_Hip()
805 …CeedCallHipblas(CeedVectorReturnCeed(x), hipblasDscal_64(handle, (int64_t)length, &alpha, impl->d_… in CeedVectorScale_Hip()
809 CeedCallBackend(CeedDeviceScale_Hip(impl->d_array, alpha, length)); in CeedVectorScale_Hip()
811 impl->h_array = NULL; in CeedVectorScale_Hip()
813 if (impl->h_array) { in CeedVectorScale_Hip()
814 CeedCallBackend(CeedHostScale_Hip(impl->h_array, alpha, length)); in CeedVectorScale_Hip()
815 impl->d_array = NULL; in CeedVectorScale_Hip()
951 CeedVector_Hip *impl; in CeedVectorDestroy_Hip() local
953 CeedCallBackend(CeedVectorGetData(vec, &impl)); in CeedVectorDestroy_Hip()
954 CeedCallHip(CeedVectorReturnCeed(vec), hipFree(impl->d_array_owned)); in CeedVectorDestroy_Hip()
955 CeedCallBackend(CeedFree(&impl->h_array_owned)); in CeedVectorDestroy_Hip()
956 CeedCallBackend(CeedFree(&impl)); in CeedVectorDestroy_Hip()
964 CeedVector_Hip *impl; in CeedVectorCreate_Hip() local
987 CeedCallBackend(CeedCalloc(1, &impl)); in CeedVectorCreate_Hip()
990 impl->has_unified_addressing = hip_impl->has_unified_addressing; in CeedVectorCreate_Hip()
991 CeedCallBackend(CeedVectorSetData(vec, impl)); in CeedVectorCreate_Hip()