266 static inline PetscErrorCode PetscSFLinkGetMPIBuffersAndRequests(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkGetMPIBuffersAndRequests()  argument
272 …if (link->InitMPIRequests) PetscCall((*link->InitMPIRequests)(sf, link, direction)); // init (pers…  in PetscSFLinkGetMPIBuffersAndRequests()
281 static inline PetscErrorCode PetscSFLinkStartCommunication(PetscSF sf, PetscSFLink link, PetscSFDir…  in PetscSFLinkStartCommunication()  argument
284   if (link->StartCommunication) PetscCall((*link->StartCommunication)(sf, link, direction));  in PetscSFLinkStartCommunication()
288 static inline PetscErrorCode PetscSFLinkFinishCommunication(PetscSF sf, PetscSFLink link, PetscSFDi…  in PetscSFLinkFinishCommunication()  argument
291   if (link->FinishCommunication) PetscCall((*link->FinishCommunication)(sf, link, direction));  in PetscSFLinkFinishCommunication()
300 static inline PetscErrorCode PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(PetscSF sf, PetscSFLi…  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()  argument
302   PetscSF_Basic *bas = (PetscSF_Basic *)sf->data;  in PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI()
321 static inline PetscErrorCode PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI(PetscSF sf, PetscSFLi…  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()  argument
324 …if (PetscMemTypeDevice(link->leafmtype) && PetscMemTypeHost(link->leafmtype_mpi) && sf->leafbuflen…  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
327     size_t count = sf->leafbuflen[PETSCSF_REMOTE] * link->unitbytes;  in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
340 static inline PetscErrorCode PetscSFLinkSyncStreamBeforeCallMPI(PetscSF sf, PetscSFLink link)  in PetscSFLinkSyncStreamBeforeCallMPI()  argument
342   PetscSF_Basic *bas = (PetscSF_Basic *)sf->data;  in PetscSFLinkSyncStreamBeforeCallMPI()
346 …as->rootbuflen[PETSCSF_REMOTE]) || (PetscMemTypeDevice(link->leafmtype) && sf->leafbuflen[PETSCSF_…  in PetscSFLinkSyncStreamBeforeCallMPI()
369 static inline PetscErrorCode PetscSFLinkGetRootPackOptAndIndices(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkGetRootPackOptAndIndices()  argument
371   PetscSF_Basic *bas = (PetscSF_Basic *)sf->data;  in PetscSFLinkGetRootPackOptAndIndices()
397 …PetscCall(PetscSFMalloc(sf, PETSC_MEMTYPE_DEVICE, size, (void **)&bas->rootpackopt_d[scope]->array…  in PetscSFLinkGetRootPackOptAndIndices()
404 …        PetscCall(PetscSFMalloc(sf, PETSC_MEMTYPE_DEVICE, size, (void **)&bas->irootloc_d[scope]));  in PetscSFLinkGetRootPackOptAndIndices()
418 static inline PetscErrorCode PetscSFLinkGetLeafPackOptAndIndices(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkGetLeafPackOptAndIndices()  argument
423   *count   = sf->leafbuflen[scope];  in PetscSFLinkGetLeafPackOptAndIndices()
424   *start   = sf->leafstart[scope];  in PetscSFLinkGetLeafPackOptAndIndices()
427   if (!sf->leafcontig[scope]) {  in PetscSFLinkGetLeafPackOptAndIndices()
428     offset = (scope == PETSCSF_LOCAL) ? 0 : sf->roffset[sf->ndranks];  in PetscSFLinkGetLeafPackOptAndIndices()
430       *opt     = sf->leafpackopt[scope];  in PetscSFLinkGetLeafPackOptAndIndices()
431       *indices = sf->rmine + offset;  in PetscSFLinkGetLeafPackOptAndIndices()
434       if (sf->leafpackopt[scope]) {  in PetscSFLinkGetLeafPackOptAndIndices()
435         if (!sf->leafpackopt_d[scope]) {  in PetscSFLinkGetLeafPackOptAndIndices()
436           PetscCall(PetscMalloc1(1, &sf->leafpackopt_d[scope]));  in PetscSFLinkGetLeafPackOptAndIndices()
437           PetscCall(PetscArraycpy(sf->leafpackopt_d[scope], sf->leafpackopt[scope], 1));  in PetscSFLinkGetLeafPackOptAndIndices()
438 …size = (sf->leafpackopt[scope]->n * 7 + 2) * sizeof(PetscInt);                                    …  in PetscSFLinkGetLeafPackOptAndIndices()
439 …PetscCall(PetscSFMalloc(sf, PETSC_MEMTYPE_DEVICE, size, (void **)&sf->leafpackopt_d[scope]->array)…  in PetscSFLinkGetLeafPackOptAndIndices()
440 …Call((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, sf->leafpackopt_d[scope]->array, PETSC_MEMTYPE_HO…  in PetscSFLinkGetLeafPackOptAndIndices()
442         *opt = sf->leafpackopt_d[scope];  in PetscSFLinkGetLeafPackOptAndIndices()
444         if (!sf->rmine_d[scope]) {  in PetscSFLinkGetLeafPackOptAndIndices()
445           size = sf->leafbuflen[scope] * sizeof(PetscInt);  in PetscSFLinkGetLeafPackOptAndIndices()
446           PetscCall(PetscSFMalloc(sf, PETSC_MEMTYPE_DEVICE, size, (void **)&sf->rmine_d[scope]));  in PetscSFLinkGetLeafPackOptAndIndices()
447 …PetscCall((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, sf->rmine_d[scope], PETSC_MEMTYPE_HOST, sf->…  in PetscSFLinkGetLeafPackOptAndIndices()
449         *indices = sf->rmine_d[scope];  in PetscSFLinkGetLeafPackOptAndIndices()