77   PetscSFLink         link = dat->avail, next;  in PetscSFReset_Allgatherv()  local
86   for (; link; link = next) {  in PetscSFReset_Allgatherv()
87     next = link->next;  in PetscSFReset_Allgatherv()
88     PetscCall(PetscSFLinkDestroy(sf, link));  in PetscSFReset_Allgatherv()
104   PetscSFLink         link;  in PetscSFBcastBegin_Allgatherv()  local
112 …l(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, PETSCSF_BCAST, &link));  in PetscSFBcastBegin_Allgatherv()
113   PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_REMOTE, rootdata));  in PetscSFBcastBegin_Allgatherv()
114 …PetscCall(PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /* device2host bef…  in PetscSFBcastBegin_Allgatherv()
118 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re…  in PetscSFBcastBegin_Allgatherv()
120 …bcast_root) PetscCall((*link->Memcpy)(link, link->leafmtype_mpi, leafbuf, link->rootmtype_mpi, roo…  in PetscSFBcastBegin_Allgatherv()
122   PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link));  in PetscSFBcastBegin_Allgatherv()
131   PetscSFLink         link;  in PetscSFReduceBegin_Allgatherv()  local
140 …(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, PETSCSF_REDUCE, &link));  in PetscSFReduceBegin_Allgatherv()
144 …cCall((*link->Memcpy)(link, rootmtype, rootdata, leafmtype, (const char *)leafdata + (size_t)rstar…  in PetscSFReduceBegin_Allgatherv()
145 …PetscMemTypeDevice(leafmtype) && PetscMemTypeHost(rootmtype)) PetscCall((*link->SyncStream)(link));  in PetscSFReduceBegin_Allgatherv()
148     PetscCall(PetscSFLinkPackLeafData(sf, link, PETSCSF_REMOTE, leafdata));  in PetscSFReduceBegin_Allgatherv()
149 …PetscCall(PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /* device2host bef…  in PetscSFReduceBegin_Allgatherv()
150 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_LEAF2ROOT, &rootbuf, &leafbuf, &re…  in PetscSFReduceBegin_Allgatherv()
151     PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link));  in PetscSFReduceBegin_Allgatherv()
184 …link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi]) PetscCall(PetscSFMalloc(sf, link->leafmt…  in PetscSFReduceBegin_Allgatherv()
186 …if (rank == 0 && link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi] == leafbuf) leafbuf = MP…  in PetscSFReduceBegin_Allgatherv()
187       PetscCall(PetscMPIIntCast(sf->nleaves * link->bs, &count));  in PetscSFReduceBegin_Allgatherv()
188 …PetscCallMPI(MPI_Reduce(leafbuf, link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi], count, …  in PetscSFReduceBegin_Allgatherv()
189 …PetscCallMPI(MPIU_Iscatterv(link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi], dat->recvcou…  in PetscSFReduceBegin_Allgatherv()
197   PetscSFLink link;  in PetscSFReduceEnd_Allgatherv()  local
207     PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFReduceEnd_Allgatherv()
208     PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFReduceEnd_Allgatherv()
217   PetscSFLink         link;  in PetscSFBcastToZero_Allgatherv()  local
226 …LinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, MPI_REPLACE, PETSCSF_BCAST, &link));  in PetscSFBcastToZero_Allgatherv()
227   PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_REMOTE, rootdata));  in PetscSFBcastToZero_Allgatherv()
228 …PetscCall(PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /* device2host bef…  in PetscSFBcastToZero_Allgatherv()
231 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re…  in PetscSFBcastToZero_Allgatherv()
232   PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link));  in PetscSFBcastToZero_Allgatherv()
235   PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFBcastToZero_Allgatherv()
236   PetscCall(PetscSFLinkFinishCommunication(sf, link, PETSCSF_ROOT2LEAF));  in PetscSFBcastToZero_Allgatherv()
238 …l((*link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, leafdata, PETSC_MEMTYPE_HOST, link->leafbuf[PETSC_ME…  in PetscSFBcastToZero_Allgatherv()
239   PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFBcastToZero_Allgatherv()
283   PetscSFLink link;  in PetscSFFetchAndOpBegin_Allgatherv()  local
291 …l(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, PETSCSF_FETCH, &link));  in PetscSFFetchAndOpBegin_Allgatherv()
292   PetscCall(PetscSFLinkPackLeafData(sf, link, PETSCSF_REMOTE, leafdata)); /* Sync the device */  in PetscSFFetchAndOpBegin_Allgatherv()
293 …PetscCall((*link->Memcpy)(link, leafmtype, leafupdate, leafmtype, leafdata, sf->nleaves * link->un…  in PetscSFFetchAndOpBegin_Allgatherv()
294   PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFFetchAndOpBegin_Allgatherv()
304 …PetscCallMPI(MPI_Sendrecv_replace(leafupdate, count, unit, next, link->tag, prev, link->tag, comm,…  in PetscSFFetchAndOpBegin_Allgatherv()
306     PetscCall(PetscMPIIntCast(sf->nleaves * link->bs, &count));  in PetscSFFetchAndOpBegin_Allgatherv()
307     PetscCallMPI(MPI_Exscan(MPI_IN_PLACE, leafupdate, count, link->basicunit, op, comm));  in PetscSFFetchAndOpBegin_Allgatherv()
309   PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFFetchAndOpBegin_Allgatherv()