33   PetscSFLink  link;  in PetscSFBcastBegin_Allgather()  local
40 …l(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, PETSCSF_BCAST, &link));  in PetscSFBcastBegin_Allgather()
41   PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_REMOTE, rootdata));  in PetscSFBcastBegin_Allgather()
42 …PetscCall(PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /* device2host bef…  in PetscSFBcastBegin_Allgather()
45 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re…  in PetscSFBcastBegin_Allgather()
46   PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link));  in PetscSFBcastBegin_Allgather()
53   PetscSFLink        link;  in PetscSFReduceBegin_Allgather()  local
62 …(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, PETSCSF_REDUCE, &link));  in PetscSFReduceBegin_Allgather()
66 …cCall((*link->Memcpy)(link, rootmtype, rootdata, leafmtype, (const char *)leafdata + (size_t)rstar…  in PetscSFReduceBegin_Allgather()
67 …ypeDevice(leafmtype) && PetscMemTypeHost(rootmtype)) PetscCall((*link->SyncStream)(link)); /* Sync…  in PetscSFReduceBegin_Allgather()
71     PetscCall(PetscSFLinkPackLeafData(sf, link, PETSCSF_REMOTE, leafdata));  in PetscSFReduceBegin_Allgather()
72 …PetscCall(PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /* device2host bef…  in PetscSFReduceBegin_Allgather()
73 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_LEAF2ROOT, &rootbuf, &leafbuf, &re…  in PetscSFReduceBegin_Allgather()
75 …link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi]) PetscCall(PetscSFMalloc(sf, link->leafmt…  in PetscSFReduceBegin_Allgather()
76 …if (rank == 0 && link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi] == leafbuf) leafbuf = MP…  in PetscSFReduceBegin_Allgather()
77     PetscCall(PetscMPIIntCast(sf->nleaves * link->bs, &count));  in PetscSFReduceBegin_Allgather()
78     PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link));  in PetscSFReduceBegin_Allgather()
79 …PetscCallMPI(MPI_Reduce(leafbuf, link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi], count, …  in PetscSFReduceBegin_Allgather()
80 …PetscCallMPI(MPIU_Iscatter(link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi], recvcount, un…  in PetscSFReduceBegin_Allgather()
87   PetscSFLink  link;  in PetscSFBcastToZero_Allgather()  local
95 …LinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, MPI_REPLACE, PETSCSF_BCAST, &link));  in PetscSFBcastToZero_Allgather()
96   PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_REMOTE, rootdata));  in PetscSFBcastToZero_Allgather()
97 …PetscCall(PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /* device2host bef…  in PetscSFBcastToZero_Allgather()
100 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re…  in PetscSFBcastToZero_Allgather()
101   PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link));  in PetscSFBcastToZero_Allgather()
103   PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFBcastToZero_Allgather()
104   PetscCall(PetscSFLinkFinishCommunication(sf, link, PETSCSF_ROOT2LEAF));  in PetscSFBcastToZero_Allgather()
107 …link->Memcpy)(link, PETSC_MEMTYPE_DEVICE, leafdata, PETSC_MEMTYPE_HOST, link->leafbuf[PETSCSF_REMO…  in PetscSFBcastToZero_Allgather()
109   PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFBcastToZero_Allgather()