6 static PetscErrorCode PetscSFLinkInitMPIRequests_Persistent_Basic(PetscSF sf, PetscSFLink link, Pet…  in PetscSFLinkInitMPIRequests_Persistent_Basic()  argument
14   MPI_Datatype       unit          = link->unit;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
15 …const PetscMemType rootmtype_mpi = link->rootmtype_mpi, leafmtype_mpi = link->leafmtype_mpi; /* Us…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
16   const PetscInt     rootdirect_mpi = link->rootdirect_mpi, leafdirect_mpi = link->leafdirect_mpi;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
19 …if (bas->rootbuflen[PETSCSF_REMOTE] && !link->rootreqsinited[direction][rootmtype_mpi][rootdirect_…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
23         disp = (rootoffset[i] - rootoffset[ndrootranks]) * link->unitbytes;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
25 …tscCallMPI(MPIU_Recv_init(link->rootbuf[PETSCSF_REMOTE][rootmtype_mpi] + disp, cnt, unit, bas->ira…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
29         disp = (rootoffset[i] - rootoffset[ndrootranks]) * link->unitbytes;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
31 …tscCallMPI(MPIU_Send_init(link->rootbuf[PETSCSF_REMOTE][rootmtype_mpi] + disp, cnt, unit, bas->ira…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
34     link->rootreqsinited[direction][rootmtype_mpi][rootdirect_mpi] = PETSC_TRUE;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
37 …if (sf->leafbuflen[PETSCSF_REMOTE] && !link->leafreqsinited[direction][leafmtype_mpi][leafdirect_m…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
41         disp = (leafoffset[i] - leafoffset[ndleafranks]) * link->unitbytes;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
43 …etscCallMPI(MPIU_Send_init(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf->ran…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
47         disp = (leafoffset[i] - leafoffset[ndleafranks]) * link->unitbytes;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
49 …etscCallMPI(MPIU_Recv_init(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf->ran…  in PetscSFLinkInitMPIRequests_Persistent_Basic()
52     link->leafreqsinited[direction][leafmtype_mpi][leafdirect_mpi] = PETSC_TRUE;  in PetscSFLinkInitMPIRequests_Persistent_Basic()
58 static PetscErrorCode PetscSFLinkStartCommunication_Persistent_Basic(PetscSF sf, PetscSFLink link, …  in PetscSFLinkStartCommunication_Persistent_Basic()  argument
70       PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, NULL, NULL, NULL, &rreqs));  in PetscSFLinkStartCommunication_Persistent_Basic()
73       PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, NULL, NULL, &rreqs, NULL));  in PetscSFLinkStartCommunication_Persistent_Basic()
81 …PetscCall(PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE /*device2host befo…  in PetscSFLinkStartCommunication_Persistent_Basic()
82       PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, NULL, NULL, &sreqs, NULL));  in PetscSFLinkStartCommunication_Persistent_Basic()
85       PetscCall(PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_TRUE));  in PetscSFLinkStartCommunication_Persistent_Basic()
86       PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, NULL, NULL, NULL, &sreqs));  in PetscSFLinkStartCommunication_Persistent_Basic()
89 …PetscCall(PetscSFLinkSyncStreamBeforeCallMPI(sf, link)); // need to sync the stream to make BOTH s…  in PetscSFLinkStartCommunication_Persistent_Basic()
90   if (rbuflen) PetscCallMPI(MPI_Startall_irecv(rbuflen, link->unit, nrreqs, rreqs));  in PetscSFLinkStartCommunication_Persistent_Basic()
91   if (sbuflen) PetscCallMPI(MPI_Startall_isend(sbuflen, link->unit, nsreqs, sreqs));  in PetscSFLinkStartCommunication_Persistent_Basic()
97 static PetscErrorCode PetscSFLinkStartCommunication_MPIX_Stream(PetscSF sf, PetscSFLink link, Petsc…  in PetscSFLinkStartCommunication_MPIX_Stream()  argument
105   MPI_Datatype       unit          = link->unit;  in PetscSFLinkStartCommunication_MPIX_Stream()
106 …const PetscMemType rootmtype_mpi = link->rootmtype_mpi, leafmtype_mpi = link->leafmtype_mpi; /* Us…  in PetscSFLinkStartCommunication_MPIX_Stream()
107   const PetscInt     rootdirect_mpi = link->rootdirect_mpi, leafdirect_mpi = link->leafdirect_mpi;  in PetscSFLinkStartCommunication_MPIX_Stream()
114         disp = (rootoffset[i] - rootoffset[ndrootranks]) * link->unitbytes;  in PetscSFLinkStartCommunication_MPIX_Stream()
116 …PI(MPIX_Irecv_enqueue(link->rootbuf[PETSCSF_REMOTE][rootmtype_mpi] + disp, cnt, unit, bas->iranks[…  in PetscSFLinkStartCommunication_MPIX_Stream()
120         disp = (rootoffset[i] - rootoffset[ndrootranks]) * link->unitbytes;  in PetscSFLinkStartCommunication_MPIX_Stream()
123 …PI(MPIX_Isend_enqueue(link->rootbuf[PETSCSF_REMOTE][rootmtype_mpi] + disp, cnt, unit, bas->iranks[…  in PetscSFLinkStartCommunication_MPIX_Stream()
132         disp = (leafoffset[i] - leafoffset[ndleafranks]) * link->unitbytes;  in PetscSFLinkStartCommunication_MPIX_Stream()
135 …MPI(MPIX_Isend_enqueue(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf->ranks[i…  in PetscSFLinkStartCommunication_MPIX_Stream()
139         disp = (leafoffset[i] - leafoffset[ndleafranks]) * link->unitbytes;  in PetscSFLinkStartCommunication_MPIX_Stream()
141 …MPI(MPIX_Irecv_enqueue(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf->ranks[i…  in PetscSFLinkStartCommunication_MPIX_Stream()
148 static PetscErrorCode PetscSFLinkFinishCommunication_MPIX_Stream(PetscSF sf, PetscSFLink link, Pets…  in PetscSFLinkFinishCommunication_MPIX_Stream()  argument
151   const PetscMemType rootmtype_mpi = link->rootmtype_mpi, leafmtype_mpi = link->leafmtype_mpi;  in PetscSFLinkFinishCommunication_MPIX_Stream()
152   const PetscInt     rootdirect_mpi = link->rootdirect_mpi, leafdirect_mpi = link->leafdirect_mpi;  in PetscSFLinkFinishCommunication_MPIX_Stream()
155 …PetscCallMPI(MPIX_Waitall_enqueue(bas->nrootreqs, link->rootreqs[direction][rootmtype_mpi][rootdir…  in PetscSFLinkFinishCommunication_MPIX_Stream()
156 …PetscCallMPI(MPIX_Waitall_enqueue(sf->nleafreqs, link->leafreqs[direction][leafmtype_mpi][leafdire…  in PetscSFLinkFinishCommunication_MPIX_Stream()
161 static PetscErrorCode PetscSFSetCommunicationOps_Basic(PetscSF sf, PetscSFLink link)  in PetscSFSetCommunicationOps_Basic()  argument
164   link->InitMPIRequests    = PetscSFLinkInitMPIRequests_Persistent_Basic;  in PetscSFSetCommunicationOps_Basic()
165   link->StartCommunication = PetscSFLinkStartCommunication_Persistent_Basic;  in PetscSFSetCommunicationOps_Basic()
167   const PetscMemType rootmtype_mpi = link->rootmtype_mpi, leafmtype_mpi = link->leafmtype_mpi;  in PetscSFSetCommunicationOps_Basic()
169     link->StartCommunication  = PetscSFLinkStartCommunication_MPIX_Stream;  in PetscSFSetCommunicationOps_Basic()
170     link->FinishCommunication = PetscSFLinkFinishCommunication_MPIX_Stream;  in PetscSFSetCommunicationOps_Basic()
261   PetscSFLink    link = bas->avail, next;  in PetscSFReset_Basic()  local
276   for (; link; link = next) {  in PetscSFReset_Basic()
277     next = link->next;  in PetscSFReset_Basic()
278     PetscCall(PetscSFLinkDestroy(sf, link));  in PetscSFReset_Basic()
354   PetscSFLink link = NULL;  in PetscSFBcastBegin_Basic()  local
358 …l(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, PETSCSF_BCAST, &link));  in PetscSFBcastBegin_Basic()
360   PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_REMOTE, rootdata));  in PetscSFBcastBegin_Basic()
362   PetscCall(PetscSFLinkStartCommunication(sf, link, PETSCSF_ROOT2LEAF));  in PetscSFBcastBegin_Basic()
364   PetscCall(PetscSFLinkScatterLocal(sf, link, PETSCSF_ROOT2LEAF, (void *)rootdata, leafdata, op));  in PetscSFBcastBegin_Basic()
370   PetscSFLink link = NULL;  in PetscSFBcastEnd_Basic()  local
374   PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFBcastEnd_Basic()
376   PetscCall(PetscSFLinkFinishCommunication(sf, link, PETSCSF_ROOT2LEAF));  in PetscSFBcastEnd_Basic()
378   PetscCall(PetscSFLinkUnpackLeafData(sf, link, PETSCSF_REMOTE, leafdata, op));  in PetscSFBcastEnd_Basic()
380   PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFBcastEnd_Basic()
387   PetscSFLink link = NULL;  in PetscSFLeafToRootBegin_Basic()  local
390   PetscCall(PetscSFLinkCreate(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, sfop, &link));  in PetscSFLeafToRootBegin_Basic()
391   PetscCall(PetscSFLinkPackLeafData(sf, link, PETSCSF_REMOTE, leafdata));  in PetscSFLeafToRootBegin_Basic()
392   PetscCall(PetscSFLinkStartCommunication(sf, link, PETSCSF_LEAF2ROOT));  in PetscSFLeafToRootBegin_Basic()
393   *out = link;  in PetscSFLeafToRootBegin_Basic()
400   PetscSFLink link = NULL;  in PetscSFReduceBegin_Basic()  local
403 …fToRootBegin_Basic(sf, unit, leafmtype, leafdata, rootmtype, rootdata, op, PETSCSF_REDUCE, &link));  in PetscSFReduceBegin_Basic()
404   PetscCall(PetscSFLinkScatterLocal(sf, link, PETSCSF_LEAF2ROOT, rootdata, (void *)leafdata, op));  in PetscSFReduceBegin_Basic()
410   PetscSFLink link = NULL;  in PetscSFReduceEnd_Basic()  local
413   PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFReduceEnd_Basic()
414   PetscCall(PetscSFLinkFinishCommunication(sf, link, PETSCSF_LEAF2ROOT));  in PetscSFReduceEnd_Basic()
415   PetscCall(PetscSFLinkUnpackRootData(sf, link, PETSCSF_REMOTE, rootdata, op));  in PetscSFReduceEnd_Basic()
416   PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFReduceEnd_Basic()
422   PetscSFLink link = NULL;  in PetscSFFetchAndOpBegin_Basic()  local
425 …afToRootBegin_Basic(sf, unit, leafmtype, leafdata, rootmtype, rootdata, op, PETSCSF_FETCH, &link));  in PetscSFFetchAndOpBegin_Basic()
426   PetscCall(PetscSFLinkFetchAndOpLocal(sf, link, rootdata, leafdata, leafupdate, op));  in PetscSFFetchAndOpBegin_Basic()
432   PetscSFLink link = NULL;  in PetscSFFetchAndOpEnd_Basic()  local
435   PetscCall(PetscSFLinkGetInUse(sf, unit, rootdata, leafdata, PETSC_OWN_POINTER, &link));  in PetscSFFetchAndOpEnd_Basic()
437   PetscCall(PetscSFLinkFinishCommunication(sf, link, PETSCSF_LEAF2ROOT));  in PetscSFFetchAndOpEnd_Basic()
439   PetscCall(PetscSFLinkFetchAndOpRemote(sf, link, rootdata, op));  in PetscSFFetchAndOpEnd_Basic()
441   PetscCall(PetscSFLinkStartCommunication(sf, link, PETSCSF_ROOT2LEAF));  in PetscSFFetchAndOpEnd_Basic()
442   PetscCall(PetscSFLinkFinishCommunication(sf, link, PETSCSF_ROOT2LEAF));  in PetscSFFetchAndOpEnd_Basic()
444   PetscCall(PetscSFLinkUnpackLeafData(sf, link, PETSCSF_REMOTE, leafupdate, MPI_REPLACE));  in PetscSFFetchAndOpEnd_Basic()
445   PetscCall(PetscSFLinkReclaim(sf, &link));  in PetscSFFetchAndOpEnd_Basic()