6 static PetscErrorCode PetscSFLinkFinishCommunication_Default(PetscSF sf, PetscSFLink link, PetscSFD…  in PetscSFLinkFinishCommunication_Default()  argument
9   const PetscMemType rootmtype_mpi = link->rootmtype_mpi, leafmtype_mpi = link->leafmtype_mpi;  in PetscSFLinkFinishCommunication_Default()
10   const PetscInt     rootdirect_mpi = link->rootdirect_mpi, leafdirect_mpi = link->leafdirect_mpi;  in PetscSFLinkFinishCommunication_Default()
23 …size_t size = (bas->ioffset[i + bas->ndiranks + 1] - bas->ioffset[i + bas->ndiranks]) * link->unit…  in PetscSFLinkFinishCommunication_Default()
24 …ytes) with MPI tag %10d ... ", rank, rootaction, bas->iranks[i + bas->ndiranks], size, link->tag));  in PetscSFLinkFinishCommunication_Default()
25 …PetscCallMPI(MPI_Wait(link->rootreqs[direction][rootmtype_mpi][rootdirect_mpi] + i, MPI_STATUS_IGN…  in PetscSFLinkFinishCommunication_Default()
29 … size_t size = (sf->roffset[i + sf->ndranks + 1] - sf->roffset[i + sf->ndranks]) * link->unitbytes;  in PetscSFLinkFinishCommunication_Default()
30 …zu bytes) with MPI tag %10d ... ", rank, leafaction, sf->ranks[i + sf->ndranks], size, link->tag));  in PetscSFLinkFinishCommunication_Default()
31 …PetscCallMPI(MPI_Wait(link->leafreqs[direction][leafmtype_mpi][leafdirect_mpi] + i, MPI_STATUS_IGN…  in PetscSFLinkFinishCommunication_Default()
36 …if (bas->nrootreqs) PetscCallMPI(MPI_Waitall(bas->nrootreqs, link->rootreqs[direction][rootmtype_m…  in PetscSFLinkFinishCommunication_Default()
37 …if (sf->nleafreqs) PetscCallMPI(MPI_Waitall(sf->nleafreqs, link->leafreqs[direction][leafmtype_mpi…  in PetscSFLinkFinishCommunication_Default()
41 …PetscCall(PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_FALSE /* host2device af…  in PetscSFLinkFinishCommunication_Default()
43     PetscCall(PetscSFLinkCopyRootBufferInCaseNotUseGpuAwareMPI(sf, link, PETSC_FALSE));  in PetscSFLinkFinishCommunication_Default()
68   PetscSFLink     *p, link;  in PetscSFLinkCreate_MPI()  local
118   for (p = &bas->avail; (link = *p); p = &link->next) {  in PetscSFLinkCreate_MPI()
119     if (!link->use_nvshmem) { /* Only check with MPI links */  in PetscSFLinkCreate_MPI()
120       PetscCall(MPIPetsc_Type_compare(unit, link->unit, &match));  in PetscSFLinkCreate_MPI()
125 …if (rootdirect_mpi && sf->persistent && link->rootreqsinited[direction][rootmtype][1] && link->roo…  in PetscSFLinkCreate_MPI()
126           reqs = link->rootreqs[direction][rootmtype][1]; /* Here, rootmtype = rootmtype_mpi */  in PetscSFLinkCreate_MPI()
130           link->rootreqsinited[direction][rootmtype][1] = PETSC_FALSE;  in PetscSFLinkCreate_MPI()
132 …if (leafdirect_mpi && sf->persistent && link->leafreqsinited[direction][leafmtype][1] && link->lea…  in PetscSFLinkCreate_MPI()
133           reqs = link->leafreqs[direction][leafmtype][1];  in PetscSFLinkCreate_MPI()
137           link->leafreqsinited[direction][leafmtype][1] = PETSC_FALSE;  in PetscSFLinkCreate_MPI()
139         *p = link->next; /* Remove from available list */  in PetscSFLinkCreate_MPI()
145   PetscCall(PetscNew(&link));  in PetscSFLinkCreate_MPI()
146   PetscCall(PetscSFLinkSetUp_Host(sf, link, unit));  in PetscSFLinkCreate_MPI()
147 …PetscCall(PetscCommGetNewTag(PetscObjectComm((PetscObject)sf), &link->tag)); /* One tag per link */  in PetscSFLinkCreate_MPI()
150   PetscCall(PetscMalloc1(nreqs, &link->reqs));  in PetscSFLinkCreate_MPI()
151 …for (i = 0; i < nreqs; i++) link->reqs[i] = MPI_REQUEST_NULL; /* Initialized to NULL so that we kn…  in PetscSFLinkCreate_MPI()
157           link->rootreqs[i][j][k] = link->reqs + nrootreqs * (4 * i + 2 * j + k);  in PetscSFLinkCreate_MPI()
158           link->leafreqs[i][j][k] = link->reqs + nrootreqs * 8 + nleafreqs * (4 * i + 2 * j + k);  in PetscSFLinkCreate_MPI()
163   link->FinishCommunication = PetscSFLinkFinishCommunication_Default;  in PetscSFLinkCreate_MPI()
166   PetscTryTypeMethod(sf, SetCommunicationOps, link);  in PetscSFLinkCreate_MPI()
171   if ((PetscMemTypeDevice(xrootmtype) || PetscMemTypeDevice(xleafmtype)) && !link->deviceinited) {  in PetscSFLinkCreate_MPI()
173 …if (sf->backend == PETSCSF_BACKEND_CUDA) PetscCall(PetscSFLinkSetUp_CUDA(sf, link, unit)); /* Setu…  in PetscSFLinkCreate_MPI()
176 …if (sf->backend == PETSCSF_BACKEND_HIP) PetscCall(PetscSFLinkSetUp_HIP(sf, link, unit)); /* Setup …  in PetscSFLinkCreate_MPI()
179     if (sf->backend == PETSCSF_BACKEND_KOKKOS) PetscCall(PetscSFLinkSetUp_Kokkos(sf, link, unit));  in PetscSFLinkCreate_MPI()
190         link->rootbuf[i][rootmtype] = (char *)rootdata + bas->rootstart[i] * link->unitbytes;  in PetscSFLinkCreate_MPI()
192 …if (!link->rootbuf_alloc[i][rootmtype]) PetscCall(PetscSFMalloc(sf, rootmtype, bas->rootbuflen[i] …  in PetscSFLinkCreate_MPI()
193         link->rootbuf[i][rootmtype] = link->rootbuf_alloc[i][rootmtype];  in PetscSFLinkCreate_MPI()
199         link->leafbuf[i][leafmtype] = (char *)leafdata + sf->leafstart[i] * link->unitbytes;  in PetscSFLinkCreate_MPI()
201 …if (!link->leafbuf_alloc[i][leafmtype]) PetscCall(PetscSFMalloc(sf, leafmtype, sf->leafbuflen[i] *…  in PetscSFLinkCreate_MPI()
202         link->leafbuf[i][leafmtype] = link->leafbuf_alloc[i][leafmtype];  in PetscSFLinkCreate_MPI()
210 …if (!link->rootbuf_alloc[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST]) PetscCall(PetscMalloc(bas->rootbufle…  in PetscSFLinkCreate_MPI()
211 …link->rootbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST] = link->rootbuf_alloc[PETSCSF_REMOTE][PETSC_MEMT…  in PetscSFLinkCreate_MPI()
214 …if (!link->leafbuf_alloc[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST]) PetscCall(PetscMalloc(sf->leafbuflen…  in PetscSFLinkCreate_MPI()
215 …link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST] = link->leafbuf_alloc[PETSCSF_REMOTE][PETSC_MEMT…  in PetscSFLinkCreate_MPI()
221     if (rootdirect_mpi) link->rootdatadirect[direction][rootmtype] = rootdata;  in PetscSFLinkCreate_MPI()
222     if (leafdirect_mpi) link->leafdatadirect[direction][leafmtype] = leafdata;  in PetscSFLinkCreate_MPI()
225   link->rootdata = rootdata; /* root/leafdata are keys to look up links in PetscSFXxxEnd */  in PetscSFLinkCreate_MPI()
226   link->leafdata = leafdata;  in PetscSFLinkCreate_MPI()
228     link->rootdirect[i] = rootdirect[i];  in PetscSFLinkCreate_MPI()
229     link->leafdirect[i] = leafdirect[i];  in PetscSFLinkCreate_MPI()
231   link->rootdirect_mpi = rootdirect_mpi;  in PetscSFLinkCreate_MPI()
232   link->leafdirect_mpi = leafdirect_mpi;  in PetscSFLinkCreate_MPI()
233   link->rootmtype      = rootmtype;  in PetscSFLinkCreate_MPI()
234   link->leafmtype      = leafmtype;  in PetscSFLinkCreate_MPI()
235   link->rootmtype_mpi  = rootmtype_mpi;  in PetscSFLinkCreate_MPI()
236   link->leafmtype_mpi  = leafmtype_mpi;  in PetscSFLinkCreate_MPI()
238   link->next = bas->inuse;  in PetscSFLinkCreate_MPI()
239   bas->inuse = link;  in PetscSFLinkCreate_MPI()
240   *mylink    = link;  in PetscSFLinkCreate_MPI()