| /petsc/src/vec/is/sf/impls/basic/allgather/ |
| H A D | sfallgather.c | 36 void *rootbuf = NULL, *leafbuf = NULL; /* buffer seen by MPI */ in PetscSFBcastBegin_Allgather() local
45 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re… in PetscSFBcastBegin_Allgather()
47 PetscCallMPI(MPIU_Iallgather(rootbuf, sendcount, unit, leafbuf, sendcount, unit, comm, req)); in PetscSFBcastBegin_Allgather()
57 void *rootbuf = NULL, *leafbuf = NULL; /* buffer seen by MPI */ in PetscSFReduceBegin_Allgather() local
73 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_LEAF2ROOT, &rootbuf, &leafbuf, &re… in PetscSFReduceBegin_Allgather()
76 …if (rank == 0 && link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi] == leafbuf) leafbuf = MP… in PetscSFReduceBegin_Allgather()
79 …PetscCallMPI(MPI_Reduce(leafbuf, link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi], count, … in PetscSFReduceBegin_Allgather()
91 void *rootbuf = NULL, *leafbuf = NULL; in PetscSFBcastToZero_Allgather() local
100 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re… in PetscSFBcastToZero_Allgather()
102 …PetscCallMPI(MPIU_Igather(rootbuf == leafbuf ? MPI_IN_PLACE : rootbuf, sendcount, unit, leafbuf, s… in PetscSFBcastToZero_Allgather()
[all …]
|
| /petsc/src/vec/is/sf/impls/basic/gather/ |
| H A D | sfgather.c | 10 void *rootbuf = NULL, *leafbuf = NULL; in PetscSFLinkStartCommunication_Gather() local
23 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, &rootbuf, &leafbuf, &req, NULL)… in PetscSFLinkStartCommunication_Gather()
27 …PetscCallMPI(MPIU_Igather(rootbuf == leafbuf ? MPI_IN_PLACE : rootbuf, count, unit, leafbuf, count… in PetscSFLinkStartCommunication_Gather()
29 …PetscCallMPI(MPIU_Iscatter(leafbuf, count, unit, rootbuf == leafbuf ? MPI_IN_PLACE : rootbuf, coun… in PetscSFLinkStartCommunication_Gather()
|
| /petsc/src/vec/is/sf/impls/basic/allgatherv/ |
| H A D | sfallgatherv.c | 107 void *rootbuf = NULL, *leafbuf = NULL; in PetscSFBcastBegin_Allgatherv() local
118 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_ROOT2LEAF, &rootbuf, &leafbuf, &re… in PetscSFBcastBegin_Allgatherv()
120 …== dat->bcast_root) PetscCall((*link->Memcpy)(link, link->leafmtype_mpi, leafbuf, link->rootmtype_… in PetscSFBcastBegin_Allgatherv()
124 …if (dat->bcast_pattern) PetscCallMPI(MPIU_Ibcast(leafbuf, nleaves, unit, dat->bcast_root, comm, re… in PetscSFBcastBegin_Allgatherv()
125 …else PetscCallMPI(MPIU_Iallgatherv(rootbuf, sendcount, unit, leafbuf, dat->recvcounts, dat->displs… in PetscSFBcastBegin_Allgatherv()
136 void *rootbuf = NULL, *leafbuf = NULL; in PetscSFReduceBegin_Allgatherv() local
150 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, PETSCSF_LEAF2ROOT, &rootbuf, &leafbuf, &re… in PetscSFReduceBegin_Allgatherv()
176 PetscCallMPI(MPIU_Reduce(leafbuf, rootbuf, nleavesi, baseunit, op, dat->bcast_root, comm)); in PetscSFReduceBegin_Allgatherv()
178 … PetscCallMPI(MPIU_Ireduce(leafbuf, rootbuf, nleavesi, baseunit, op, dat->bcast_root, comm, req)); in PetscSFReduceBegin_Allgatherv()
186 …if (rank == 0 && link->leafbuf_alloc[PETSCSF_REMOTE][link->leafmtype_mpi] == leafbuf) leafbuf = MP… in PetscSFReduceBegin_Allgatherv()
[all …]
|
| /petsc/src/vec/is/sf/impls/basic/gatherv/ |
| H A D | sfgatherv.c | 13 void *rootbuf = NULL, *leafbuf = NULL; /* buffer seen by MPI */ in PetscSFLinkStartCommunication_Gatherv() local
25 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, &rootbuf, &leafbuf, &req, NULL)… in PetscSFLinkStartCommunication_Gatherv()
29 …PetscCallMPI(MPIU_Igatherv(rootbuf, count, unit, leafbuf, dat->recvcounts, dat->displs, unit, 0 /*… in PetscSFLinkStartCommunication_Gatherv()
31 …PetscCallMPI(MPIU_Iscatterv(leafbuf, dat->recvcounts, dat->displs, unit, rootbuf, count, unit, 0, … in PetscSFLinkStartCommunication_Gatherv()
|
| /petsc/src/vec/is/sf/impls/basic/neighbor/ |
| H A D | sfneighbor.c | 91 void *rootbuf = NULL, *leafbuf = NULL; in PetscSFLinkStartCommunication_Neighbor() local
102 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, &rootbuf, &leafbuf, &req, NULL)… in PetscSFLinkStartCommunication_Neighbor()
107 …eighbor_alltoallv(rootbuf, dat->rootcounts, dat->rootdispls, link->unit, leafbuf, dat->leafcounts,… in PetscSFLinkStartCommunication_Neighbor()
110 …PetscCallMPI(MPIU_Ineighbor_alltoallv(leafbuf, dat->leafcounts, dat->leafdispls, link->unit, rootb… in PetscSFLinkStartCommunication_Neighbor()
125 … *rootbuf = link->rootbuf[PETSCSF_REMOTE][rootmtype_mpi], *leafbuf = link->leafbuf[PETSCSF_R… in PetscSFLinkInitMPIRequests_Persistent_Neighbor() local
134 …or_alltoallv_init(rootbuf, dat->rootcounts, dat->rootdispls, link->unit, leafbuf, dat->leafcounts,… in PetscSFLinkInitMPIRequests_Persistent_Neighbor()
136 …PetscCallMPI(MPIU_Neighbor_alltoallv_init(leafbuf, dat->leafcounts, dat->leafdispls, link->unit, r… in PetscSFLinkInitMPIRequests_Persistent_Neighbor()
|
| /petsc/src/vec/is/sf/impls/basic/alltoall/ |
| H A D | sfalltoall.c | 11 void *rootbuf = NULL, *leafbuf = NULL; in PetscSFLinkStartCommunication_Alltoall() local
22 …PetscCall(PetscSFLinkGetMPIBuffersAndRequests(sf, link, direction, &rootbuf, &leafbuf, &req, NULL)… in PetscSFLinkStartCommunication_Alltoall()
26 PetscCallMPI(MPIU_Ialltoall(rootbuf, 1, unit, leafbuf, 1, unit, comm, req)); in PetscSFLinkStartCommunication_Alltoall()
28 PetscCallMPI(MPIU_Ialltoall(leafbuf, 1, unit, rootbuf, 1, unit, comm, req)); in PetscSFLinkStartCommunication_Alltoall()
|
| /petsc/src/vec/is/sf/impls/basic/ |
| H A D | sfpack.h | 189 …char *leafbuf[2][2]; /* Buffers for packed leaves, in layout of [PETSCSF_L… member
266 …sf, PetscSFLink link, PetscSFDirection direction, void **rootbuf, void **leafbuf, MPI_Request **ro… in PetscSFLinkGetMPIBuffersAndRequests() argument
275 if (leafbuf) *leafbuf = link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi]; in PetscSFLinkGetMPIBuffersAndRequests()
325 void *h_buf = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST]; in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
326 void *d_buf = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE]; in PetscSFLinkCopyLeafBufferInCaseNotUseGpuAwareMPI()
|
| H A D | sfmpi.c | 199 link->leafbuf[i][leafmtype] = (char *)leafdata + sf->leafstart[i] * link->unitbytes; in PetscSFLinkCreate_MPI()
202 link->leafbuf[i][leafmtype] = link->leafbuf_alloc[i][leafmtype]; in PetscSFLinkCreate_MPI()
215 …link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_HOST] = link->leafbuf_alloc[PETSCSF_REMOTE][PETSC_MEMT… in PetscSFLinkCreate_MPI()
|
| H A D | sfbasic.c | 43 …PetscCallMPI(MPIU_Send_init(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf->ra… in PetscSFLinkInitMPIRequests_Persistent_Basic()
49 …PetscCallMPI(MPIU_Recv_init(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf->ra… in PetscSFLinkInitMPIRequests_Persistent_Basic()
135 …PetscCallMPI(MPIX_Isend_enqueue(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf… in PetscSFLinkStartCommunication_MPIX_Stream()
141 …PetscCallMPI(MPIX_Irecv_enqueue(link->leafbuf[PETSCSF_REMOTE][leafmtype_mpi] + disp, cnt, unit, sf… in PetscSFLinkStartCommunication_MPIX_Stream()
|
| H A D | sfpack.c | 949 …PetscCall((*Pack)(link, count, start, opt, leafindices, leafdata, link->leafbuf[scope][leafmtype])… in PetscSFLinkPackLeafData_Private()
1020 …PetscCall((*UnpackAndOp)(link, count, start, opt, leafindices, leafdata, link->leafbuf[scope][leaf… in PetscSFLinkUnpackLeafData_Private()
1023 …aWithMPIReduceLocal(sf, link, count, start, leafindices, leafdata, link->leafbuf[scope][leafmtype]… in PetscSFLinkUnpackLeafData_Private()
1102 dstbuf = link->leafbuf[PETSCSF_LOCAL][leafmtype]; in PetscSFLinkScatterLocal()
1106 srcbuf = link->leafbuf[PETSCSF_LOCAL][leafmtype]; in PetscSFLinkScatterLocal()
|
| /petsc/src/vec/is/sf/impls/basic/nvshmem/ |
| H A D | sfnvshmem.cu | 366 …dst = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE]; /* recv buf is the local leaf buf… in PetscSFLinkGetDataBegin_NVSHMEM()
376 src = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE]; /* leaf buf is the send buf */ in PetscSFLinkGetDataBegin_NVSHMEM()
531 dst = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE]; in PetscSFLinkPutDataBegin_NVSHMEM()
543 src = link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE]; in PetscSFLinkPutDataBegin_NVSHMEM()
776 …link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE] = (char *)leafdata + sf->leafstart[PETSCSF_REM… in PetscSFLinkCreate_NVSHMEM()
779 …link->leafbuf[PETSCSF_REMOTE][PETSC_MEMTYPE_DEVICE] = link->leafbuf_alloc[PETSCSF_REMOTE][PETSC_ME… in PetscSFLinkCreate_NVSHMEM()
|
| /petsc/src/vec/is/sf/impls/basic/kokkos/ |
| H A D | sfkok.kokkos.cxx | 279 Type *rootdata = static_cast<Type *>(data), *leafbuf = static_cast<Type *>(buf); in FetchAndOp()
287 for (int i = 0; i < MBS; i++) leafbuf[l + i] = op(rootdata[r + i], leafbuf[l + i]); in FetchAndOp()
|
| /petsc/src/vec/is/sf/impls/basic/cupm/ |
| H A D | sfcupm_impl.hpp | 83 …etscInt rootstart, const PetscInt *rootopt, const PetscInt *rootidx, Type *rootdata, Type *leafbuf) in d_FetchAndOp() argument
91 for (PetscInt i = 0; i < MBS; i++) leafbuf[l + i] = op(rootdata[r + i], leafbuf[l + i]); in d_FetchAndOp()
|