Lines Matching refs:sf
395 PetscErrorCode PetscSFLinkDestroy(PetscSF sf, PetscSFLink link) in PetscSFLinkDestroy() argument
397 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkDestroy()
398 PetscInt i, nreqs = (bas->nrootreqs + sf->nleafreqs) * 8; in PetscSFLinkDestroy()
402 if (link->deviceinited) PetscCall((*link->Destroy)(sf, link)); in PetscSFLinkDestroy()
420 PetscErrorCode PetscSFLinkCreate(PetscSF sf, MPI_Datatype unit, PetscMemType rootmtype, const void … in PetscSFLinkCreate() argument
423 PetscCall(PetscSFSetErrorOnUnsupportedOverlap(sf, unit, rootdata, leafdata)); in PetscSFLinkCreate()
427 PetscCall(PetscSFLinkNvshmemCheck(sf, rootmtype, rootdata, leafmtype, leafdata, &use_nvshmem)); in PetscSFLinkCreate()
429 …PetscCall(PetscSFLinkCreate_NVSHMEM(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, sfop, … in PetscSFLinkCreate()
434 …PetscCall(PetscSFLinkCreate_MPI(sf, unit, rootmtype, rootdata, leafmtype, leafdata, op, sfop, myli… in PetscSFLinkCreate()
438 PetscErrorCode PetscSFLinkGetInUse(PetscSF sf, MPI_Datatype unit, const void *rootdata, const void … in PetscSFLinkGetInUse() argument
441 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkGetInUse()
465 PetscErrorCode PetscSFLinkReclaim(PetscSF sf, PetscSFLink *mylink) in PetscSFLinkReclaim() argument
467 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkReclaim()
480 PetscErrorCode PetscSFSetErrorOnUnsupportedOverlap(PetscSF sf, MPI_Datatype unit, const void *rootd… in PetscSFSetErrorOnUnsupportedOverlap() argument
483 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFSetErrorOnUnsupportedOverlap()
508 PetscErrorCode PetscSFLinkSetUp_Host(PetscSF sf, PetscSFLink link, MPI_Datatype unit) in PetscSFLinkSetUp_Host() argument
819 static inline PetscErrorCode PetscSFLinkLogFlopsAfterUnpackRootData(PetscSF sf, PetscSFLink link, P… in PetscSFLinkLogFlopsAfterUnpackRootData() argument
821 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkLogFlopsAfterUnpackRootData()
834 static inline PetscErrorCode PetscSFLinkLogFlopsAfterUnpackLeafData(PetscSF sf, PetscSFLink link, P… in PetscSFLinkLogFlopsAfterUnpackLeafData() argument
839 …if (PetscMemTypeDevice(link->leafmtype)) PetscCall(PetscLogGpuFlops(sf->leafbuflen[scope] * link->… in PetscSFLinkLogFlopsAfterUnpackLeafData()
842 PetscCall(PetscLogFlops(sf->leafbuflen[scope] * link->bs)); in PetscSFLinkLogFlopsAfterUnpackLeafData()
860 static inline PetscErrorCode PetscSFLinkUnpackDataWithMPIReduceLocal(PetscSF sf, PetscSFLink link, … in PetscSFLinkUnpackDataWithMPIReduceLocal() argument
882 static inline PetscErrorCode PetscSFLinkScatterDataWithMPIReduceLocal(PetscSF sf, PetscSFLink link,… in PetscSFLinkScatterDataWithMPIReduceLocal() argument
889 …PetscCall(PetscSFLinkUnpackDataWithMPIReduceLocal(sf, link, count, dstStart, dstIdx, dst, (const c… in PetscSFLinkScatterDataWithMPIReduceLocal()
919 static PetscErrorCode PetscSFLinkPackRootData_Private(PetscSF sf, PetscSFLink link, PetscSFScope sc… in PetscSFLinkPackRootData_Private() argument
929 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, scope, &count, &start, &opt, &r… in PetscSFLinkPackRootData_Private()
937 static PetscErrorCode PetscSFLinkPackLeafData_Private(PetscSF sf, PetscSFLink link, PetscSFScope sc… in PetscSFLinkPackLeafData_Private() argument
947 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, scope, &count, &start, &opt, &l… in PetscSFLinkPackLeafData_Private()
955 PetscErrorCode PetscSFLinkPackRootData(PetscSF sf, PetscSFLink link, PetscSFScope scope, const void… in PetscSFLinkPackRootData() argument
957 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkPackRootData()
961 …if (PetscMemTypeDevice(link->rootmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkPackRootData()
962 …if (link->PrePack) PetscCall((*link->PrePack)(sf, link, PETSCSF_ROOT2LEAF)); /* Used by SF nvshmem… in PetscSFLinkPackRootData()
964 PetscCall(PetscLogEventBegin(PETSCSF_Pack, sf, 0, 0, 0)); in PetscSFLinkPackRootData()
965 if (bas->rootbuflen[scope]) PetscCall(PetscSFLinkPackRootData_Private(sf, link, scope, rootdata)); in PetscSFLinkPackRootData()
966 PetscCall(PetscLogEventEnd(PETSCSF_Pack, sf, 0, 0, 0)); in PetscSFLinkPackRootData()
970 PetscErrorCode PetscSFLinkPackLeafData(PetscSF sf, PetscSFLink link, PetscSFScope scope, const void… in PetscSFLinkPackLeafData() argument
974 …if (PetscMemTypeDevice(link->leafmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkPackLeafData()
975 …if (link->PrePack) PetscCall((*link->PrePack)(sf, link, PETSCSF_LEAF2ROOT)); /* Used by SF nvshmem… in PetscSFLinkPackLeafData()
977 PetscCall(PetscLogEventBegin(PETSCSF_Pack, sf, 0, 0, 0)); in PetscSFLinkPackLeafData()
978 if (sf->leafbuflen[scope]) PetscCall(PetscSFLinkPackLeafData_Private(sf, link, scope, leafdata)); in PetscSFLinkPackLeafData()
979 PetscCall(PetscLogEventEnd(PETSCSF_Pack, sf, 0, 0, 0)); in PetscSFLinkPackLeafData()
983 static PetscErrorCode PetscSFLinkUnpackRootData_Private(PetscSF sf, PetscSFLink link, PetscSFScope … in PetscSFLinkUnpackRootData_Private() argument
987 … = (PetscSF_Basic *)sf->data; in PetscSFLinkUnpackRootData_Private()
996 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, scope, &count, &start, &opt, &r… in PetscSFLinkUnpackRootData_Private()
999 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, scope, &count, &start,… in PetscSFLinkUnpackRootData_Private()
1000 …PetscCall(PetscSFLinkUnpackDataWithMPIReduceLocal(sf, link, count, start, rootindices, rootdata, l… in PetscSFLinkUnpackRootData_Private()
1003 PetscCall(PetscSFLinkLogFlopsAfterUnpackRootData(sf, link, scope, op)); in PetscSFLinkUnpackRootData_Private()
1007 static PetscErrorCode PetscSFLinkUnpackLeafData_Private(PetscSF sf, PetscSFLink link, PetscSFScope … in PetscSFLinkUnpackLeafData_Private() argument
1017 PetscCall(PetscSFLinkGetUnpackAndOp(link, leafmtype, op, sf->leafdups[scope], &UnpackAndOp)); in PetscSFLinkUnpackLeafData_Private()
1019 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, scope, &count, &start, &opt, &l… in PetscSFLinkUnpackLeafData_Private()
1022 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, scope, &count, &start,… in PetscSFLinkUnpackLeafData_Private()
1023 …PetscCall(PetscSFLinkUnpackDataWithMPIReduceLocal(sf, link, count, start, leafindices, leafdata, l… in PetscSFLinkUnpackLeafData_Private()
1026 PetscCall(PetscSFLinkLogFlopsAfterUnpackLeafData(sf, link, scope, op)); in PetscSFLinkUnpackLeafData_Private()
1030 PetscErrorCode PetscSFLinkUnpackRootData(PetscSF sf, PetscSFLink link, PetscSFScope scope, void *ro… in PetscSFLinkUnpackRootData() argument
1032 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkUnpackRootData()
1035 …PetscCall(PetscLogEventBegin(PETSCSF_Unpack, sf, 0, 0, 0)); // call it even no data is unpacked so… in PetscSFLinkUnpackRootData()
1036 …] && !link->rootdirect[scope]) PetscCall(PetscSFLinkUnpackRootData_Private(sf, link, scope, rootda… in PetscSFLinkUnpackRootData()
1037 PetscCall(PetscLogEventEnd(PETSCSF_Unpack, sf, 0, 0, 0)); in PetscSFLinkUnpackRootData()
1039 …if (link->PostUnpack) PetscCall((*link->PostUnpack)(sf, link, PETSCSF_LEAF2ROOT)); /* Used by SF n… in PetscSFLinkUnpackRootData()
1040 …if (PetscMemTypeDevice(link->rootmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkUnpackRootData()
1046 PetscErrorCode PetscSFLinkUnpackLeafData(PetscSF sf, PetscSFLink link, PetscSFScope scope, void *le… in PetscSFLinkUnpackLeafData() argument
1049 PetscCall(PetscLogEventBegin(PETSCSF_Unpack, sf, 0, 0, 0)); in PetscSFLinkUnpackLeafData()
1050 …if (sf->leafbuflen[scope] && !link->leafdirect[scope]) PetscCall(PetscSFLinkUnpackLeafData_Private… in PetscSFLinkUnpackLeafData()
1051 PetscCall(PetscLogEventEnd(PETSCSF_Unpack, sf, 0, 0, 0)); in PetscSFLinkUnpackLeafData()
1053 …if (link->PostUnpack) PetscCall((*link->PostUnpack)(sf, link, PETSCSF_ROOT2LEAF)); /* Used by SF n… in PetscSFLinkUnpackLeafData()
1054 …if (PetscMemTypeDevice(link->leafmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkUnpackLeafData()
1060 PetscErrorCode PetscSFLinkFetchAndOpRemote(PetscSF sf, PetscSFLink link, void *rootdata, MPI_Op op) in PetscSFLinkFetchAndOpRemote() argument
1064 … = (PetscSF_Basic *)sf->data; in PetscSFLinkFetchAndOpRemote()
1070 PetscCall(PetscLogEventBegin(PETSCSF_Unpack, sf, 0, 0, 0)); in PetscSFLinkFetchAndOpRemote()
1074 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, PETSCSF_REMOTE, &count, &start,… in PetscSFLinkFetchAndOpRemote()
1077 PetscCall(PetscSFLinkLogFlopsAfterUnpackRootData(sf, link, PETSCSF_REMOTE, op)); in PetscSFLinkFetchAndOpRemote()
1078 PetscCall(PetscLogEventEnd(PETSCSF_Unpack, sf, 0, 0, 0)); in PetscSFLinkFetchAndOpRemote()
1082 PetscErrorCode PetscSFLinkScatterLocal(PetscSF sf, PetscSFLink link, PetscSFDirection direction, vo… in PetscSFLinkScatterLocal() argument
1086 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkScatterLocal()
1089 PetscInt buflen = sf->leafbuflen[PETSCSF_LOCAL]; in PetscSFLinkScatterLocal()
1098 PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_LOCAL, rootdata)); in PetscSFLinkScatterLocal()
1104 PetscCall(PetscSFLinkPackLeafData(sf, link, PETSCSF_LOCAL, leafdata)); in PetscSFLinkScatterLocal()
1114 PetscCall(PetscSFLinkUnpackLeafData(sf, link, PETSCSF_LOCAL, leafdata, op)); in PetscSFLinkScatterLocal()
1116 PetscCall(PetscSFLinkUnpackRootData(sf, link, PETSCSF_LOCAL, rootdata, op)); in PetscSFLinkScatterLocal()
1119 …dstdups = (direction == PETSCSF_ROOT2LEAF) ? sf->leafdups[PETSCSF_LOCAL] : bas->rootdups[PETSCSF_… in PetscSFLinkScatterLocal()
1123 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, PETSCSF_LOCAL, &count, &rootsta… in PetscSFLinkScatterLocal()
1124 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, PETSCSF_LOCAL, &count, &leafsta… in PetscSFLinkScatterLocal()
1131 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, PETSCSF_LOCAL, &count,… in PetscSFLinkScatterLocal()
1132 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, PETSCSF_LOCAL, &count,… in PetscSFLinkScatterLocal()
1134 …PetscCall(PetscSFLinkScatterDataWithMPIReduceLocal(sf, link, count, rootstart, rootindices, rootda… in PetscSFLinkScatterLocal()
1136 …PetscCall(PetscSFLinkScatterDataWithMPIReduceLocal(sf, link, count, leafstart, leafindices, leafda… in PetscSFLinkScatterLocal()
1144 PetscErrorCode PetscSFLinkFetchAndOpLocal(PetscSF sf, PetscSFLink link, void *rootdata, const void … in PetscSFLinkFetchAndOpLocal() argument
1148 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFLinkFetchAndOpLocal()
1159 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, PETSCSF_LOCAL, &count, &rootsta… in PetscSFLinkFetchAndOpLocal()
1160 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, PETSCSF_LOCAL, &count, &leafsta… in PetscSFLinkFetchAndOpLocal()
1267 static inline PetscErrorCode PetscSFDestroyPackOpt(PetscSF sf, PetscMemType mtype, PetscSFPackOpt *… in PetscSFDestroyPackOpt() argument
1273 PetscCall(PetscSFFree(sf, mtype, opt->array)); in PetscSFDestroyPackOpt()
1280 PetscErrorCode PetscSFSetUpPackFields(PetscSF sf) in PetscSFSetUpPackFields() argument
1282 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFSetUpPackFields()
1288 sf->leafstart[i] = 0; in PetscSFSetUpPackFields()
1289 sf->leafcontig[i] = PETSC_TRUE; in PetscSFSetUpPackFields()
1290 sf->leafdups[i] = PETSC_FALSE; in PetscSFSetUpPackFields()
1296 sf->leafbuflen[0] = sf->roffset[sf->ndranks]; in PetscSFSetUpPackFields()
1297 sf->leafbuflen[1] = sf->roffset[sf->nranks] - sf->roffset[sf->ndranks]; in PetscSFSetUpPackFields()
1299 if (sf->leafbuflen[0]) sf->leafstart[0] = sf->rmine[0]; in PetscSFSetUpPackFields()
1300 if (sf->leafbuflen[1]) sf->leafstart[1] = sf->rmine[sf->roffset[sf->ndranks]]; in PetscSFSetUpPackFields()
1303 for (i = 0; i < sf->roffset[sf->ndranks]; i++) { /* self */ in PetscSFSetUpPackFields()
1304 if (sf->rmine[i] != sf->leafstart[0] + i) { in PetscSFSetUpPackFields()
1305 sf->leafcontig[0] = PETSC_FALSE; in PetscSFSetUpPackFields()
1309 for (i = sf->roffset[sf->ndranks], j = 0; i < sf->roffset[sf->nranks]; i++, j++) { /* remote */ in PetscSFSetUpPackFields()
1310 if (sf->rmine[i] != sf->leafstart[1] + j) { in PetscSFSetUpPackFields()
1311 sf->leafcontig[1] = PETSC_FALSE; in PetscSFSetUpPackFields()
1317 …if (!sf->leafcontig[0]) PetscCall(PetscSFCreatePackOpt(sf->ndranks, sf->roffset, sf->rmine, &sf->l… in PetscSFSetUpPackFields()
1318 …if (!sf->leafcontig[1]) PetscCall(PetscSFCreatePackOpt(sf->nranks - sf->ndranks, sf->roffset + sf-… in PetscSFSetUpPackFields()
1345 PetscBool ismulti = (sf->multi == sf) ? PETSC_TRUE : PETSC_FALSE; in PetscSFSetUpPackFields()
1346 …if (!sf->leafcontig[0] && !ismulti) PetscCall(PetscCheckDupsInt(sf->leafbuflen[0], sf->rmine, &sf-… in PetscSFSetUpPackFields()
1347 …if (!sf->leafcontig[1] && !ismulti) PetscCall(PetscCheckDupsInt(sf->leafbuflen[1], sf->rmine + sf-… in PetscSFSetUpPackFields()
1354 PetscErrorCode PetscSFResetPackFields(PetscSF sf) in PetscSFResetPackFields() argument
1356 PetscSF_Basic *bas = (PetscSF_Basic *)sf->data; in PetscSFResetPackFields()
1361 PetscCall(PetscSFDestroyPackOpt(sf, PETSC_MEMTYPE_HOST, &sf->leafpackopt[i])); in PetscSFResetPackFields()
1362 PetscCall(PetscSFDestroyPackOpt(sf, PETSC_MEMTYPE_HOST, &bas->rootpackopt[i])); in PetscSFResetPackFields()
1364 PetscCall(PetscSFDestroyPackOpt(sf, PETSC_MEMTYPE_DEVICE, &sf->leafpackopt_d[i])); in PetscSFResetPackFields()
1365 PetscCall(PetscSFDestroyPackOpt(sf, PETSC_MEMTYPE_DEVICE, &bas->rootpackopt_d[i])); in PetscSFResetPackFields()