Lines Matching refs:link
55 …static PetscErrorCode CPPJoin4(Pack, Type, BS, EQ)(PetscSFLink link, PetscInt count, PetscInt star…
59 PetscInt i, j, k, X, Y, r, bs = link->bs; \
96 …static PetscErrorCode CPPJoin4(UnpackAndInsert, Type, BS, EQ)(PetscSFLink link, PetscInt count, Pe…
100 PetscInt i, j, k, X, Y, r, bs = link->bs; \
137 …static PetscErrorCode CPPJoin4(UnpackAnd##Opname, Type, BS, EQ)(PetscSFLink link, PetscInt count, …
141 PetscInt i, j, k, X, Y, r, bs = link->bs; \
170 …static PetscErrorCode CPPJoin4(FetchAnd##Opname, Type, BS, EQ)(PetscSFLink link, PetscInt count, P…
173 PetscInt i, j, k, r, l, bs = link->bs; \
191 …static PetscErrorCode CPPJoin4(ScatterAnd##Opname, Type, BS, EQ)(PetscSFLink link, PetscInt count,…
195 PetscInt i, j, k, s, t, X, Y, bs = link->bs; \
201 …PetscCall(CPPJoin4(UnpackAnd##Opname, Type, BS, EQ)(link, count, dstStart, dstOpt, dstIdx, dst, u)…
224 …static PetscErrorCode CPPJoin4(FetchAnd##Opname##Local, Type, BS, EQ)(PetscSFLink link, PetscInt c…
228 PetscInt i, j, k, r, l, bs = link->bs; \
246 …BS, EQ, Insert, =, OP_ASSIGN) static void CPPJoin4(PackInit_Pack, Type, BS, EQ)(PetscSFLink link) \
248 link->h_Pack = CPPJoin4(Pack, Type, BS, EQ); \
249 link->h_UnpackAndInsert = CPPJoin4(UnpackAndInsert, Type, BS, EQ); \
250 link->h_ScatterAndInsert = CPPJoin4(ScatterAndInsert, Type, BS, EQ); \
255 …pe, BS, EQ, Add, +, OP_BINARY) static void CPPJoin4(PackInit_Add, Type, BS, EQ)(PetscSFLink link) \
257 link->h_UnpackAndAdd = CPPJoin4(UnpackAndAdd, Type, BS, EQ); \
258 link->h_UnpackAndMult = CPPJoin4(UnpackAndMult, Type, BS, EQ); \
259 link->h_FetchAndAdd = CPPJoin4(FetchAndAdd, Type, BS, EQ); \
260 link->h_ScatterAndAdd = CPPJoin4(ScatterAndAdd, Type, BS, EQ); \
261 link->h_ScatterAndMult = CPPJoin4(ScatterAndMult, Type, BS, EQ); \
262 link->h_FetchAndAddLocal = CPPJoin4(FetchAndAddLocal, Type, BS, EQ); \
267 …in, PetscMin, OP_FUNCTION) static void CPPJoin4(PackInit_Compare, Type, BS, EQ)(PetscSFLink link) \
269 link->h_UnpackAndMax = CPPJoin4(UnpackAndMax, Type, BS, EQ); \
270 link->h_UnpackAndMin = CPPJoin4(UnpackAndMin, Type, BS, EQ); \
271 link->h_ScatterAndMax = CPPJoin4(ScatterAndMax, Type, BS, EQ); \
272 link->h_ScatterAndMin = CPPJoin4(ScatterAndMin, Type, BS, EQ); \
280 …BS, EQ, LXOR, ||, OP_LXOR) static void CPPJoin4(PackInit_Logical, Type, BS, EQ)(PetscSFLink link) \
282 link->h_UnpackAndLAND = CPPJoin4(UnpackAndLAND, Type, BS, EQ); \
283 link->h_UnpackAndLOR = CPPJoin4(UnpackAndLOR, Type, BS, EQ); \
284 link->h_UnpackAndLXOR = CPPJoin4(UnpackAndLXOR, Type, BS, EQ); \
285 link->h_ScatterAndLAND = CPPJoin4(ScatterAndLAND, Type, BS, EQ); \
286 link->h_ScatterAndLOR = CPPJoin4(ScatterAndLOR, Type, BS, EQ); \
287 link->h_ScatterAndLXOR = CPPJoin4(ScatterAndLXOR, Type, BS, EQ); \
292 …S, EQ, BXOR, ^, OP_BINARY) static void CPPJoin4(PackInit_Bitwise, Type, BS, EQ)(PetscSFLink link) \
294 link->h_UnpackAndBAND = CPPJoin4(UnpackAndBAND, Type, BS, EQ); \
295 link->h_UnpackAndBOR = CPPJoin4(UnpackAndBOR, Type, BS, EQ); \
296 link->h_UnpackAndBXOR = CPPJoin4(UnpackAndBXOR, Type, BS, EQ); \
297 link->h_ScatterAndBAND = CPPJoin4(ScatterAndBAND, Type, BS, EQ); \
298 link->h_ScatterAndBOR = CPPJoin4(ScatterAndBOR, Type, BS, EQ); \
299 link->h_ScatterAndBXOR = CPPJoin4(ScatterAndBXOR, Type, BS, EQ); \
304 …ype, BS, EQ, Min, <, OP_XLOC) static void CPPJoin4(PackInit_Xloc, Type, BS, EQ)(PetscSFLink link) \
306 link->h_UnpackAndMaxloc = CPPJoin4(UnpackAndMax, Type, BS, EQ); \
307 link->h_UnpackAndMinloc = CPPJoin4(UnpackAndMin, Type, BS, EQ); \
308 link->h_ScatterAndMaxloc = CPPJoin4(ScatterAndMax, Type, BS, EQ); \
309 link->h_ScatterAndMinloc = CPPJoin4(ScatterAndMin, Type, BS, EQ); \
313 … DEF_Bit(Type, BS, EQ) static void CPPJoin4(PackInit_IntegerType, Type, BS, EQ)(PetscSFLink link) \
315 CPPJoin4(PackInit_Pack, Type, BS, EQ)(link); \
316 CPPJoin4(PackInit_Add, Type, BS, EQ)(link); \
317 CPPJoin4(PackInit_Compare, Type, BS, EQ)(link); \
318 CPPJoin4(PackInit_Logical, Type, BS, EQ)(link); \
319 CPPJoin4(PackInit_Bitwise, Type, BS, EQ)(link); \
323 …EQ) DEF_Cmp(Type, BS, EQ) static void CPPJoin4(PackInit_RealType, Type, BS, EQ)(PetscSFLink link) \
325 CPPJoin4(PackInit_Pack, Type, BS, EQ)(link); \
326 CPPJoin4(PackInit_Add, Type, BS, EQ)(link); \
327 CPPJoin4(PackInit_Compare, Type, BS, EQ)(link); \
332 … DEF_Add(Type, BS, EQ) static void CPPJoin4(PackInit_ComplexType, Type, BS, EQ)(PetscSFLink link) \
334 CPPJoin4(PackInit_Pack, Type, BS, EQ)(link); \
335 CPPJoin4(PackInit_Add, Type, BS, EQ)(link); \
340 DEF_Pack(Type, BS, EQ) static void CPPJoin4(PackInit_DumbType, Type, BS, EQ)(PetscSFLink link) \
342 CPPJoin4(PackInit_Pack, Type, BS, EQ)(link); \
347 …Q) DEF_Xloc(Type, BS, EQ) static void CPPJoin4(PackInit_PairType, Type, BS, EQ)(PetscSFLink link) \
349 CPPJoin4(PackInit_Pack, Type, BS, EQ)(link); \
350 CPPJoin4(PackInit_Xloc, Type, BS, EQ)(link); \
395 PetscErrorCode PetscSFLinkDestroy(PetscSF sf, PetscSFLink link) in PetscSFLinkDestroy() argument
402 if (link->deviceinited) PetscCall((*link->Destroy)(sf, link)); in PetscSFLinkDestroy()
405 if (!link->isbuiltin) PetscCallMPI(MPI_Type_free(&link->unit)); in PetscSFLinkDestroy()
406 if (!link->use_nvshmem) { in PetscSFLinkDestroy()
408 if (link->reqs[i] != MPI_REQUEST_NULL) PetscCallMPI(MPI_Request_free(&link->reqs[i])); in PetscSFLinkDestroy()
410 PetscCall(PetscFree(link->reqs)); in PetscSFLinkDestroy()
412 PetscCall(PetscFree(link->rootbuf_alloc[i][PETSC_MEMTYPE_HOST])); in PetscSFLinkDestroy()
413 PetscCall(PetscFree(link->leafbuf_alloc[i][PETSC_MEMTYPE_HOST])); in PetscSFLinkDestroy()
416 PetscCall(PetscFree(link)); in PetscSFLinkDestroy()
440 PetscSFLink link, *p; in PetscSFLinkGetInUse() local
445 for (p = &bas->inuse; (link = *p); p = &link->next) { in PetscSFLinkGetInUse()
447 PetscCall(MPIPetsc_Type_compare(unit, link->unit, &match)); in PetscSFLinkGetInUse()
448 if (match && (rootdata == link->rootdata) && (leafdata == link->leafdata)) { in PetscSFLinkGetInUse()
451 *p = link->next; in PetscSFLinkGetInUse()
458 *mylink = link; in PetscSFLinkGetInUse()
468 PetscSFLink link = *mylink; in PetscSFLinkReclaim() local
471 link->rootdata = NULL; in PetscSFLinkReclaim()
472 link->leafdata = NULL; in PetscSFLinkReclaim()
473 link->next = bas->avail; in PetscSFLinkReclaim()
474 bas->avail = link; in PetscSFLinkReclaim()
482 PetscSFLink link, *p; in PetscSFSetErrorOnUnsupportedOverlap() local
492 for (p = &bas->inuse; (link = *p); p = &link->next) { in PetscSFSetErrorOnUnsupportedOverlap()
493 PetscCall(MPIPetsc_Type_compare(unit, link->unit, &match)); in PetscSFSetErrorOnUnsupportedOverlap()
494 …PetscCheck(!match || rootdata != link->rootdata || leafdata != link->leafdata, PETSC_COMM_SELF, PE… in PetscSFSetErrorOnUnsupportedOverlap()
501 static PetscErrorCode PetscSFLinkMemcpy_Host(PetscSFLink link, PetscMemType dstmtype, void *dst, Pe… in PetscSFLinkMemcpy_Host() argument
508 PetscErrorCode PetscSFLinkSetUp_Host(PetscSF sf, PetscSFLink link, MPI_Datatype unit) in PetscSFLinkSetUp_Host() argument
532 …link->isbuiltin = (combiner == MPI_COMBINER_NAMED) ? PETSC_TRUE : PETSC_FALSE; /* unit is MPI buil… in PetscSFLinkSetUp_Host()
533 link->bs = 1; /* default */ in PetscSFLinkSetUp_Host()
536 PackInit_PairType_int_int_1_1(link); in PetscSFLinkSetUp_Host()
537 link->bs = 1; in PetscSFLinkSetUp_Host()
538 link->unitbytes = 2 * sizeof(int); in PetscSFLinkSetUp_Host()
539 link->isbuiltin = PETSC_TRUE; /* unit is PETSc builtin */ in PetscSFLinkSetUp_Host()
540 link->basicunit = MPI_2INT; in PetscSFLinkSetUp_Host()
541 link->unit = MPI_2INT; in PetscSFLinkSetUp_Host()
543 PackInit_PairType_PetscInt_PetscInt_1_1(link); in PetscSFLinkSetUp_Host()
544 link->bs = 1; in PetscSFLinkSetUp_Host()
545 link->unitbytes = 2 * sizeof(PetscInt); in PetscSFLinkSetUp_Host()
546 link->basicunit = MPIU_2INT; in PetscSFLinkSetUp_Host()
547 link->isbuiltin = PETSC_TRUE; /* unit is PETSc builtin */ in PetscSFLinkSetUp_Host()
548 link->unit = MPIU_2INT; in PetscSFLinkSetUp_Host()
550 if (nPetscReal == 8) PackInit_RealType_PetscReal_8_1(link); in PetscSFLinkSetUp_Host()
551 else if (nPetscReal % 8 == 0) PackInit_RealType_PetscReal_8_0(link); in PetscSFLinkSetUp_Host()
552 else if (nPetscReal == 4) PackInit_RealType_PetscReal_4_1(link); in PetscSFLinkSetUp_Host()
553 else if (nPetscReal % 4 == 0) PackInit_RealType_PetscReal_4_0(link); in PetscSFLinkSetUp_Host()
554 else if (nPetscReal == 2) PackInit_RealType_PetscReal_2_1(link); in PetscSFLinkSetUp_Host()
555 else if (nPetscReal % 2 == 0) PackInit_RealType_PetscReal_2_0(link); in PetscSFLinkSetUp_Host()
556 else if (nPetscReal == 1) PackInit_RealType_PetscReal_1_1(link); in PetscSFLinkSetUp_Host()
557 else if (nPetscReal % 1 == 0) PackInit_RealType_PetscReal_1_0(link); in PetscSFLinkSetUp_Host()
558 link->bs = nPetscReal; in PetscSFLinkSetUp_Host()
559 link->unitbytes = nPetscReal * sizeof(PetscReal); in PetscSFLinkSetUp_Host()
560 link->basicunit = MPIU_REAL; in PetscSFLinkSetUp_Host()
561 if (link->bs == 1) { in PetscSFLinkSetUp_Host()
562 link->isbuiltin = PETSC_TRUE; in PetscSFLinkSetUp_Host()
563 link->unit = MPIU_REAL; in PetscSFLinkSetUp_Host()
566 if (nPetscInt == 8) PackInit_IntegerType_PetscInt_8_1(link); in PetscSFLinkSetUp_Host()
567 else if (nPetscInt % 8 == 0) PackInit_IntegerType_PetscInt_8_0(link); in PetscSFLinkSetUp_Host()
568 else if (nPetscInt == 4) PackInit_IntegerType_PetscInt_4_1(link); in PetscSFLinkSetUp_Host()
569 else if (nPetscInt % 4 == 0) PackInit_IntegerType_PetscInt_4_0(link); in PetscSFLinkSetUp_Host()
570 else if (nPetscInt == 2) PackInit_IntegerType_PetscInt_2_1(link); in PetscSFLinkSetUp_Host()
571 else if (nPetscInt % 2 == 0) PackInit_IntegerType_PetscInt_2_0(link); in PetscSFLinkSetUp_Host()
572 else if (nPetscInt == 1) PackInit_IntegerType_PetscInt_1_1(link); in PetscSFLinkSetUp_Host()
573 else if (nPetscInt % 1 == 0) PackInit_IntegerType_PetscInt_1_0(link); in PetscSFLinkSetUp_Host()
574 link->bs = nPetscInt; in PetscSFLinkSetUp_Host()
575 link->unitbytes = nPetscInt * sizeof(PetscInt); in PetscSFLinkSetUp_Host()
576 link->basicunit = MPIU_INT; in PetscSFLinkSetUp_Host()
577 if (link->bs == 1) { in PetscSFLinkSetUp_Host()
578 link->isbuiltin = PETSC_TRUE; in PetscSFLinkSetUp_Host()
579 link->unit = MPIU_INT; in PetscSFLinkSetUp_Host()
583 if (nInt == 8) PackInit_IntegerType_int_8_1(link); in PetscSFLinkSetUp_Host()
584 else if (nInt % 8 == 0) PackInit_IntegerType_int_8_0(link); in PetscSFLinkSetUp_Host()
585 else if (nInt == 4) PackInit_IntegerType_int_4_1(link); in PetscSFLinkSetUp_Host()
586 else if (nInt % 4 == 0) PackInit_IntegerType_int_4_0(link); in PetscSFLinkSetUp_Host()
587 else if (nInt == 2) PackInit_IntegerType_int_2_1(link); in PetscSFLinkSetUp_Host()
588 else if (nInt % 2 == 0) PackInit_IntegerType_int_2_0(link); in PetscSFLinkSetUp_Host()
589 else if (nInt == 1) PackInit_IntegerType_int_1_1(link); in PetscSFLinkSetUp_Host()
590 else if (nInt % 1 == 0) PackInit_IntegerType_int_1_0(link); in PetscSFLinkSetUp_Host()
591 link->bs = nInt; in PetscSFLinkSetUp_Host()
592 link->unitbytes = nInt * sizeof(int); in PetscSFLinkSetUp_Host()
593 link->basicunit = MPI_INT; in PetscSFLinkSetUp_Host()
594 if (link->bs == 1) { in PetscSFLinkSetUp_Host()
595 link->isbuiltin = PETSC_TRUE; in PetscSFLinkSetUp_Host()
596 link->unit = MPI_INT; in PetscSFLinkSetUp_Host()
600 if (nSignedChar == 8) PackInit_IntegerType_SignedChar_8_1(link); in PetscSFLinkSetUp_Host()
601 else if (nSignedChar % 8 == 0) PackInit_IntegerType_SignedChar_8_0(link); in PetscSFLinkSetUp_Host()
602 else if (nSignedChar == 4) PackInit_IntegerType_SignedChar_4_1(link); in PetscSFLinkSetUp_Host()
603 else if (nSignedChar % 4 == 0) PackInit_IntegerType_SignedChar_4_0(link); in PetscSFLinkSetUp_Host()
604 else if (nSignedChar == 2) PackInit_IntegerType_SignedChar_2_1(link); in PetscSFLinkSetUp_Host()
605 else if (nSignedChar % 2 == 0) PackInit_IntegerType_SignedChar_2_0(link); in PetscSFLinkSetUp_Host()
606 else if (nSignedChar == 1) PackInit_IntegerType_SignedChar_1_1(link); in PetscSFLinkSetUp_Host()
607 else if (nSignedChar % 1 == 0) PackInit_IntegerType_SignedChar_1_0(link); in PetscSFLinkSetUp_Host()
608 link->bs = nSignedChar; in PetscSFLinkSetUp_Host()
609 link->unitbytes = nSignedChar * sizeof(SignedChar); in PetscSFLinkSetUp_Host()
610 link->basicunit = MPI_SIGNED_CHAR; in PetscSFLinkSetUp_Host()
611 if (link->bs == 1) { in PetscSFLinkSetUp_Host()
612 link->isbuiltin = PETSC_TRUE; in PetscSFLinkSetUp_Host()
613 link->unit = MPI_SIGNED_CHAR; in PetscSFLinkSetUp_Host()
616 if (nUnsignedChar == 8) PackInit_IntegerType_UnsignedChar_8_1(link); in PetscSFLinkSetUp_Host()
617 else if (nUnsignedChar % 8 == 0) PackInit_IntegerType_UnsignedChar_8_0(link); in PetscSFLinkSetUp_Host()
618 else if (nUnsignedChar == 4) PackInit_IntegerType_UnsignedChar_4_1(link); in PetscSFLinkSetUp_Host()
619 else if (nUnsignedChar % 4 == 0) PackInit_IntegerType_UnsignedChar_4_0(link); in PetscSFLinkSetUp_Host()
620 else if (nUnsignedChar == 2) PackInit_IntegerType_UnsignedChar_2_1(link); in PetscSFLinkSetUp_Host()
621 else if (nUnsignedChar % 2 == 0) PackInit_IntegerType_UnsignedChar_2_0(link); in PetscSFLinkSetUp_Host()
622 else if (nUnsignedChar == 1) PackInit_IntegerType_UnsignedChar_1_1(link); in PetscSFLinkSetUp_Host()
623 else if (nUnsignedChar % 1 == 0) PackInit_IntegerType_UnsignedChar_1_0(link); in PetscSFLinkSetUp_Host()
624 link->bs = nUnsignedChar; in PetscSFLinkSetUp_Host()
625 link->unitbytes = nUnsignedChar * sizeof(UnsignedChar); in PetscSFLinkSetUp_Host()
626 link->basicunit = MPI_UNSIGNED_CHAR; in PetscSFLinkSetUp_Host()
627 if (link->bs == 1) { in PetscSFLinkSetUp_Host()
628 link->isbuiltin = PETSC_TRUE; in PetscSFLinkSetUp_Host()
629 link->unit = MPI_UNSIGNED_CHAR; in PetscSFLinkSetUp_Host()
633 if (nPetscComplex == 8) PackInit_ComplexType_PetscComplex_8_1(link); in PetscSFLinkSetUp_Host()
634 else if (nPetscComplex % 8 == 0) PackInit_ComplexType_PetscComplex_8_0(link); in PetscSFLinkSetUp_Host()
635 else if (nPetscComplex == 4) PackInit_ComplexType_PetscComplex_4_1(link); in PetscSFLinkSetUp_Host()
636 else if (nPetscComplex % 4 == 0) PackInit_ComplexType_PetscComplex_4_0(link); in PetscSFLinkSetUp_Host()
637 else if (nPetscComplex == 2) PackInit_ComplexType_PetscComplex_2_1(link); in PetscSFLinkSetUp_Host()
638 else if (nPetscComplex % 2 == 0) PackInit_ComplexType_PetscComplex_2_0(link); in PetscSFLinkSetUp_Host()
639 else if (nPetscComplex == 1) PackInit_ComplexType_PetscComplex_1_1(link); in PetscSFLinkSetUp_Host()
640 else if (nPetscComplex % 1 == 0) PackInit_ComplexType_PetscComplex_1_0(link); in PetscSFLinkSetUp_Host()
641 link->bs = nPetscComplex; in PetscSFLinkSetUp_Host()
642 link->unitbytes = nPetscComplex * sizeof(PetscComplex); in PetscSFLinkSetUp_Host()
643 link->basicunit = MPIU_COMPLEX; in PetscSFLinkSetUp_Host()
644 if (link->bs == 1) { in PetscSFLinkSetUp_Host()
645 link->isbuiltin = PETSC_TRUE; in PetscSFLinkSetUp_Host()
646 link->unit = MPIU_COMPLEX; in PetscSFLinkSetUp_Host()
655 if (nbyte == 4) PackInit_DumbType_char_4_1(link); in PetscSFLinkSetUp_Host()
656 else if (nbyte % 4 == 0) PackInit_DumbType_char_4_0(link); in PetscSFLinkSetUp_Host()
657 else if (nbyte == 2) PackInit_DumbType_char_2_1(link); in PetscSFLinkSetUp_Host()
658 else if (nbyte % 2 == 0) PackInit_DumbType_char_2_0(link); in PetscSFLinkSetUp_Host()
659 else if (nbyte == 1) PackInit_DumbType_char_1_1(link); in PetscSFLinkSetUp_Host()
660 else if (nbyte % 1 == 0) PackInit_DumbType_char_1_0(link); in PetscSFLinkSetUp_Host()
661 PetscCall(PetscIntCast(nbyte, &link->bs)); in PetscSFLinkSetUp_Host()
662 link->unitbytes = nbyte; in PetscSFLinkSetUp_Host()
663 link->basicunit = MPI_BYTE; in PetscSFLinkSetUp_Host()
666 if (nInt == 8) PackInit_DumbType_DumbInt_8_1(link); in PetscSFLinkSetUp_Host()
667 else if (nInt % 8 == 0) PackInit_DumbType_DumbInt_8_0(link); in PetscSFLinkSetUp_Host()
668 else if (nInt == 4) PackInit_DumbType_DumbInt_4_1(link); in PetscSFLinkSetUp_Host()
669 else if (nInt % 4 == 0) PackInit_DumbType_DumbInt_4_0(link); in PetscSFLinkSetUp_Host()
670 else if (nInt == 2) PackInit_DumbType_DumbInt_2_1(link); in PetscSFLinkSetUp_Host()
671 else if (nInt % 2 == 0) PackInit_DumbType_DumbInt_2_0(link); in PetscSFLinkSetUp_Host()
672 else if (nInt == 1) PackInit_DumbType_DumbInt_1_1(link); in PetscSFLinkSetUp_Host()
673 else if (nInt % 1 == 0) PackInit_DumbType_DumbInt_1_0(link); in PetscSFLinkSetUp_Host()
674 link->bs = nInt; in PetscSFLinkSetUp_Host()
675 link->unitbytes = nbyte; in PetscSFLinkSetUp_Host()
676 link->basicunit = MPI_INT; in PetscSFLinkSetUp_Host()
678 if (link->isbuiltin) link->unit = unit; in PetscSFLinkSetUp_Host()
681 if (!link->isbuiltin) PetscCallMPI(MPI_Type_dup(unit, &link->unit)); in PetscSFLinkSetUp_Host()
683 link->Memcpy = PetscSFLinkMemcpy_Host; in PetscSFLinkSetUp_Host()
687 PetscErrorCode PetscSFLinkGetUnpackAndOp(PetscSFLink link, PetscMemType mtype, MPI_Op op, PetscBool… in PetscSFLinkGetUnpackAndOp() argument
692 if (op == MPI_REPLACE) *UnpackAndOp = link->h_UnpackAndInsert; in PetscSFLinkGetUnpackAndOp()
693 else if (op == MPI_SUM || op == MPIU_SUM) *UnpackAndOp = link->h_UnpackAndAdd; in PetscSFLinkGetUnpackAndOp()
694 else if (op == MPI_PROD) *UnpackAndOp = link->h_UnpackAndMult; in PetscSFLinkGetUnpackAndOp()
695 else if (op == MPI_MAX || op == MPIU_MAX) *UnpackAndOp = link->h_UnpackAndMax; in PetscSFLinkGetUnpackAndOp()
696 else if (op == MPI_MIN || op == MPIU_MIN) *UnpackAndOp = link->h_UnpackAndMin; in PetscSFLinkGetUnpackAndOp()
697 else if (op == MPI_LAND) *UnpackAndOp = link->h_UnpackAndLAND; in PetscSFLinkGetUnpackAndOp()
698 else if (op == MPI_BAND) *UnpackAndOp = link->h_UnpackAndBAND; in PetscSFLinkGetUnpackAndOp()
699 else if (op == MPI_LOR) *UnpackAndOp = link->h_UnpackAndLOR; in PetscSFLinkGetUnpackAndOp()
700 else if (op == MPI_BOR) *UnpackAndOp = link->h_UnpackAndBOR; in PetscSFLinkGetUnpackAndOp()
701 else if (op == MPI_LXOR) *UnpackAndOp = link->h_UnpackAndLXOR; in PetscSFLinkGetUnpackAndOp()
702 else if (op == MPI_BXOR) *UnpackAndOp = link->h_UnpackAndBXOR; in PetscSFLinkGetUnpackAndOp()
703 else if (op == MPI_MAXLOC) *UnpackAndOp = link->h_UnpackAndMaxloc; in PetscSFLinkGetUnpackAndOp()
704 else if (op == MPI_MINLOC) *UnpackAndOp = link->h_UnpackAndMinloc; in PetscSFLinkGetUnpackAndOp()
708 if (op == MPI_REPLACE) *UnpackAndOp = link->d_UnpackAndInsert; in PetscSFLinkGetUnpackAndOp()
709 else if (op == MPI_SUM || op == MPIU_SUM) *UnpackAndOp = link->d_UnpackAndAdd; in PetscSFLinkGetUnpackAndOp()
710 else if (op == MPI_PROD) *UnpackAndOp = link->d_UnpackAndMult; in PetscSFLinkGetUnpackAndOp()
711 else if (op == MPI_MAX || op == MPIU_MAX) *UnpackAndOp = link->d_UnpackAndMax; in PetscSFLinkGetUnpackAndOp()
712 else if (op == MPI_MIN || op == MPIU_MIN) *UnpackAndOp = link->d_UnpackAndMin; in PetscSFLinkGetUnpackAndOp()
713 else if (op == MPI_LAND) *UnpackAndOp = link->d_UnpackAndLAND; in PetscSFLinkGetUnpackAndOp()
714 else if (op == MPI_BAND) *UnpackAndOp = link->d_UnpackAndBAND; in PetscSFLinkGetUnpackAndOp()
715 else if (op == MPI_LOR) *UnpackAndOp = link->d_UnpackAndLOR; in PetscSFLinkGetUnpackAndOp()
716 else if (op == MPI_BOR) *UnpackAndOp = link->d_UnpackAndBOR; in PetscSFLinkGetUnpackAndOp()
717 else if (op == MPI_LXOR) *UnpackAndOp = link->d_UnpackAndLXOR; in PetscSFLinkGetUnpackAndOp()
718 else if (op == MPI_BXOR) *UnpackAndOp = link->d_UnpackAndBXOR; in PetscSFLinkGetUnpackAndOp()
719 else if (op == MPI_MAXLOC) *UnpackAndOp = link->d_UnpackAndMaxloc; in PetscSFLinkGetUnpackAndOp()
720 else if (op == MPI_MINLOC) *UnpackAndOp = link->d_UnpackAndMinloc; in PetscSFLinkGetUnpackAndOp()
722 if (op == MPI_REPLACE) *UnpackAndOp = link->da_UnpackAndInsert; in PetscSFLinkGetUnpackAndOp()
723 else if (op == MPI_SUM || op == MPIU_SUM) *UnpackAndOp = link->da_UnpackAndAdd; in PetscSFLinkGetUnpackAndOp()
724 else if (op == MPI_PROD) *UnpackAndOp = link->da_UnpackAndMult; in PetscSFLinkGetUnpackAndOp()
725 else if (op == MPI_MAX || op == MPIU_MAX) *UnpackAndOp = link->da_UnpackAndMax; in PetscSFLinkGetUnpackAndOp()
726 else if (op == MPI_MIN || op == MPIU_MIN) *UnpackAndOp = link->da_UnpackAndMin; in PetscSFLinkGetUnpackAndOp()
727 else if (op == MPI_LAND) *UnpackAndOp = link->da_UnpackAndLAND; in PetscSFLinkGetUnpackAndOp()
728 else if (op == MPI_BAND) *UnpackAndOp = link->da_UnpackAndBAND; in PetscSFLinkGetUnpackAndOp()
729 else if (op == MPI_LOR) *UnpackAndOp = link->da_UnpackAndLOR; in PetscSFLinkGetUnpackAndOp()
730 else if (op == MPI_BOR) *UnpackAndOp = link->da_UnpackAndBOR; in PetscSFLinkGetUnpackAndOp()
731 else if (op == MPI_LXOR) *UnpackAndOp = link->da_UnpackAndLXOR; in PetscSFLinkGetUnpackAndOp()
732 else if (op == MPI_BXOR) *UnpackAndOp = link->da_UnpackAndBXOR; in PetscSFLinkGetUnpackAndOp()
733 else if (op == MPI_MAXLOC) *UnpackAndOp = link->da_UnpackAndMaxloc; in PetscSFLinkGetUnpackAndOp()
734 else if (op == MPI_MINLOC) *UnpackAndOp = link->da_UnpackAndMinloc; in PetscSFLinkGetUnpackAndOp()
740 PetscErrorCode PetscSFLinkGetScatterAndOp(PetscSFLink link, PetscMemType mtype, MPI_Op op, PetscBoo… in PetscSFLinkGetScatterAndOp() argument
745 if (op == MPI_REPLACE) *ScatterAndOp = link->h_ScatterAndInsert; in PetscSFLinkGetScatterAndOp()
746 else if (op == MPI_SUM || op == MPIU_SUM) *ScatterAndOp = link->h_ScatterAndAdd; in PetscSFLinkGetScatterAndOp()
747 else if (op == MPI_PROD) *ScatterAndOp = link->h_ScatterAndMult; in PetscSFLinkGetScatterAndOp()
748 else if (op == MPI_MAX || op == MPIU_MAX) *ScatterAndOp = link->h_ScatterAndMax; in PetscSFLinkGetScatterAndOp()
749 else if (op == MPI_MIN || op == MPIU_MIN) *ScatterAndOp = link->h_ScatterAndMin; in PetscSFLinkGetScatterAndOp()
750 else if (op == MPI_LAND) *ScatterAndOp = link->h_ScatterAndLAND; in PetscSFLinkGetScatterAndOp()
751 else if (op == MPI_BAND) *ScatterAndOp = link->h_ScatterAndBAND; in PetscSFLinkGetScatterAndOp()
752 else if (op == MPI_LOR) *ScatterAndOp = link->h_ScatterAndLOR; in PetscSFLinkGetScatterAndOp()
753 else if (op == MPI_BOR) *ScatterAndOp = link->h_ScatterAndBOR; in PetscSFLinkGetScatterAndOp()
754 else if (op == MPI_LXOR) *ScatterAndOp = link->h_ScatterAndLXOR; in PetscSFLinkGetScatterAndOp()
755 else if (op == MPI_BXOR) *ScatterAndOp = link->h_ScatterAndBXOR; in PetscSFLinkGetScatterAndOp()
756 else if (op == MPI_MAXLOC) *ScatterAndOp = link->h_ScatterAndMaxloc; in PetscSFLinkGetScatterAndOp()
757 else if (op == MPI_MINLOC) *ScatterAndOp = link->h_ScatterAndMinloc; in PetscSFLinkGetScatterAndOp()
761 if (op == MPI_REPLACE) *ScatterAndOp = link->d_ScatterAndInsert; in PetscSFLinkGetScatterAndOp()
762 else if (op == MPI_SUM || op == MPIU_SUM) *ScatterAndOp = link->d_ScatterAndAdd; in PetscSFLinkGetScatterAndOp()
763 else if (op == MPI_PROD) *ScatterAndOp = link->d_ScatterAndMult; in PetscSFLinkGetScatterAndOp()
764 else if (op == MPI_MAX || op == MPIU_MAX) *ScatterAndOp = link->d_ScatterAndMax; in PetscSFLinkGetScatterAndOp()
765 else if (op == MPI_MIN || op == MPIU_MIN) *ScatterAndOp = link->d_ScatterAndMin; in PetscSFLinkGetScatterAndOp()
766 else if (op == MPI_LAND) *ScatterAndOp = link->d_ScatterAndLAND; in PetscSFLinkGetScatterAndOp()
767 else if (op == MPI_BAND) *ScatterAndOp = link->d_ScatterAndBAND; in PetscSFLinkGetScatterAndOp()
768 else if (op == MPI_LOR) *ScatterAndOp = link->d_ScatterAndLOR; in PetscSFLinkGetScatterAndOp()
769 else if (op == MPI_BOR) *ScatterAndOp = link->d_ScatterAndBOR; in PetscSFLinkGetScatterAndOp()
770 else if (op == MPI_LXOR) *ScatterAndOp = link->d_ScatterAndLXOR; in PetscSFLinkGetScatterAndOp()
771 else if (op == MPI_BXOR) *ScatterAndOp = link->d_ScatterAndBXOR; in PetscSFLinkGetScatterAndOp()
772 else if (op == MPI_MAXLOC) *ScatterAndOp = link->d_ScatterAndMaxloc; in PetscSFLinkGetScatterAndOp()
773 else if (op == MPI_MINLOC) *ScatterAndOp = link->d_ScatterAndMinloc; in PetscSFLinkGetScatterAndOp()
775 if (op == MPI_REPLACE) *ScatterAndOp = link->da_ScatterAndInsert; in PetscSFLinkGetScatterAndOp()
776 else if (op == MPI_SUM || op == MPIU_SUM) *ScatterAndOp = link->da_ScatterAndAdd; in PetscSFLinkGetScatterAndOp()
777 else if (op == MPI_PROD) *ScatterAndOp = link->da_ScatterAndMult; in PetscSFLinkGetScatterAndOp()
778 else if (op == MPI_MAX || op == MPIU_MAX) *ScatterAndOp = link->da_ScatterAndMax; in PetscSFLinkGetScatterAndOp()
779 else if (op == MPI_MIN || op == MPIU_MIN) *ScatterAndOp = link->da_ScatterAndMin; in PetscSFLinkGetScatterAndOp()
780 else if (op == MPI_LAND) *ScatterAndOp = link->da_ScatterAndLAND; in PetscSFLinkGetScatterAndOp()
781 else if (op == MPI_BAND) *ScatterAndOp = link->da_ScatterAndBAND; in PetscSFLinkGetScatterAndOp()
782 else if (op == MPI_LOR) *ScatterAndOp = link->da_ScatterAndLOR; in PetscSFLinkGetScatterAndOp()
783 else if (op == MPI_BOR) *ScatterAndOp = link->da_ScatterAndBOR; in PetscSFLinkGetScatterAndOp()
784 else if (op == MPI_LXOR) *ScatterAndOp = link->da_ScatterAndLXOR; in PetscSFLinkGetScatterAndOp()
785 else if (op == MPI_BXOR) *ScatterAndOp = link->da_ScatterAndBXOR; in PetscSFLinkGetScatterAndOp()
786 else if (op == MPI_MAXLOC) *ScatterAndOp = link->da_ScatterAndMaxloc; in PetscSFLinkGetScatterAndOp()
787 else if (op == MPI_MINLOC) *ScatterAndOp = link->da_ScatterAndMinloc; in PetscSFLinkGetScatterAndOp()
793 PetscErrorCode PetscSFLinkGetFetchAndOp(PetscSFLink link, PetscMemType mtype, MPI_Op op, PetscBool … in PetscSFLinkGetFetchAndOp() argument
798 if (PetscMemTypeHost(mtype)) *FetchAndOp = link->h_FetchAndAdd; in PetscSFLinkGetFetchAndOp()
800 else if (PetscMemTypeDevice(mtype) && !atomic) *FetchAndOp = link->d_FetchAndAdd; in PetscSFLinkGetFetchAndOp()
801 else if (PetscMemTypeDevice(mtype) && atomic) *FetchAndOp = link->da_FetchAndAdd; in PetscSFLinkGetFetchAndOp()
806 PetscErrorCode PetscSFLinkGetFetchAndOpLocal(PetscSFLink link, PetscMemType mtype, MPI_Op op, Petsc… in PetscSFLinkGetFetchAndOpLocal() argument
811 if (PetscMemTypeHost(mtype)) *FetchAndOpLocal = link->h_FetchAndAddLocal; in PetscSFLinkGetFetchAndOpLocal()
813 else if (PetscMemTypeDevice(mtype) && !atomic) *FetchAndOpLocal = link->d_FetchAndAddLocal; in PetscSFLinkGetFetchAndOpLocal()
814 else if (PetscMemTypeDevice(mtype) && atomic) *FetchAndOpLocal = link->da_FetchAndAddLocal; in PetscSFLinkGetFetchAndOpLocal()
819 static inline PetscErrorCode PetscSFLinkLogFlopsAfterUnpackRootData(PetscSF sf, PetscSFLink link, P… in PetscSFLinkLogFlopsAfterUnpackRootData() argument
824 if (op != MPI_REPLACE && link->basicunit == MPIU_SCALAR) { /* op is a reduction on PetscScalars */ in PetscSFLinkLogFlopsAfterUnpackRootData()
826 …if (PetscMemTypeDevice(link->rootmtype)) PetscCall(PetscLogGpuFlops(bas->rootbuflen[scope] * link-… in PetscSFLinkLogFlopsAfterUnpackRootData()
829 …PetscCall(PetscLogFlops(bas->rootbuflen[scope] * link->bs)); /* # of roots in buffer x # of scalar… in PetscSFLinkLogFlopsAfterUnpackRootData()
834 static inline PetscErrorCode PetscSFLinkLogFlopsAfterUnpackLeafData(PetscSF sf, PetscSFLink link, P… in PetscSFLinkLogFlopsAfterUnpackLeafData() argument
837 if (op != MPI_REPLACE && link->basicunit == MPIU_SCALAR) { /* op is a reduction on PetscScalars */ in PetscSFLinkLogFlopsAfterUnpackLeafData()
839 …if (PetscMemTypeDevice(link->leafmtype)) PetscCall(PetscLogGpuFlops(sf->leafbuflen[scope] * link->… in PetscSFLinkLogFlopsAfterUnpackLeafData()
842 PetscCall(PetscLogFlops(sf->leafbuflen[scope] * link->bs)); in PetscSFLinkLogFlopsAfterUnpackLeafData()
860 static inline PetscErrorCode PetscSFLinkUnpackDataWithMPIReduceLocal(PetscSF sf, PetscSFLink link, … in PetscSFLinkUnpackDataWithMPIReduceLocal() argument
870 …(MPI_Reduce_local((const char *)buf + i * link->unitbytes, (char *)data + indices[i] * link->unitb… in PetscSFLinkUnpackDataWithMPIReduceLocal()
873 …PetscCallMPI(MPIU_Reduce_local(buf, (char *)data + start * link->unitbytes, count, link->unit, op)… in PetscSFLinkUnpackDataWithMPIReduceLocal()
882 static inline PetscErrorCode PetscSFLinkScatterDataWithMPIReduceLocal(PetscSF sf, PetscSFLink link,… in PetscSFLinkScatterDataWithMPIReduceLocal() argument
889 …scSFLinkUnpackDataWithMPIReduceLocal(sf, link, count, dstStart, dstIdx, dst, (const char *)src + s… in PetscSFLinkScatterDataWithMPIReduceLocal()
894 …Reduce_local((const char *)src + srcIdx[i] * link->unitbytes, (char *)dst + disp * link->unitbytes… in PetscSFLinkScatterDataWithMPIReduceLocal()
919 static PetscErrorCode PetscSFLinkPackRootData_Private(PetscSF sf, PetscSFLink link, PetscSFScope sc… in PetscSFLinkPackRootData_Private() argument
923 … = link->rootmtype; in PetscSFLinkPackRootData_Private()
928 if (!link->rootdirect[scope]) { /* If rootdata works directly as rootbuf, skip packing */ in PetscSFLinkPackRootData_Private()
929 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, scope, &count, &start, &opt, &r… in PetscSFLinkPackRootData_Private()
930 PetscCall(PetscSFLinkGetPack(link, rootmtype, &Pack)); in PetscSFLinkPackRootData_Private()
931 …PetscCall((*Pack)(link, count, start, opt, rootindices, rootdata, link->rootbuf[scope][rootmtype])… in PetscSFLinkPackRootData_Private()
937 static PetscErrorCode PetscSFLinkPackLeafData_Private(PetscSF sf, PetscSFLink link, PetscSFScope sc… in PetscSFLinkPackLeafData_Private() argument
941 … = link->leafmtype; in PetscSFLinkPackLeafData_Private()
946 if (!link->leafdirect[scope]) { /* If leafdata works directly as rootbuf, skip packing */ in PetscSFLinkPackLeafData_Private()
947 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, scope, &count, &start, &opt, &l… in PetscSFLinkPackLeafData_Private()
948 PetscCall(PetscSFLinkGetPack(link, leafmtype, &Pack)); in PetscSFLinkPackLeafData_Private()
949 …PetscCall((*Pack)(link, count, start, opt, leafindices, leafdata, link->leafbuf[scope][leafmtype])… in PetscSFLinkPackLeafData_Private()
955 PetscErrorCode PetscSFLinkPackRootData(PetscSF sf, PetscSFLink link, PetscSFScope scope, const void… in PetscSFLinkPackRootData() argument
961 …if (PetscMemTypeDevice(link->rootmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkPackRootData()
962 …if (link->PrePack) PetscCall((*link->PrePack)(sf, link, PETSCSF_ROOT2LEAF)); /* Used by SF nvshmem… in PetscSFLinkPackRootData()
965 if (bas->rootbuflen[scope]) PetscCall(PetscSFLinkPackRootData_Private(sf, link, scope, rootdata)); in PetscSFLinkPackRootData()
970 PetscErrorCode PetscSFLinkPackLeafData(PetscSF sf, PetscSFLink link, PetscSFScope scope, const void… in PetscSFLinkPackLeafData() argument
974 …if (PetscMemTypeDevice(link->leafmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkPackLeafData()
975 …if (link->PrePack) PetscCall((*link->PrePack)(sf, link, PETSCSF_LEAF2ROOT)); /* Used by SF nvshmem… in PetscSFLinkPackLeafData()
978 if (sf->leafbuflen[scope]) PetscCall(PetscSFLinkPackLeafData_Private(sf, link, scope, leafdata)); in PetscSFLinkPackLeafData()
983 static PetscErrorCode PetscSFLinkUnpackRootData_Private(PetscSF sf, PetscSFLink link, PetscSFScope … in PetscSFLinkUnpackRootData_Private() argument
988 … = link->rootmtype; in PetscSFLinkUnpackRootData_Private()
993 if (!link->rootdirect[scope]) { /* If rootdata works directly as rootbuf, skip unpacking */ in PetscSFLinkUnpackRootData_Private()
994 PetscCall(PetscSFLinkGetUnpackAndOp(link, rootmtype, op, bas->rootdups[scope], &UnpackAndOp)); in PetscSFLinkUnpackRootData_Private()
996 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, scope, &count, &start, &opt, &r… in PetscSFLinkUnpackRootData_Private()
997 …PetscCall((*UnpackAndOp)(link, count, start, opt, rootindices, rootdata, link->rootbuf[scope][root… in PetscSFLinkUnpackRootData_Private()
999 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, scope, &count, &start,… in PetscSFLinkUnpackRootData_Private()
1000 …etscCall(PetscSFLinkUnpackDataWithMPIReduceLocal(sf, link, count, start, rootindices, rootdata, li… in PetscSFLinkUnpackRootData_Private()
1003 PetscCall(PetscSFLinkLogFlopsAfterUnpackRootData(sf, link, scope, op)); in PetscSFLinkUnpackRootData_Private()
1007 static PetscErrorCode PetscSFLinkUnpackLeafData_Private(PetscSF sf, PetscSFLink link, PetscSFScope … in PetscSFLinkUnpackLeafData_Private() argument
1012 … = link->leafmtype; in PetscSFLinkUnpackLeafData_Private()
1016 if (!link->leafdirect[scope]) { /* If leafdata works directly as rootbuf, skip unpacking */ in PetscSFLinkUnpackLeafData_Private()
1017 PetscCall(PetscSFLinkGetUnpackAndOp(link, leafmtype, op, sf->leafdups[scope], &UnpackAndOp)); in PetscSFLinkUnpackLeafData_Private()
1019 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, scope, &count, &start, &opt, &l… in PetscSFLinkUnpackLeafData_Private()
1020 …PetscCall((*UnpackAndOp)(link, count, start, opt, leafindices, leafdata, link->leafbuf[scope][leaf… in PetscSFLinkUnpackLeafData_Private()
1022 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, scope, &count, &start,… in PetscSFLinkUnpackLeafData_Private()
1023 …etscCall(PetscSFLinkUnpackDataWithMPIReduceLocal(sf, link, count, start, leafindices, leafdata, li… in PetscSFLinkUnpackLeafData_Private()
1026 PetscCall(PetscSFLinkLogFlopsAfterUnpackLeafData(sf, link, scope, op)); in PetscSFLinkUnpackLeafData_Private()
1030 PetscErrorCode PetscSFLinkUnpackRootData(PetscSF sf, PetscSFLink link, PetscSFScope scope, void *ro… in PetscSFLinkUnpackRootData() argument
1036 …if (bas->rootbuflen[scope] && !link->rootdirect[scope]) PetscCall(PetscSFLinkUnpackRootData_Privat… in PetscSFLinkUnpackRootData()
1039 …if (link->PostUnpack) PetscCall((*link->PostUnpack)(sf, link, PETSCSF_LEAF2ROOT)); /* Used by SF n… in PetscSFLinkUnpackRootData()
1040 …if (PetscMemTypeDevice(link->rootmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkUnpackRootData()
1046 PetscErrorCode PetscSFLinkUnpackLeafData(PetscSF sf, PetscSFLink link, PetscSFScope scope, void *le… in PetscSFLinkUnpackLeafData() argument
1050 …if (sf->leafbuflen[scope] && !link->leafdirect[scope]) PetscCall(PetscSFLinkUnpackLeafData_Private… in PetscSFLinkUnpackLeafData()
1053 …if (link->PostUnpack) PetscCall((*link->PostUnpack)(sf, link, PETSCSF_ROOT2LEAF)); /* Used by SF n… in PetscSFLinkUnpackLeafData()
1054 …if (PetscMemTypeDevice(link->leafmtype) && link->SyncDevice && sf->unknown_input_stream) PetscCall… in PetscSFLinkUnpackLeafData()
1060 PetscErrorCode PetscSFLinkFetchAndOpRemote(PetscSF sf, PetscSFLink link, void *rootdata, MPI_Op op) in PetscSFLinkFetchAndOpRemote() argument
1066 … = link->rootmtype; in PetscSFLinkFetchAndOpRemote()
1073 …PetscCall(PetscSFLinkGetFetchAndOp(link, rootmtype, op, bas->rootdups[PETSCSF_REMOTE], &FetchAndOp… in PetscSFLinkFetchAndOpRemote()
1074 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, PETSCSF_REMOTE, &count, &start,… in PetscSFLinkFetchAndOpRemote()
1075 …PetscCall((*FetchAndOp)(link, count, start, opt, rootindices, rootdata, link->rootbuf[PETSCSF_REMO… in PetscSFLinkFetchAndOpRemote()
1077 PetscCall(PetscSFLinkLogFlopsAfterUnpackRootData(sf, link, PETSCSF_REMOTE, op)); in PetscSFLinkFetchAndOpRemote()
1082 PetscErrorCode PetscSFLinkScatterLocal(PetscSF sf, PetscSFLink link, PetscSFDirection direction, vo… in PetscSFLinkScatterLocal() argument
1087 PetscMemType rootmtype = link->rootmtype, leafmtype = link->leafmtype, srcmtype, dstmtype; in PetscSFLinkScatterLocal()
1098 PetscCall(PetscSFLinkPackRootData(sf, link, PETSCSF_LOCAL, rootdata)); in PetscSFLinkScatterLocal()
1100 srcbuf = link->rootbuf[PETSCSF_LOCAL][rootmtype]; in PetscSFLinkScatterLocal()
1102 dstbuf = link->leafbuf[PETSCSF_LOCAL][leafmtype]; in PetscSFLinkScatterLocal()
1104 PetscCall(PetscSFLinkPackLeafData(sf, link, PETSCSF_LOCAL, leafdata)); in PetscSFLinkScatterLocal()
1106 srcbuf = link->leafbuf[PETSCSF_LOCAL][leafmtype]; in PetscSFLinkScatterLocal()
1108 dstbuf = link->rootbuf[PETSCSF_LOCAL][rootmtype]; in PetscSFLinkScatterLocal()
1110 PetscCall((*link->Memcpy)(link, dstmtype, dstbuf, srcmtype, srcbuf, buflen * link->unitbytes)); in PetscSFLinkScatterLocal()
1112 if (PetscMemTypeHost(dstmtype)) PetscCall((*link->SyncStream)(link)); in PetscSFLinkScatterLocal()
1114 PetscCall(PetscSFLinkUnpackLeafData(sf, link, PETSCSF_LOCAL, leafdata, op)); in PetscSFLinkScatterLocal()
1116 PetscCall(PetscSFLinkUnpackRootData(sf, link, PETSCSF_LOCAL, rootdata, op)); in PetscSFLinkScatterLocal()
1120 dstmtype = (direction == PETSCSF_ROOT2LEAF) ? link->leafmtype : link->rootmtype; in PetscSFLinkScatterLocal()
1121 PetscCall(PetscSFLinkGetScatterAndOp(link, dstmtype, op, dstdups, &ScatterAndOp)); in PetscSFLinkScatterLocal()
1123 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, PETSCSF_LOCAL, &count, &rootsta… in PetscSFLinkScatterLocal()
1124 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, PETSCSF_LOCAL, &count, &leafsta… in PetscSFLinkScatterLocal()
1126 …PetscCall((*ScatterAndOp)(link, count, rootstart, rootopt, rootindices, rootdata, leafstart, leafo… in PetscSFLinkScatterLocal()
1128 …PetscCall((*ScatterAndOp)(link, count, leafstart, leafopt, leafindices, leafdata, rootstart, rooto… in PetscSFLinkScatterLocal()
1131 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, PETSCSF_LOCAL, &count,… in PetscSFLinkScatterLocal()
1132 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, PETSC_MEMTYPE_HOST, PETSCSF_LOCAL, &count,… in PetscSFLinkScatterLocal()
1134 …PetscCall(PetscSFLinkScatterDataWithMPIReduceLocal(sf, link, count, rootstart, rootindices, rootda… in PetscSFLinkScatterLocal()
1136 …PetscCall(PetscSFLinkScatterDataWithMPIReduceLocal(sf, link, count, leafstart, leafindices, leafda… in PetscSFLinkScatterLocal()
1144 PetscErrorCode PetscSFLinkFetchAndOpLocal(PetscSF sf, PetscSFLink link, void *rootdata, const void … in PetscSFLinkFetchAndOpLocal() argument
1149 const PetscMemType rootmtype = link->rootmtype, leafmtype = link->leafmtype; in PetscSFLinkFetchAndOpLocal()
1159 …PetscCall(PetscSFLinkGetRootPackOptAndIndices(sf, link, rootmtype, PETSCSF_LOCAL, &count, &rootsta… in PetscSFLinkFetchAndOpLocal()
1160 …PetscCall(PetscSFLinkGetLeafPackOptAndIndices(sf, link, leafmtype, PETSCSF_LOCAL, &count, &leafsta… in PetscSFLinkFetchAndOpLocal()
1161 …PetscCall(PetscSFLinkGetFetchAndOpLocal(link, rootmtype, op, bas->rootdups[PETSCSF_LOCAL], &FetchA… in PetscSFLinkFetchAndOpLocal()
1162 …PetscCall((*FetchAndOpLocal)(link, count, rootstart, rootopt, rootindices, rootdata, leafstart, le… in PetscSFLinkFetchAndOpLocal()