Lines Matching refs:stash
28 PetscErrorCode MatStashCreate_Private(MPI_Comm comm, PetscInt bs, MatStash *stash) in MatStashCreate_Private() argument
35 stash->comm = comm; in MatStashCreate_Private()
37 PetscCall(PetscCommGetNewTag(stash->comm, &stash->tag1)); in MatStashCreate_Private()
38 PetscCall(PetscCommGetNewTag(stash->comm, &stash->tag2)); in MatStashCreate_Private()
39 PetscCallMPI(MPI_Comm_size(stash->comm, &stash->size)); in MatStashCreate_Private()
40 PetscCallMPI(MPI_Comm_rank(stash->comm, &stash->rank)); in MatStashCreate_Private()
41 PetscCall(PetscMalloc1(2 * stash->size, &stash->flg_v)); in MatStashCreate_Private()
42 for (i = 0; i < 2 * stash->size; i++) stash->flg_v[i] = -1; in MatStashCreate_Private()
44 nopt = stash->size; in MatStashCreate_Private()
49 else if (nopt == stash->size) max = opt[stash->rank]; in MatStashCreate_Private()
50 else if (stash->rank < nopt) max = opt[stash->rank]; in MatStashCreate_Private()
52 stash->umax = max; in MatStashCreate_Private()
54 stash->umax = 0; in MatStashCreate_Private()
59 stash->bs = bs; in MatStashCreate_Private()
60 stash->nmax = 0; in MatStashCreate_Private()
61 stash->oldnmax = 0; in MatStashCreate_Private()
62 stash->n = 0; in MatStashCreate_Private()
63 stash->reallocs = -1; in MatStashCreate_Private()
64 stash->space_head = NULL; in MatStashCreate_Private()
65 stash->space = NULL; in MatStashCreate_Private()
67 stash->send_waits = NULL; in MatStashCreate_Private()
68 stash->recv_waits = NULL; in MatStashCreate_Private()
69 stash->send_status = NULL; in MatStashCreate_Private()
70 stash->nsends = 0; in MatStashCreate_Private()
71 stash->nrecvs = 0; in MatStashCreate_Private()
72 stash->svalues = NULL; in MatStashCreate_Private()
73 stash->rvalues = NULL; in MatStashCreate_Private()
74 stash->rindices = NULL; in MatStashCreate_Private()
75 stash->nprocessed = 0; in MatStashCreate_Private()
76 stash->reproduce = PETSC_FALSE; in MatStashCreate_Private()
77 stash->blocktype = MPI_DATATYPE_NULL; in MatStashCreate_Private()
79 PetscCall(PetscOptionsGetBool(NULL, NULL, "-matstash_reproduce", &stash->reproduce, NULL)); in MatStashCreate_Private()
84 stash->ScatterBegin = MatStashScatterBegin_BTS; in MatStashCreate_Private()
85 stash->ScatterGetMesg = MatStashScatterGetMesg_BTS; in MatStashCreate_Private()
86 stash->ScatterEnd = MatStashScatterEnd_BTS; in MatStashCreate_Private()
87 stash->ScatterDestroy = MatStashScatterDestroy_BTS; in MatStashCreate_Private()
90 stash->ScatterBegin = MatStashScatterBegin_Ref; in MatStashCreate_Private()
91 stash->ScatterGetMesg = MatStashScatterGetMesg_Ref; in MatStashCreate_Private()
92 stash->ScatterEnd = MatStashScatterEnd_Ref; in MatStashCreate_Private()
93 stash->ScatterDestroy = NULL; in MatStashCreate_Private()
103 PetscErrorCode MatStashDestroy_Private(MatStash *stash) in MatStashDestroy_Private() argument
106 PetscCall(PetscMatStashSpaceDestroy(&stash->space_head)); in MatStashDestroy_Private()
107 if (stash->ScatterDestroy) PetscCall((*stash->ScatterDestroy)(stash)); in MatStashDestroy_Private()
108 stash->space = NULL; in MatStashDestroy_Private()
109 PetscCall(PetscFree(stash->flg_v)); in MatStashDestroy_Private()
121 PetscErrorCode MatStashScatterEnd_Private(MatStash *stash) in MatStashScatterEnd_Private() argument
124 PetscCall((*stash->ScatterEnd)(stash)); in MatStashScatterEnd_Private()
128 PETSC_INTERN PetscErrorCode MatStashScatterEnd_Ref(MatStash *stash) in MatStashScatterEnd_Ref() argument
130 PetscMPIInt nsends = stash->nsends; in MatStashScatterEnd_Ref()
135 for (PetscMPIInt i = 0; i < 2 * stash->size; i++) stash->flg_v[i] = -1; in MatStashScatterEnd_Ref()
139 PetscCallMPI(MPI_Waitall(2 * nsends, stash->send_waits, send_status)); in MatStashScatterEnd_Ref()
146 if (stash->n) { in MatStashScatterEnd_Ref()
147 bs2 = stash->bs * stash->bs; in MatStashScatterEnd_Ref()
148 oldnmax = ((int)(stash->n * 1.1) + 5) * bs2; in MatStashScatterEnd_Ref()
149 if (oldnmax > stash->oldnmax) stash->oldnmax = oldnmax; in MatStashScatterEnd_Ref()
152 stash->nmax = 0; in MatStashScatterEnd_Ref()
153 stash->n = 0; in MatStashScatterEnd_Ref()
154 stash->reallocs = -1; in MatStashScatterEnd_Ref()
155 stash->nprocessed = 0; in MatStashScatterEnd_Ref()
157 PetscCall(PetscMatStashSpaceDestroy(&stash->space_head)); in MatStashScatterEnd_Ref()
159 stash->space = NULL; in MatStashScatterEnd_Ref()
161 PetscCall(PetscFree(stash->send_waits)); in MatStashScatterEnd_Ref()
162 PetscCall(PetscFree(stash->recv_waits)); in MatStashScatterEnd_Ref()
163 PetscCall(PetscFree2(stash->svalues, stash->sindices)); in MatStashScatterEnd_Ref()
164 PetscCall(PetscFree(stash->rvalues[0])); in MatStashScatterEnd_Ref()
165 PetscCall(PetscFree(stash->rvalues)); in MatStashScatterEnd_Ref()
166 PetscCall(PetscFree(stash->rindices[0])); in MatStashScatterEnd_Ref()
167 PetscCall(PetscFree(stash->rindices)); in MatStashScatterEnd_Ref()
180 PetscErrorCode MatStashGetInfo_Private(MatStash *stash, PetscInt *nstash, PetscInt *reallocs) in MatStashGetInfo_Private() argument
182 PetscInt bs2 = stash->bs * stash->bs; in MatStashGetInfo_Private()
185 if (nstash) *nstash = stash->n * bs2; in MatStashGetInfo_Private()
187 if (stash->reallocs < 0) *reallocs = 0; in MatStashGetInfo_Private()
188 else *reallocs = stash->reallocs; in MatStashGetInfo_Private()
201 PetscErrorCode MatStashSetInitialSize_Private(MatStash *stash, PetscInt max) in MatStashSetInitialSize_Private() argument
204 stash->umax = max; in MatStashSetInitialSize_Private()
219 static PetscErrorCode MatStashExpand_Private(MatStash *stash, PetscInt incr) in MatStashExpand_Private() argument
221 PetscInt newnmax, bs2 = stash->bs * stash->bs; in MatStashExpand_Private()
226 if (!stash->oldnmax && !stash->nmax) { /* new stash */ in MatStashExpand_Private()
227 if (stash->umax) cnewnmax = stash->umax / bs2; in MatStashExpand_Private()
229 } else if (!stash->nmax) { /* reusing stash */ in MatStashExpand_Private()
230 if (stash->umax > stash->oldnmax) cnewnmax = stash->umax / bs2; in MatStashExpand_Private()
231 else cnewnmax = stash->oldnmax / bs2; in MatStashExpand_Private()
232 } else cnewnmax = stash->nmax * 2; in MatStashExpand_Private()
233 if (cnewnmax < (stash->nmax + incr)) cnewnmax += 2 * incr; in MatStashExpand_Private()
237 PetscCall(PetscMatStashSpaceGet(bs2, newnmax, &stash->space)); in MatStashExpand_Private()
238 if (!stash->space_head) { /* new stash or reusing stash->oldnmax */ in MatStashExpand_Private()
239 stash->space_head = stash->space; in MatStashExpand_Private()
242 stash->reallocs++; in MatStashExpand_Private()
243 stash->nmax = newnmax; in MatStashExpand_Private()
258 PetscErrorCode MatStashValuesRow_Private(MatStash *stash, PetscInt row, PetscInt n, const PetscInt … in MatStashValuesRow_Private() argument
261 PetscMatStashSpace space = stash->space; in MatStashValuesRow_Private()
265 if (!space || space->local_remaining < n) PetscCall(MatStashExpand_Private(stash, n)); in MatStashValuesRow_Private()
266 space = stash->space; in MatStashValuesRow_Private()
276 stash->n += cnt; in MatStashValuesRow_Private()
296 PetscErrorCode MatStashValuesCol_Private(MatStash *stash, PetscInt row, PetscInt n, const PetscInt … in MatStashValuesCol_Private() argument
299 PetscMatStashSpace space = stash->space; in MatStashValuesCol_Private()
303 if (!space || space->local_remaining < n) PetscCall(MatStashExpand_Private(stash, n)); in MatStashValuesCol_Private()
304 space = stash->space; in MatStashValuesCol_Private()
314 stash->n += cnt; in MatStashValuesCol_Private()
338 PetscErrorCode MatStashValuesRowBlocked_Private(MatStash *stash, PetscInt row, PetscInt n, const Pe… in MatStashValuesRowBlocked_Private() argument
340 PetscInt i, j, k, bs2, bs = stash->bs, l; in MatStashValuesRowBlocked_Private()
343 PetscMatStashSpace space = stash->space; in MatStashValuesRowBlocked_Private()
346 if (!space || space->local_remaining < n) PetscCall(MatStashExpand_Private(stash, n)); in MatStashValuesRowBlocked_Private()
347 space = stash->space; in MatStashValuesRowBlocked_Private()
365 stash->n += n; in MatStashValuesRowBlocked_Private()
389 PetscErrorCode MatStashValuesColBlocked_Private(MatStash *stash, PetscInt row, PetscInt n, const Pe… in MatStashValuesColBlocked_Private() argument
391 PetscInt i, j, k, bs2, bs = stash->bs, l; in MatStashValuesColBlocked_Private()
394 PetscMatStashSpace space = stash->space; in MatStashValuesColBlocked_Private()
397 if (!space || space->local_remaining < n) PetscCall(MatStashExpand_Private(stash, n)); in MatStashValuesColBlocked_Private()
398 space = stash->space; in MatStashValuesColBlocked_Private()
416 stash->n += n; in MatStashValuesColBlocked_Private()
437 PetscErrorCode MatStashScatterBegin_Private(Mat mat, MatStash *stash, PetscInt *owners) in MatStashScatterBegin_Private() argument
440 PetscCall((*stash->ScatterBegin)(mat, stash, owners)); in MatStashScatterBegin_Private()
444 static PetscErrorCode MatStashScatterBegin_Ref(Mat mat, MatStash *stash, PetscInt *owners) in MatStashScatterBegin_Ref() argument
447 PetscInt size = stash->size; in MatStashScatterBegin_Ref()
450 MPI_Comm comm = stash->comm; in MatStashScatterBegin_Ref()
452 PetscMPIInt *sizes, *nlengths, nreceives, nsends, tag1 = stash->tag1, tag2 = stash->tag2; in MatStashScatterBegin_Ref()
465 bs2 = stash->bs * stash->bs; in MatStashScatterBegin_Ref()
469 PetscCall(PetscMalloc1(stash->n + 1, &owner)); in MatStashScatterBegin_Ref()
473 space = stash->space_head; in MatStashScatterBegin_Ref()
522 PetscCall(PetscMalloc2(bs2 * stash->n, &svalues, 2 * (stash->n + 1), &sindices)); in MatStashScatterBegin_Ref()
534 space = stash->space_head; in MatStashScatterBegin_Ref()
586 stash->recv_waits = recv_waits; in MatStashScatterBegin_Ref()
591 stash->svalues = svalues; in MatStashScatterBegin_Ref()
592 stash->sindices = sindices; in MatStashScatterBegin_Ref()
593 stash->rvalues = rvalues; in MatStashScatterBegin_Ref()
594 stash->rindices = rindices; in MatStashScatterBegin_Ref()
595 stash->send_waits = send_waits; in MatStashScatterBegin_Ref()
596 stash->nsends = nsends; in MatStashScatterBegin_Ref()
597 stash->nrecvs = nreceives; in MatStashScatterBegin_Ref()
598 stash->reproduce_count = 0; in MatStashScatterBegin_Ref()
620 PetscErrorCode MatStashScatterGetMesg_Private(MatStash *stash, PetscMPIInt *nvals, PetscInt **rows,… in MatStashScatterGetMesg_Private() argument
623 PetscCall((*stash->ScatterGetMesg)(stash, nvals, rows, cols, vals, flg)); in MatStashScatterGetMesg_Private()
627 PETSC_INTERN PetscErrorCode MatStashScatterGetMesg_Ref(MatStash *stash, PetscMPIInt *nvals, PetscIn… in MatStashScatterGetMesg_Ref() argument
629 PetscMPIInt i, *flg_v = stash->flg_v, i1, i2; in MatStashScatterGetMesg_Ref()
637 if (stash->nprocessed == stash->nrecvs) PetscFunctionReturn(PETSC_SUCCESS); in MatStashScatterGetMesg_Ref()
639 bs2 = stash->bs * stash->bs; in MatStashScatterGetMesg_Ref()
643 if (stash->reproduce) { in MatStashScatterGetMesg_Ref()
644 i = stash->reproduce_count++; in MatStashScatterGetMesg_Ref()
645 PetscCallMPI(MPI_Wait(stash->recv_waits + i, &recv_status)); in MatStashScatterGetMesg_Ref()
647 PetscCallMPI(MPI_Waitany(2 * stash->nrecvs, stash->recv_waits, &i, &recv_status)); in MatStashScatterGetMesg_Ref()
666 *rows = stash->rindices[i2]; in MatStashScatterGetMesg_Ref()
668 *vals = stash->rvalues[i1]; in MatStashScatterGetMesg_Ref()
670 stash->nprocessed++; in MatStashScatterGetMesg_Ref()
684 static PetscErrorCode MatStashSortCompress_Private(MatStash *stash, InsertMode insertmode) in MatStashSortCompress_Private() argument
687 …PetscInt n = stash->n, bs = stash->bs, bs2 = bs * bs, cnt, *row, *col, *perm, rowstart, … in MatStashSortCompress_Private()
692 for (space = stash->space_head, cnt = 0; space; space = space->next) { in MatStashSortCompress_Private()
711 PetscCall(PetscSegBufferGet(stash->segsendblocks, 1, &block)); in MatStashSortCompress_Private()
731 static PetscErrorCode MatStashBlockTypeSetUp(MatStash *stash) in MatStashBlockTypeSetUp() argument
734 if (stash->blocktype == MPI_DATATYPE_NULL) { in MatStashBlockTypeSetUp()
735 PetscInt bs2 = PetscSqr(stash->bs); in MatStashBlockTypeSetUp()
761 stash->blocktype_size = offsetof(struct DummyBlock, vals) + bs2 * sizeof(PetscScalar); in MatStashBlockTypeSetUp()
762 …if (stash->blocktype_size % sizeof(PetscInt)) { /* Implies that PetscInt is larger and does not sa… in MatStashBlockTypeSetUp()
763 stash->blocktype_size += sizeof(PetscInt) - stash->blocktype_size % sizeof(PetscInt); in MatStashBlockTypeSetUp()
765 PetscCall(PetscSegBufferCreate(stash->blocktype_size, 1, &stash->segsendblocks)); in MatStashBlockTypeSetUp()
766 PetscCall(PetscSegBufferCreate(stash->blocktype_size, 1, &stash->segrecvblocks)); in MatStashBlockTypeSetUp()
767 PetscCall(PetscSegBufferCreate(sizeof(MatStashFrame), 1, &stash->segrecvframe)); in MatStashBlockTypeSetUp()
776 PetscCallMPI(MPI_Type_create_resized(stype, 0, stash->blocktype_size, &stash->blocktype)); in MatStashBlockTypeSetUp()
777 PetscCallMPI(MPI_Type_commit(&stash->blocktype)); in MatStashBlockTypeSetUp()
788 MatStash *stash = (MatStash *)ctx; in MatStashBTSSend_Private() local
792 …tscCheck(rank == stash->sendranks[rankid], comm, PETSC_ERR_PLIB, "BTS Send rank %d does not match … in MatStashBTSSend_Private()
793 …PetscCallMPI(MPIU_Isend(stash->sendframes[rankid].buffer, hdr->count, stash->blocktype, rank, tag[… in MatStashBTSSend_Private()
794 stash->sendframes[rankid].count = hdr->count; in MatStashBTSSend_Private()
795 stash->sendframes[rankid].pending = 1; in MatStashBTSSend_Private()
805 MatStash *stash = (MatStash *)ctx; in MatStashBTSRecv_Private() local
810 PetscCall(PetscSegBufferGet(stash->segrecvframe, 1, &frame)); in MatStashBTSRecv_Private()
811 PetscCall(PetscSegBufferGet(stash->segrecvblocks, hdr->count, &frame->buffer)); in MatStashBTSRecv_Private()
812 …PetscCallMPI(MPIU_Irecv(frame->buffer, hdr->count, stash->blocktype, rank, tag[0], comm, &req[0])); in MatStashBTSRecv_Private()
821 static PetscErrorCode MatStashScatterBegin_BTS(Mat mat, MatStash *stash, PetscInt owners[]) in MatStashScatterBegin_BTS() argument
833 PetscCall(MatStashBlockTypeSetUp(stash)); in MatStashScatterBegin_BTS()
834 PetscCall(MatStashSortCompress_Private(stash, mat->insertmode)); in MatStashScatterBegin_BTS()
835 PetscCall(PetscSegBufferGetSize(stash->segsendblocks, &nblocks)); in MatStashScatterBegin_BTS()
836 PetscCall(PetscSegBufferExtractInPlace(stash->segsendblocks, &sendblocks)); in MatStashScatterBegin_BTS()
837 …if (stash->first_assembly_done) { /* Set up sendhdrs and sendframes for each rank that we sent bef… in MatStashScatterBegin_BTS()
840 for (i = 0, b = 0; i < stash->nsendranks; i++) { in MatStashScatterBegin_BTS()
841 stash->sendframes[i].buffer = &sendblocks[b * stash->blocktype_size]; in MatStashScatterBegin_BTS()
843 …stash->sendhdr[i].count = 0; /* Might remain empty (in which case we send a zero-sized message) if… in MatStashScatterBegin_BTS()
845 MatStashBlock *sendblock_b = (MatStashBlock *)&sendblocks[b * stash->blocktype_size]; in MatStashScatterBegin_BTS()
846 …stash->sendranks[i]], stash->comm, PETSC_ERR_ARG_WRONG, "MAT_SUBSET_OFF_PROC_ENTRIES set, but row … in MatStashScatterBegin_BTS()
847 if (sendblock_b->row >= owners[stash->sendranks[i] + 1]) break; in MatStashScatterBegin_BTS()
848 stash->sendhdr[i].count++; in MatStashScatterBegin_BTS()
856 stash->nsendranks = 0; in MatStashScatterBegin_BTS()
859 …MatStashBlock *sendblock_rowstart = (MatStashBlock *)&sendblocks[rowstart * stash->blocktype_size]; in MatStashScatterBegin_BTS()
861 PetscCall(PetscFindInt(sendblock_rowstart->row, stash->size + 1, owners, &owner)); in MatStashScatterBegin_BTS()
864 MatStashBlock *sendblock_i = (MatStashBlock *)&sendblocks[i * stash->blocktype_size]; in MatStashScatterBegin_BTS()
868 stash->nsendranks++; in MatStashScatterBegin_BTS()
871 …PetscCall(PetscMalloc3(stash->nsendranks, &stash->sendranks, stash->nsendranks, &stash->sendhdr, s… in MatStashScatterBegin_BTS()
878 …MatStashBlock *sendblock_rowstart = (MatStashBlock *)&sendblocks[rowstart * stash->blocktype_size]; in MatStashScatterBegin_BTS()
880 PetscCall(PetscFindInt(sendblock_rowstart->row, stash->size + 1, owners, &iowner)); in MatStashScatterBegin_BTS()
883 stash->sendranks[sendno] = owner; in MatStashScatterBegin_BTS()
885 MatStashBlock *sendblock_i = (MatStashBlock *)&sendblocks[i * stash->blocktype_size]; in MatStashScatterBegin_BTS()
889 stash->sendframes[sendno].buffer = sendblock_rowstart; in MatStashScatterBegin_BTS()
890 stash->sendframes[sendno].pending = 0; in MatStashScatterBegin_BTS()
891 PetscCall(PetscIntCast(i - rowstart, &stash->sendhdr[sendno].count)); in MatStashScatterBegin_BTS()
895 …PetscCheck(sendno == stash->nsendranks, stash->comm, PETSC_ERR_PLIB, "BTS counted %d sendranks, bu… in MatStashScatterBegin_BTS()
902 MatStashBlock *sendblock_i = (MatStashBlock *)&sendblocks[i * stash->blocktype_size]; in MatStashScatterBegin_BTS()
907 if (stash->first_assembly_done) { in MatStashScatterBegin_BTS()
910 PetscCall(PetscCommGetNewTag(stash->comm, &tag)); in MatStashScatterBegin_BTS()
911 …i < stash->nrecvranks; i++) PetscCall(MatStashBTSRecv_Private(stash->comm, &tag, stash->recvranks[… in MatStashScatterBegin_BTS()
912 … < stash->nsendranks; i++) PetscCall(MatStashBTSSend_Private(stash->comm, &tag, i, stash->sendrank… in MatStashScatterBegin_BTS()
913 stash->use_status = PETSC_TRUE; /* Use count from message status. */ in MatStashScatterBegin_BTS()
915 …stash->comm, 1, MPIU_INT, stash->nsendranks, stash->sendranks, (PetscInt *)stash->sendhdr, &stash-… in MatStashScatterBegin_BTS()
916 …PetscCall(PetscMalloc2(stash->nrecvranks, &stash->some_indices, stash->nrecvranks, &stash->some_st… in MatStashScatterBegin_BTS()
917 stash->use_status = PETSC_FALSE; /* Use count from header instead of from message. */ in MatStashScatterBegin_BTS()
920 PetscCall(PetscSegBufferExtractInPlace(stash->segrecvframe, &stash->recvframes)); in MatStashScatterBegin_BTS()
921 stash->recvframe_active = NULL; in MatStashScatterBegin_BTS()
922 stash->recvframe_i = 0; in MatStashScatterBegin_BTS()
923 stash->some_i = 0; in MatStashScatterBegin_BTS()
924 stash->some_count = 0; in MatStashScatterBegin_BTS()
925 stash->recvcount = 0; in MatStashScatterBegin_BTS()
926 …stash->first_assembly_done = mat->assembly_subset; /* See the same logic in VecAssemblyBegin_MPI_B… in MatStashScatterBegin_BTS()
927 stash->insertmode = &mat->insertmode; in MatStashScatterBegin_BTS()
931 static PetscErrorCode MatStashScatterGetMesg_BTS(MatStash *stash, PetscMPIInt *n, PetscInt **row, P… in MatStashScatterGetMesg_BTS() argument
937 while (!stash->recvframe_active || stash->recvframe_i == stash->recvframe_count) { in MatStashScatterGetMesg_BTS()
938 if (stash->some_i == stash->some_count) { in MatStashScatterGetMesg_BTS()
939 if (stash->recvcount == stash->nrecvranks) PetscFunctionReturn(PETSC_SUCCESS); /* Done */ in MatStashScatterGetMesg_BTS()
940 …etscCallMPI(MPI_Waitsome(stash->nrecvranks, stash->recvreqs, &stash->some_count, stash->some_indic… in MatStashScatterGetMesg_BTS()
941 stash->some_i = 0; in MatStashScatterGetMesg_BTS()
943 stash->recvframe_active = &stash->recvframes[stash->some_indices[stash->some_i]]; in MatStashScatterGetMesg_BTS()
944 stash->recvframe_count = stash->recvframe_active->count; /* From header; maximum count */ in MatStashScatterGetMesg_BTS()
945 if (stash->use_status) { /* Count what was actually sent */ in MatStashScatterGetMesg_BTS()
948 PetscCallMPI(MPI_Get_count(&stash->some_statuses[stash->some_i], stash->blocktype, &ic)); in MatStashScatterGetMesg_BTS()
949 stash->recvframe_count = ic; in MatStashScatterGetMesg_BTS()
951 if (stash->recvframe_count > 0) { /* Check for InsertMode consistency */ in MatStashScatterGetMesg_BTS()
952 block = (MatStashBlock *)&((char *)stash->recvframe_active->buffer)[0]; in MatStashScatterGetMesg_BTS()
953 …if (PetscUnlikely(*stash->insertmode == NOT_SET_VALUES)) *stash->insertmode = block->row < 0 ? INS… in MatStashScatterGetMesg_BTS()
954 …stash->insertmode != INSERT_VALUES || block->row < 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "… in MatStashScatterGetMesg_BTS()
955 …stash->insertmode != ADD_VALUES || block->row >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "As… in MatStashScatterGetMesg_BTS()
957 stash->some_i++; in MatStashScatterGetMesg_BTS()
958 stash->recvcount++; in MatStashScatterGetMesg_BTS()
959 stash->recvframe_i = 0; in MatStashScatterGetMesg_BTS()
962 …block = (MatStashBlock *)&((char *)stash->recvframe_active->buffer)[stash->recvframe_i * stash->bl… in MatStashScatterGetMesg_BTS()
967 stash->recvframe_i++; in MatStashScatterGetMesg_BTS()
972 static PetscErrorCode MatStashScatterEnd_BTS(MatStash *stash) in MatStashScatterEnd_BTS() argument
975 PetscCallMPI(MPI_Waitall(stash->nsendranks, stash->sendreqs, MPI_STATUSES_IGNORE)); in MatStashScatterEnd_BTS()
976 …if (stash->first_assembly_done) { /* Reuse the communication contexts, so consolidate and reset se… in MatStashScatterEnd_BTS()
977 PetscCall(PetscSegBufferExtractInPlace(stash->segrecvblocks, NULL)); in MatStashScatterEnd_BTS()
979 PetscCall(MatStashScatterDestroy_BTS(stash)); in MatStashScatterEnd_BTS()
985 if (stash->n) { in MatStashScatterEnd_BTS()
986 PetscInt bs2 = stash->bs * stash->bs; in MatStashScatterEnd_BTS()
987 PetscInt oldnmax = ((int)(stash->n * 1.1) + 5) * bs2; in MatStashScatterEnd_BTS()
988 if (oldnmax > stash->oldnmax) stash->oldnmax = oldnmax; in MatStashScatterEnd_BTS()
991 stash->nmax = 0; in MatStashScatterEnd_BTS()
992 stash->n = 0; in MatStashScatterEnd_BTS()
993 stash->reallocs = -1; in MatStashScatterEnd_BTS()
994 stash->nprocessed = 0; in MatStashScatterEnd_BTS()
996 PetscCall(PetscMatStashSpaceDestroy(&stash->space_head)); in MatStashScatterEnd_BTS()
998 stash->space = NULL; in MatStashScatterEnd_BTS()
1002 PetscErrorCode MatStashScatterDestroy_BTS(MatStash *stash) in MatStashScatterDestroy_BTS() argument
1005 PetscCall(PetscSegBufferDestroy(&stash->segsendblocks)); in MatStashScatterDestroy_BTS()
1006 PetscCall(PetscSegBufferDestroy(&stash->segrecvframe)); in MatStashScatterDestroy_BTS()
1007 stash->recvframes = NULL; in MatStashScatterDestroy_BTS()
1008 PetscCall(PetscSegBufferDestroy(&stash->segrecvblocks)); in MatStashScatterDestroy_BTS()
1009 if (stash->blocktype != MPI_DATATYPE_NULL) PetscCallMPI(MPI_Type_free(&stash->blocktype)); in MatStashScatterDestroy_BTS()
1010 stash->nsendranks = 0; in MatStashScatterDestroy_BTS()
1011 stash->nrecvranks = 0; in MatStashScatterDestroy_BTS()
1012 PetscCall(PetscFree3(stash->sendranks, stash->sendhdr, stash->sendframes)); in MatStashScatterDestroy_BTS()
1013 PetscCall(PetscFree(stash->sendreqs)); in MatStashScatterDestroy_BTS()
1014 PetscCall(PetscFree(stash->recvreqs)); in MatStashScatterDestroy_BTS()
1015 PetscCall(PetscFree(stash->recvranks)); in MatStashScatterDestroy_BTS()
1016 PetscCall(PetscFree(stash->recvhdr)); in MatStashScatterDestroy_BTS()
1017 PetscCall(PetscFree2(stash->some_indices, stash->some_statuses)); in MatStashScatterDestroy_BTS()