xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision ac0ced25246c2957133fb7f70299f8f7572cb799)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12     case DM_POLYTOPE_POINT_PRISM_TENSOR:
13     case DM_POLYTOPE_SEG_PRISM_TENSOR:
14     case DM_POLYTOPE_TRI_PRISM_TENSOR:
15     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16       *isHybrid = PETSC_TRUE;
17     default: *isHybrid = PETSC_FALSE;
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
23 {
24   DMLabel        ctLabel;
25 
26   PetscFunctionBegin;
27   if (cStart) *cStart = -1;
28   if (cEnd)   *cEnd   = -1;
29   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
30   switch (dim) {
31     case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));break;
32     case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));break;
33     case 3:
34       PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
35       if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
36       break;
37     default: PetscFunctionReturn(0);
38   }
39   PetscFunctionReturn(0);
40 }
41 
42 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
43 {
44   PetscInt       fStart, fEnd, f;
45 
46   PetscFunctionBegin;
47   PetscCall(DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd));
48   for (f = fStart; f < fEnd; ++f) {
49     PetscInt supportSize;
50 
51     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
52     if (supportSize == 1) {
53       if (val < 0) {
54         PetscInt *closure = NULL;
55         PetscInt  clSize, cl, cval;
56 
57         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
58         for (cl = 0; cl < clSize*2; cl += 2) {
59           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
60           if (cval < 0) continue;
61           PetscCall(DMLabelSetValue(label, f, cval));
62           break;
63         }
64         if (cl == clSize*2) PetscCall(DMLabelSetValue(label, f, 1));
65         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
66       } else {
67         PetscCall(DMLabelSetValue(label, f, val));
68       }
69     }
70   }
71   PetscFunctionReturn(0);
72 }
73 
74 /*@
75   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
76 
77   Not Collective
78 
79   Input Parameters:
80 + dm - The original DM
81 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
82 
83   Output Parameter:
84 . label - The DMLabel marking boundary faces with the given value
85 
86   Level: developer
87 
88 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
89 @*/
90 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
91 {
92   DMPlexInterpolatedFlag  flg;
93 
94   PetscFunctionBegin;
95   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
96   PetscCall(DMPlexIsInterpolated(dm, &flg));
97   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
98   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
99   PetscFunctionReturn(0);
100 }
101 
102 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
103 {
104   IS              valueIS;
105   PetscSF         sfPoint;
106   const PetscInt *values;
107   PetscInt        numValues, v, cStart, cEnd, nroots;
108 
109   PetscFunctionBegin;
110   PetscCall(DMLabelGetNumValues(label, &numValues));
111   PetscCall(DMLabelGetValueIS(label, &valueIS));
112   PetscCall(DMPlexGetHeightStratum(dm,0,&cStart,&cEnd));
113   PetscCall(ISGetIndices(valueIS, &values));
114   for (v = 0; v < numValues; ++v) {
115     IS              pointIS;
116     const PetscInt *points;
117     PetscInt        numPoints, p;
118 
119     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
120     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
121     PetscCall(ISGetIndices(pointIS, &points));
122     for (p = 0; p < numPoints; ++p) {
123       PetscInt  q = points[p];
124       PetscInt *closure = NULL;
125       PetscInt  closureSize, c;
126 
127       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
128         continue;
129       }
130       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
131       for (c = 0; c < closureSize*2; c += 2) {
132         PetscCall(DMLabelSetValue(label, closure[c], values[v]));
133       }
134       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
135     }
136     PetscCall(ISRestoreIndices(pointIS, &points));
137     PetscCall(ISDestroy(&pointIS));
138   }
139   PetscCall(ISRestoreIndices(valueIS, &values));
140   PetscCall(ISDestroy(&valueIS));
141   PetscCall(DMGetPointSF(dm, &sfPoint));
142   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
143   if (nroots >= 0) {
144     DMLabel         lblRoots, lblLeaves;
145     IS              valueIS, pointIS;
146     const PetscInt *values;
147     PetscInt        numValues, v;
148 
149     /* Pull point contributions from remote leaves into local roots */
150     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
151     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
152     PetscCall(ISGetLocalSize(valueIS, &numValues));
153     PetscCall(ISGetIndices(valueIS, &values));
154     for (v = 0; v < numValues; ++v) {
155       const PetscInt value = values[v];
156 
157       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
158       PetscCall(DMLabelInsertIS(label, pointIS, value));
159       PetscCall(ISDestroy(&pointIS));
160     }
161     PetscCall(ISRestoreIndices(valueIS, &values));
162     PetscCall(ISDestroy(&valueIS));
163     PetscCall(DMLabelDestroy(&lblLeaves));
164     /* Push point contributions from roots into remote leaves */
165     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
166     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
167     PetscCall(ISGetLocalSize(valueIS, &numValues));
168     PetscCall(ISGetIndices(valueIS, &values));
169     for (v = 0; v < numValues; ++v) {
170       const PetscInt value = values[v];
171 
172       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
173       PetscCall(DMLabelInsertIS(label, pointIS, value));
174       PetscCall(ISDestroy(&pointIS));
175     }
176     PetscCall(ISRestoreIndices(valueIS, &values));
177     PetscCall(ISDestroy(&valueIS));
178     PetscCall(DMLabelDestroy(&lblRoots));
179   }
180   PetscFunctionReturn(0);
181 }
182 
183 /*@
184   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
185 
186   Input Parameters:
187 + dm - The DM
188 - label - A DMLabel marking the surface points
189 
190   Output Parameter:
191 . label - A DMLabel marking all surface points in the transitive closure
192 
193   Level: developer
194 
195 .seealso: `DMPlexLabelCohesiveComplete()`
196 @*/
197 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
198 {
199   PetscFunctionBegin;
200   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
201   PetscFunctionReturn(0);
202 }
203 
204 /*@
205   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
206 
207   Input Parameters:
208 + dm - The DM
209 - label - A DMLabel marking the surface points
210 
211   Output Parameter:
212 . label - A DMLabel incorporating cells
213 
214   Level: developer
215 
216   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
217 
218 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
219 @*/
220 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
221 {
222   IS              valueIS;
223   const PetscInt *values;
224   PetscInt        numValues, v, cStart, cEnd;
225 
226   PetscFunctionBegin;
227   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
228   PetscCall(DMLabelGetNumValues(label, &numValues));
229   PetscCall(DMLabelGetValueIS(label, &valueIS));
230   PetscCall(ISGetIndices(valueIS, &values));
231   for (v = 0; v < numValues; ++v) {
232     IS              pointIS;
233     const PetscInt *points;
234     PetscInt        numPoints, p;
235 
236     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
237     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
238     PetscCall(ISGetIndices(pointIS, &points));
239     for (p = 0; p < numPoints; ++p) {
240       PetscInt *closure = NULL;
241       PetscInt  closureSize, cl;
242 
243       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
244       for (cl = closureSize-1; cl > 0; --cl) {
245         const PetscInt cell = closure[cl*2];
246         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
247       }
248       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
249     }
250     PetscCall(ISRestoreIndices(pointIS, &points));
251     PetscCall(ISDestroy(&pointIS));
252   }
253   PetscCall(ISRestoreIndices(valueIS, &values));
254   PetscCall(ISDestroy(&valueIS));
255   PetscFunctionReturn(0);
256 }
257 
258 /*@
259   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
260 
261   Input Parameters:
262 + dm - The DM
263 - label - A DMLabel marking the surface points
264 
265   Output Parameter:
266 . label - A DMLabel incorporating cells
267 
268   Level: developer
269 
270   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
271 
272 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
273 @*/
274 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
275 {
276   IS              valueIS;
277   const PetscInt *values;
278   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
279 
280   PetscFunctionBegin;
281   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
282   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
283   PetscCall(DMLabelGetNumValues(label, &numValues));
284   PetscCall(DMLabelGetValueIS(label, &valueIS));
285   PetscCall(ISGetIndices(valueIS, &values));
286   for (v = 0; v < numValues; ++v) {
287     IS              pointIS;
288     const PetscInt *points;
289     PetscInt        numPoints, p;
290 
291     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
292     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
293     PetscCall(ISGetIndices(pointIS, &points));
294     for (p = 0; p < numPoints; ++p) {
295       const PetscInt face = points[p];
296       PetscInt      *closure = NULL;
297       PetscInt       closureSize, cl;
298 
299       if ((face < fStart) || (face >= fEnd)) continue;
300       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301       for (cl = closureSize-1; cl > 0; --cl) {
302         const PetscInt cell = closure[cl*2];
303         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
304       }
305       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
306     }
307     PetscCall(ISRestoreIndices(pointIS, &points));
308     PetscCall(ISDestroy(&pointIS));
309   }
310   PetscCall(ISRestoreIndices(valueIS, &values));
311   PetscCall(ISDestroy(&valueIS));
312   PetscFunctionReturn(0);
313 }
314 
315 /*@
316   DMPlexLabelClearCells - Remove cells from a label
317 
318   Input Parameters:
319 + dm - The DM
320 - label - A DMLabel marking surface points and their adjacent cells
321 
322   Output Parameter:
323 . label - A DMLabel without cells
324 
325   Level: developer
326 
327   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
328 
329 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
330 @*/
331 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
332 {
333   IS              valueIS;
334   const PetscInt *values;
335   PetscInt        numValues, v, cStart, cEnd;
336 
337   PetscFunctionBegin;
338   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
339   PetscCall(DMLabelGetNumValues(label, &numValues));
340   PetscCall(DMLabelGetValueIS(label, &valueIS));
341   PetscCall(ISGetIndices(valueIS, &values));
342   for (v = 0; v < numValues; ++v) {
343     IS              pointIS;
344     const PetscInt *points;
345     PetscInt        numPoints, p;
346 
347     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
348     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
349     PetscCall(ISGetIndices(pointIS, &points));
350     for (p = 0; p < numPoints; ++p) {
351       PetscInt point = points[p];
352 
353       if (point >= cStart && point < cEnd) {
354         PetscCall(DMLabelClearValue(label,point,values[v]));
355       }
356     }
357     PetscCall(ISRestoreIndices(pointIS, &points));
358     PetscCall(ISDestroy(&pointIS));
359   }
360   PetscCall(ISRestoreIndices(valueIS, &values));
361   PetscCall(ISDestroy(&valueIS));
362   PetscFunctionReturn(0);
363 }
364 
365 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
366  * index (skipping first, which is (0,0)) */
367 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
368 {
369   PetscInt d, off = 0;
370 
371   PetscFunctionBegin;
372   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
373   for (d = 0; d < depth; d++) {
374     PetscInt firstd = d;
375     PetscInt firstStart = depthShift[2*d];
376     PetscInt e;
377 
378     for (e = d+1; e <= depth; e++) {
379       if (depthShift[2*e] < firstStart) {
380         firstd = e;
381         firstStart = depthShift[2*d];
382       }
383     }
384     if (firstd != d) {
385       PetscInt swap[2];
386 
387       e = firstd;
388       swap[0] = depthShift[2*d];
389       swap[1] = depthShift[2*d+1];
390       depthShift[2*d]   = depthShift[2*e];
391       depthShift[2*d+1] = depthShift[2*e+1];
392       depthShift[2*e]   = swap[0];
393       depthShift[2*e+1] = swap[1];
394     }
395   }
396   /* convert (oldstart, added) to (oldstart, newstart) */
397   for (d = 0; d <= depth; d++) {
398     off += depthShift[2*d+1];
399     depthShift[2*d+1] = depthShift[2*d] + off;
400   }
401   PetscFunctionReturn(0);
402 }
403 
404 /* depthShift is a list of (old, new) pairs */
405 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
406 {
407   PetscInt d;
408   PetscInt newOff = 0;
409 
410   for (d = 0; d <= depth; d++) {
411     if (p < depthShift[2*d]) return p + newOff;
412     else newOff = depthShift[2*d+1] - depthShift[2*d];
413   }
414   return p + newOff;
415 }
416 
417 /* depthShift is a list of (old, new) pairs */
418 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
419 {
420   PetscInt d;
421   PetscInt newOff = 0;
422 
423   for (d = 0; d <= depth; d++) {
424     if (p < depthShift[2*d+1]) return p + newOff;
425     else newOff = depthShift[2*d] - depthShift[2*d+1];
426   }
427   return p + newOff;
428 }
429 
430 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
431 {
432   PetscInt       depth = 0, d, pStart, pEnd, p;
433   DMLabel        depthLabel;
434 
435   PetscFunctionBegin;
436   PetscCall(DMPlexGetDepth(dm, &depth));
437   if (depth < 0) PetscFunctionReturn(0);
438   /* Step 1: Expand chart */
439   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
440   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
441   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
442   PetscCall(DMCreateLabel(dmNew,"depth"));
443   PetscCall(DMPlexGetDepthLabel(dmNew,&depthLabel));
444   PetscCall(DMCreateLabel(dmNew, "celltype"));
445   /* Step 2: Set cone and support sizes */
446   for (d = 0; d <= depth; ++d) {
447     PetscInt pStartNew, pEndNew;
448     IS pIS;
449 
450     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
451     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
452     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
453     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
454     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
455     PetscCall(ISDestroy(&pIS));
456     for (p = pStart; p < pEnd; ++p) {
457       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
458       PetscInt       size;
459       DMPolytopeType ct;
460 
461       PetscCall(DMPlexGetConeSize(dm, p, &size));
462       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
463       PetscCall(DMPlexGetSupportSize(dm, p, &size));
464       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
465       PetscCall(DMPlexGetCellType(dm, p, &ct));
466       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
467     }
468   }
469   PetscFunctionReturn(0);
470 }
471 
472 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
473 {
474   PetscInt      *newpoints;
475   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
476 
477   PetscFunctionBegin;
478   PetscCall(DMPlexGetDepth(dm, &depth));
479   if (depth < 0) PetscFunctionReturn(0);
480   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
481   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
482   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints));
483   /* Step 5: Set cones and supports */
484   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
485   for (p = pStart; p < pEnd; ++p) {
486     const PetscInt *points = NULL, *orientations = NULL;
487     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
488 
489     PetscCall(DMPlexGetConeSize(dm, p, &size));
490     PetscCall(DMPlexGetCone(dm, p, &points));
491     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
492     for (i = 0; i < size; ++i) {
493       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
494     }
495     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
496     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
497     PetscCall(DMPlexGetSupportSize(dm, p, &size));
498     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
499     PetscCall(DMPlexGetSupport(dm, p, &points));
500     for (i = 0; i < size; ++i) {
501       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
502     }
503     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
504     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
505   }
506   PetscCall(PetscFree(newpoints));
507   PetscFunctionReturn(0);
508 }
509 
510 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
511 {
512   PetscSection   coordSection, newCoordSection;
513   Vec            coordinates, newCoordinates;
514   PetscScalar   *coords, *newCoords;
515   PetscInt       coordSize, sStart, sEnd;
516   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
517   PetscBool      hasCells;
518 
519   PetscFunctionBegin;
520   PetscCall(DMGetCoordinateDim(dm, &dim));
521   PetscCall(DMSetCoordinateDim(dmNew, dim));
522   PetscCall(DMPlexGetDepth(dm, &depth));
523   /* Step 8: Convert coordinates */
524   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
525   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
526   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
527   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
528   PetscCall(DMGetCoordinateSection(dm, &coordSection));
529   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
530   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
531   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
532   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
533   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
534   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
535   if (hasCells) {
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
538 
539       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
540       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
541       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
542     }
543   }
544   for (v = vStartNew; v < vEndNew; ++v) {
545     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
546     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
547   }
548   PetscCall(PetscSectionSetUp(newCoordSection));
549   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
550   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
551   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
552   PetscCall(PetscObjectSetName((PetscObject) newCoordinates, "coordinates"));
553   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
554   PetscCall(VecSetBlockSize(newCoordinates, dim));
555   PetscCall(VecSetType(newCoordinates,VECSTANDARD));
556   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
557   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
558   PetscCall(VecGetArray(coordinates, &coords));
559   PetscCall(VecGetArray(newCoordinates, &newCoords));
560   if (hasCells) {
561     for (c = cStart; c < cEnd; ++c) {
562       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
563 
564       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
565       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
566       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
567       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
568     }
569   }
570   for (v = vStart; v < vEnd; ++v) {
571     PetscInt dof, off, noff, d;
572 
573     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
574     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
575     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
576     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
577   }
578   PetscCall(VecRestoreArray(coordinates, &coords));
579   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
580   PetscCall(VecDestroy(&newCoordinates));
581   PetscCall(PetscSectionDestroy(&newCoordSection));
582   PetscFunctionReturn(0);
583 }
584 
585 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
586 {
587   const PetscSFNode *remotePoints;
588   PetscSFNode       *gremotePoints;
589   const PetscInt    *localPoints;
590   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
591   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
592 
593   PetscFunctionBegin;
594   PetscCall(DMPlexGetDepth(dm, &depth));
595   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
596   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
597   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
598   if (numRoots >= 0) {
599     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd-pStart, &newRemoteLocation));
600     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
601     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
602     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
603     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
604     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
605     for (l = 0; l < numLeaves; ++l) {
606       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
607       gremotePoints[l].rank  = remotePoints[l].rank;
608       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
609     }
610     PetscCall(PetscFree2(newLocation, newRemoteLocation));
611     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
612   }
613   PetscFunctionReturn(0);
614 }
615 
616 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
617 {
618   PetscSF        sfPoint, sfPointNew;
619   PetscBool      useNatural;
620 
621   PetscFunctionBegin;
622   /* Step 9: Convert pointSF */
623   PetscCall(DMGetPointSF(dm, &sfPoint));
624   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
625   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
626   /* Step 9b: Convert naturalSF */
627   PetscCall(DMGetUseNatural(dm, &useNatural));
628   if (useNatural) {
629     PetscSF sfNat, sfNatNew;
630 
631     PetscCall(DMSetUseNatural(dmNew, useNatural));
632     PetscCall(DMGetNaturalSF(dm, &sfNat));
633     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
634     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
635   }
636   PetscFunctionReturn(0);
637 }
638 
639 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
640 {
641   PetscInt           depth = 0, numLabels, l;
642 
643   PetscFunctionBegin;
644   PetscCall(DMPlexGetDepth(dm, &depth));
645   /* Step 10: Convert labels */
646   PetscCall(DMGetNumLabels(dm, &numLabels));
647   for (l = 0; l < numLabels; ++l) {
648     DMLabel         label, newlabel;
649     const char     *lname;
650     PetscBool       isDepth, isDim;
651     IS              valueIS;
652     const PetscInt *values;
653     PetscInt        numValues, val;
654 
655     PetscCall(DMGetLabelName(dm, l, &lname));
656     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
657     if (isDepth) continue;
658     PetscCall(PetscStrcmp(lname, "dim", &isDim));
659     if (isDim) continue;
660     PetscCall(DMCreateLabel(dmNew, lname));
661     PetscCall(DMGetLabel(dm, lname, &label));
662     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
663     PetscCall(DMLabelGetDefaultValue(label,&val));
664     PetscCall(DMLabelSetDefaultValue(newlabel,val));
665     PetscCall(DMLabelGetValueIS(label, &valueIS));
666     PetscCall(ISGetLocalSize(valueIS, &numValues));
667     PetscCall(ISGetIndices(valueIS, &values));
668     for (val = 0; val < numValues; ++val) {
669       IS              pointIS;
670       const PetscInt *points;
671       PetscInt        numPoints, p;
672 
673       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
674       PetscCall(ISGetLocalSize(pointIS, &numPoints));
675       PetscCall(ISGetIndices(pointIS, &points));
676       for (p = 0; p < numPoints; ++p) {
677         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
678 
679         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
680       }
681       PetscCall(ISRestoreIndices(pointIS, &points));
682       PetscCall(ISDestroy(&pointIS));
683     }
684     PetscCall(ISRestoreIndices(valueIS, &values));
685     PetscCall(ISDestroy(&valueIS));
686   }
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
691 {
692   PetscSF            sfPoint;
693   DMLabel            vtkLabel, ghostLabel = NULL;
694   const PetscSFNode *leafRemote;
695   const PetscInt    *leafLocal;
696   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
697   PetscMPIInt        rank;
698 
699   PetscFunctionBegin;
700    /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
702   PetscCall(DMGetPointSF(dm, &sfPoint));
703   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
704   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
705   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
706   PetscCall(DMCreateLabel(dmNew, "vtk"));
707   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
708   if (createGhostLabel) {
709     PetscCall(DMCreateLabel(dmNew, "ghost"));
710     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
711   }
712   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
713     for (; c < leafLocal[l] && c < cEnd; ++c) {
714       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
715     }
716     if (leafLocal[l] >= cEnd) break;
717     if (leafRemote[l].rank == rank) {
718       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
719     } else if (ghostLabel) {
720       PetscCall(DMLabelSetValue(ghostLabel, c, 2));
721     }
722   }
723   for (; c < cEnd; ++c) {
724     PetscCall(DMLabelSetValue(vtkLabel, c, 1));
725   }
726   if (ghostLabel) {
727     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
728     for (f = fStart; f < fEnd; ++f) {
729       PetscInt numCells;
730 
731       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
732       if (numCells < 2) {
733         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
734       } else {
735         const PetscInt *cells = NULL;
736         PetscInt        vA, vB;
737 
738         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
739         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
740         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
741         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
742       }
743     }
744   }
745   PetscFunctionReturn(0);
746 }
747 
748 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
749 {
750   DM             refTree;
751   PetscSection   pSec;
752   PetscInt       *parents, *childIDs;
753 
754   PetscFunctionBegin;
755   PetscCall(DMPlexGetReferenceTree(dm,&refTree));
756   PetscCall(DMPlexSetReferenceTree(dmNew,refTree));
757   PetscCall(DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL));
758   if (pSec) {
759     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
760     PetscInt *childIDsShifted;
761     PetscSection pSecShifted;
762 
763     PetscCall(PetscSectionGetChart(pSec,&pStart,&pEnd));
764     PetscCall(DMPlexGetDepth(dm,&depth));
765     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
766     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
767     PetscCall(PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted));
768     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted));
769     PetscCall(PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted));
770     for (p = pStartShifted; p < pEndShifted; p++) {
771       /* start off assuming no children */
772       PetscCall(PetscSectionSetDof(pSecShifted,p,0));
773     }
774     for (p = pStart; p < pEnd; p++) {
775       PetscInt dof;
776       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
777 
778       PetscCall(PetscSectionGetDof(pSec,p,&dof));
779       PetscCall(PetscSectionSetDof(pSecShifted,pNew,dof));
780     }
781     PetscCall(PetscSectionSetUp(pSecShifted));
782     for (p = pStart; p < pEnd; p++) {
783       PetscInt dof;
784       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
785 
786       PetscCall(PetscSectionGetDof(pSec,p,&dof));
787       if (dof) {
788         PetscInt off, offNew;
789 
790         PetscCall(PetscSectionGetOffset(pSec,p,&off));
791         PetscCall(PetscSectionGetOffset(pSecShifted,pNew,&offNew));
792         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
793         childIDsShifted[offNew] = childIDs[off];
794       }
795     }
796     PetscCall(DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted));
797     PetscCall(PetscFree2(parentsShifted,childIDsShifted));
798     PetscCall(PetscSectionDestroy(&pSecShifted));
799   }
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
804 {
805   PetscSF               sf;
806   IS                    valueIS;
807   const PetscInt       *values, *leaves;
808   PetscInt             *depthShift;
809   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
810   PetscBool             isper;
811   const PetscReal      *maxCell, *L;
812   const DMBoundaryType *bd;
813 
814   PetscFunctionBegin;
815   PetscCall(DMGetPointSF(dm, &sf));
816   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
817   nleaves = PetscMax(0, nleaves);
818   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
819   /* Count ghost cells */
820   PetscCall(DMLabelGetValueIS(label, &valueIS));
821   PetscCall(ISGetLocalSize(valueIS, &numFS));
822   PetscCall(ISGetIndices(valueIS, &values));
823   Ng   = 0;
824   for (fs = 0; fs < numFS; ++fs) {
825     IS              faceIS;
826     const PetscInt *faces;
827     PetscInt        numFaces, f, numBdFaces = 0;
828 
829     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
830     PetscCall(ISGetLocalSize(faceIS, &numFaces));
831     PetscCall(ISGetIndices(faceIS, &faces));
832     for (f = 0; f < numFaces; ++f) {
833       PetscInt numChildren;
834 
835       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
836       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
837       /* non-local and ancestors points don't get to register ghosts */
838       if (loc >= 0 || numChildren) continue;
839       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
840     }
841     Ng += numBdFaces;
842     PetscCall(ISRestoreIndices(faceIS, &faces));
843     PetscCall(ISDestroy(&faceIS));
844   }
845   PetscCall(DMPlexGetDepth(dm, &depth));
846   PetscCall(PetscMalloc1(2*(depth+1), &depthShift));
847   for (d = 0; d <= depth; d++) {
848     PetscInt dEnd;
849 
850     PetscCall(DMPlexGetDepthStratum(dm,d,NULL,&dEnd));
851     depthShift[2*d]   = dEnd;
852     depthShift[2*d+1] = 0;
853   }
854   if (depth >= 0) depthShift[2*depth+1] = Ng;
855   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
856   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
857   /* Step 3: Set cone/support sizes for new points */
858   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
859   for (c = cEnd; c < cEnd + Ng; ++c) {
860     PetscCall(DMPlexSetConeSize(gdm, c, 1));
861   }
862   for (fs = 0; fs < numFS; ++fs) {
863     IS              faceIS;
864     const PetscInt *faces;
865     PetscInt        numFaces, f;
866 
867     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
868     PetscCall(ISGetLocalSize(faceIS, &numFaces));
869     PetscCall(ISGetIndices(faceIS, &faces));
870     for (f = 0; f < numFaces; ++f) {
871       PetscInt size, numChildren;
872 
873       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
874       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
875       if (loc >= 0 || numChildren) continue;
876       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
877       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
878       PetscCheck(size == 1,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
879       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
880     }
881     PetscCall(ISRestoreIndices(faceIS, &faces));
882     PetscCall(ISDestroy(&faceIS));
883   }
884   /* Step 4: Setup ghosted DM */
885   PetscCall(DMSetUp(gdm));
886   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
887   /* Step 6: Set cones and supports for new points */
888   ghostCell = cEnd;
889   for (fs = 0; fs < numFS; ++fs) {
890     IS              faceIS;
891     const PetscInt *faces;
892     PetscInt        numFaces, f;
893 
894     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
895     PetscCall(ISGetLocalSize(faceIS, &numFaces));
896     PetscCall(ISGetIndices(faceIS, &faces));
897     for (f = 0; f < numFaces; ++f) {
898       PetscInt newFace = faces[f] + Ng, numChildren;
899 
900       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
901       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
902       if (loc >= 0 || numChildren) continue;
903       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
904       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
905       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
906       ++ghostCell;
907     }
908     PetscCall(ISRestoreIndices(faceIS, &faces));
909     PetscCall(ISDestroy(&faceIS));
910   }
911   PetscCall(ISRestoreIndices(valueIS, &values));
912   PetscCall(ISDestroy(&valueIS));
913   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
914   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
915   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
916   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
917   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
918   PetscCall(PetscFree(depthShift));
919   for (c = cEnd; c < cEnd + Ng; ++c) {
920     PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
921   }
922   /* Step 7: Periodicity */
923   PetscCall(DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd));
924   PetscCall(DMSetPeriodicity(gdm, isper, maxCell,  L,  bd));
925   if (numGhostCells) *numGhostCells = Ng;
926   PetscFunctionReturn(0);
927 }
928 
929 /*@C
930   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
931 
932   Collective on dm
933 
934   Input Parameters:
935 + dm - The original DM
936 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
937 
938   Output Parameters:
939 + numGhostCells - The number of ghost cells added to the DM
940 - dmGhosted - The new DM
941 
942   Note: If no label exists of that name, one will be created marking all boundary faces
943 
944   Level: developer
945 
946 .seealso: `DMCreate()`
947 @*/
948 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
949 {
950   DM             gdm;
951   DMLabel        label;
952   const char    *name = labelName ? labelName : "Face Sets";
953   PetscInt       dim, Ng = 0;
954   PetscBool      useCone, useClosure;
955 
956   PetscFunctionBegin;
957   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
958   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
959   PetscValidPointer(dmGhosted, 4);
960   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
961   PetscCall(DMSetType(gdm, DMPLEX));
962   PetscCall(DMGetDimension(dm, &dim));
963   PetscCall(DMSetDimension(gdm, dim));
964   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
965   PetscCall(DMSetBasicAdjacency(gdm, useCone,  useClosure));
966   PetscCall(DMGetLabel(dm, name, &label));
967   if (!label) {
968     /* Get label for boundary faces */
969     PetscCall(DMCreateLabel(dm, name));
970     PetscCall(DMGetLabel(dm, name, &label));
971     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
972   }
973   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
974   PetscCall(DMCopyDisc(dm, gdm));
975   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
976   gdm->setfromoptionscalled = dm->setfromoptionscalled;
977   if (numGhostCells) *numGhostCells = Ng;
978   *dmGhosted = gdm;
979   PetscFunctionReturn(0);
980 }
981 
982 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
983 {
984   PetscInt dim, d, shift = 100, *pStart, *pEnd;
985 
986   PetscFunctionBegin;
987   PetscCall(DMGetDimension(dm, &dim));
988   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
989   for (d = 0; d < dim; ++d) {
990     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
991   }
992   while (!DMPlexPointQueueEmpty(queue)) {
993     PetscInt  cell    = -1;
994     PetscInt *closure = NULL;
995     PetscInt  closureSize, cl, cval;
996 
997     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
998     PetscCall(DMLabelGetValue(label, cell, &cval));
999     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1000     /* Mark points in the cell closure that touch the fault */
1001     for (d = 0; d < dim; ++d) {
1002       for (cl = 0; cl < closureSize*2; cl += 2) {
1003         const PetscInt clp = closure[cl];
1004         PetscInt       clval;
1005 
1006         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1007         PetscCall(DMLabelGetValue(label, clp, &clval));
1008         if (clval == -1) {
1009           const PetscInt *cone;
1010           PetscInt        coneSize, c;
1011 
1012           /* If a cone point touches the fault, then this point touches the fault */
1013           PetscCall(DMPlexGetCone(dm, clp, &cone));
1014           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1015           for (c = 0; c < coneSize; ++c) {
1016             PetscInt cpval;
1017 
1018             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1019             if (cpval != -1) {
1020               PetscInt dep;
1021 
1022               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1023               clval = cval < 0 ? -(shift+dep) : shift+dep;
1024               PetscCall(DMLabelSetValue(label, clp, clval));
1025               break;
1026             }
1027           }
1028         }
1029         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1030         if (d == dim-1 && clval != -1) {
1031           const PetscInt *support;
1032           PetscInt        supportSize, s, nval;
1033 
1034           PetscCall(DMPlexGetSupport(dm, clp, &support));
1035           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1036           for (s = 0; s < supportSize; ++s) {
1037             PetscCall(DMLabelGetValue(label, support[s], &nval));
1038             if (nval == -1) {
1039               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval-1 : clval+1));
1040               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1041             }
1042           }
1043         }
1044       }
1045     }
1046     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1047   }
1048   PetscCall(PetscFree2(pStart, pEnd));
1049   PetscFunctionReturn(0);
1050 }
1051 
1052 typedef struct {
1053   DM               dm;
1054   DMPlexPointQueue queue;
1055 } PointDivision;
1056 
1057 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1058 {
1059   PointDivision  *div  = (PointDivision *) ctx;
1060   PetscInt        cval = val < 0 ? val-1 : val+1;
1061   const PetscInt *support;
1062   PetscInt        supportSize, s;
1063 
1064   PetscFunctionBegin;
1065   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1066   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1067   for (s = 0; s < supportSize; ++s) {
1068     PetscCall(DMLabelSetValue(label, support[s], cval));
1069     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1070   }
1071   PetscFunctionReturn(0);
1072 }
1073 
1074 /* Mark cells by label propagation */
1075 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1076 {
1077   DMPlexPointQueue queue = NULL;
1078   PointDivision    div;
1079   PetscSF          pointSF;
1080   IS               pointIS;
1081   const PetscInt  *points;
1082   PetscBool        empty;
1083   PetscInt         dim, shift = 100, n, i;
1084 
1085   PetscFunctionBegin;
1086   PetscCall(DMGetDimension(dm, &dim));
1087   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1088   div.dm    = dm;
1089   div.queue = queue;
1090   /* Enqueue cells on fault */
1091   PetscCall(DMLabelGetStratumIS(faultLabel, shift+dim, &pointIS));
1092   if (pointIS) {
1093     PetscCall(ISGetLocalSize(pointIS, &n));
1094     PetscCall(ISGetIndices(pointIS, &points));
1095     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1096     PetscCall(ISRestoreIndices(pointIS, &points));
1097     PetscCall(ISDestroy(&pointIS));
1098   }
1099   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift+dim), &pointIS));
1100   if (pointIS) {
1101     PetscCall(ISGetLocalSize(pointIS, &n));
1102     PetscCall(ISGetIndices(pointIS, &points));
1103     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1104     PetscCall(ISRestoreIndices(pointIS, &points));
1105     PetscCall(ISDestroy(&pointIS));
1106   }
1107 
1108   PetscCall(DMGetPointSF(dm, &pointSF));
1109   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1110   /* While edge queue is not empty: */
1111   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1112   while (!empty) {
1113     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1114     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1115     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1116   }
1117   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1118   PetscCall(DMPlexPointQueueDestroy(&queue));
1119   PetscFunctionReturn(0);
1120 }
1121 
1122 /*
1123   We are adding three kinds of points here:
1124     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1125     Non-replicated: Points which exist on the fault, but are not replicated
1126     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1127     Hybrid:         Entirely new points, such as cohesive cells
1128 
1129   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1130   each depth so that the new split/hybrid points can be inserted as a block.
1131 */
1132 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1133 {
1134   MPI_Comm         comm;
1135   IS               valueIS;
1136   PetscInt         numSP = 0;          /* The number of depths for which we have replicated points */
1137   const PetscInt  *values;             /* List of depths for which we have replicated points */
1138   IS              *splitIS;
1139   IS              *unsplitIS;
1140   IS               ghostIS;
1141   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1142   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1143   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1144   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1145   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1146   const PetscInt **splitPoints;        /* Replicated points for each depth */
1147   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1148   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1149   PetscSection     coordSection;
1150   Vec              coordinates;
1151   PetscScalar     *coords;
1152   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
1153   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
1154   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
1155   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
1156   PetscInt        *coneNew, *coneONew, *supportNew;
1157   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1158 
1159   PetscFunctionBegin;
1160   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
1161   PetscCall(DMGetDimension(dm, &dim));
1162   PetscCall(DMPlexGetDepth(dm, &depth));
1163   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1164   /* We do not want this label automatically computed, instead we compute it here */
1165   PetscCall(DMCreateLabel(sdm, "celltype"));
1166   /* Count split points and add cohesive cells */
1167   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1168   PetscCall(PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld));
1169   PetscCall(PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints));
1170   for (d = 0; d <= depth; ++d) {
1171     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1172     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1173     depthEnd[d]           = pMaxNew[d];
1174     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1175     numSplitPoints[d]     = 0;
1176     numUnsplitPoints[d]   = 0;
1177     numHybridPoints[d]    = 0;
1178     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1179     splitPoints[d]        = NULL;
1180     unsplitPoints[d]      = NULL;
1181     splitIS[d]            = NULL;
1182     unsplitIS[d]          = NULL;
1183     /* we are shifting the existing hybrid points with the stratum behind them, so
1184      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1185     depthShift[2*d]       = depthMax[d];
1186     depthShift[2*d+1]     = 0;
1187   }
1188   numGhostPoints = 0;
1189   ghostPoints    = NULL;
1190   if (label) {
1191     PetscCall(DMLabelGetValueIS(label, &valueIS));
1192     PetscCall(ISGetLocalSize(valueIS, &numSP));
1193     PetscCall(ISGetIndices(valueIS, &values));
1194   }
1195   for (sp = 0; sp < numSP; ++sp) {
1196     const PetscInt dep = values[sp];
1197 
1198     if ((dep < 0) || (dep > depth)) continue;
1199     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1200     if (splitIS[dep]) {
1201       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1202       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1203     }
1204     PetscCall(DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]));
1205     if (unsplitIS[dep]) {
1206       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1207       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1208     }
1209   }
1210   PetscCall(DMLabelGetStratumIS(label, shift2+dim-1, &ghostIS));
1211   if (ghostIS) {
1212     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1213     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1214   }
1215   /* Calculate number of hybrid points */
1216   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1217   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1218   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
1219   /* the end of the points in this stratum that come before the new points:
1220    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1221    * added points */
1222   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1223   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1224   /* Step 3: Set cone/support sizes for new points */
1225   for (dep = 0; dep <= depth; ++dep) {
1226     for (p = 0; p < numSplitPoints[dep]; ++p) {
1227       const PetscInt  oldp   = splitPoints[dep][p];
1228       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1229       const PetscInt  splitp = p    + pMaxNew[dep];
1230       const PetscInt *support;
1231       DMPolytopeType  ct;
1232       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1233 
1234       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1235       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1236       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1237       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1238       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1239       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1240       if (dep == depth-1) {
1241         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1242 
1243         /* Add cohesive cells, they are prisms */
1244         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1245         switch (coneSize) {
1246           case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));break;
1247           case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));break;
1248           case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));break;
1249         }
1250         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1251         /*   TODO Check thaat oldp has rootdegree == 1 */
1252         if (supportSize == 1) {
1253           const PetscInt *support;
1254           PetscInt        val;
1255 
1256           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1257           PetscCall(DMLabelGetValue(label, support[0], &val));
1258           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1259           else         PetscCall(DMPlexSetSupportSize(sdm, newp,   2));
1260         }
1261       } else if (dep == 0) {
1262         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1263 
1264         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1265         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1266           PetscInt val;
1267 
1268           PetscCall(DMLabelGetValue(label, support[e], &val));
1269           if (val == 1) ++qf;
1270           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1271           if ((val == 1) || (val == -(shift + 1))) ++qp;
1272         }
1273         /* Split old vertex: Edges into original vertex and new cohesive edge */
1274         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1275         /* Split new vertex: Edges into split vertex and new cohesive edge */
1276         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1277         /* Add hybrid edge */
1278         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1279         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1280         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1281       } else if (dep == dim-2) {
1282         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1283 
1284         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1285         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1286           PetscInt val;
1287 
1288           PetscCall(DMLabelGetValue(label, support[e], &val));
1289           if (val == dim-1) ++qf;
1290           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1291           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1292         }
1293         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1294         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1295         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1296         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1297         /* Add hybrid face */
1298         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1299         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1300         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1301       }
1302     }
1303   }
1304   for (dep = 0; dep <= depth; ++dep) {
1305     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1306       const PetscInt  oldp   = unsplitPoints[dep][p];
1307       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1308       const PetscInt *support;
1309       PetscInt        coneSize, supportSize, qf, e, s;
1310 
1311       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1312       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1313       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1314       if (dep == 0) {
1315         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1316 
1317         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1318         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1319           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1320           if (e >= 0) ++qf;
1321         }
1322         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1323         /* Add hybrid edge */
1324         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1325         for (e = 0, qf = 0; e < supportSize; ++e) {
1326           PetscInt val;
1327 
1328           PetscCall(DMLabelGetValue(label, support[e], &val));
1329           /* Split and unsplit edges produce hybrid faces */
1330           if (val == 1) ++qf;
1331           if (val == (shift2 + 1)) ++qf;
1332         }
1333         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1334         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1335       } else if (dep == dim-2) {
1336         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1337         PetscInt       val;
1338 
1339         for (e = 0, qf = 0; e < supportSize; ++e) {
1340           PetscCall(DMLabelGetValue(label, support[e], &val));
1341           if (val == dim-1) qf += 2;
1342           else              ++qf;
1343         }
1344         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1345         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1346         /* Add hybrid face */
1347         for (e = 0, qf = 0; e < supportSize; ++e) {
1348           PetscCall(DMLabelGetValue(label, support[e], &val));
1349           if (val == dim-1) ++qf;
1350         }
1351         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1352         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1353         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1354       }
1355     }
1356   }
1357   /* Step 4: Setup split DM */
1358   PetscCall(DMSetUp(sdm));
1359   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1360   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1361   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew));
1362   /* Step 6: Set cones and supports for new points */
1363   for (dep = 0; dep <= depth; ++dep) {
1364     for (p = 0; p < numSplitPoints[dep]; ++p) {
1365       const PetscInt  oldp   = splitPoints[dep][p];
1366       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1367       const PetscInt  splitp = p    + pMaxNew[dep];
1368       const PetscInt *cone, *support, *ornt;
1369       DMPolytopeType  ct;
1370       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1371 
1372       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1373       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1374       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1375       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1376       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1377       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1378       if (dep == depth-1) {
1379         PetscBool       hasUnsplit = PETSC_FALSE;
1380         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1381         const PetscInt *supportF;
1382 
1383         coneONew[0] = coneONew[1] = -1000;
1384         /* Split face:       copy in old face to new face to start */
1385         PetscCall(DMPlexGetSupport(sdm, newp,  &supportF));
1386         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1387         /* Split old face:   old vertices/edges in cone so no change */
1388         /* Split new face:   new vertices/edges in cone */
1389         for (q = 0; q < coneSize; ++q) {
1390           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1391           if (v < 0) {
1392             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1393             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1394             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1395             hasUnsplit   = PETSC_TRUE;
1396           } else {
1397             coneNew[2+q] = v + pMaxNew[dep-1];
1398             if (dep > 1) {
1399               const PetscInt *econe;
1400               PetscInt        econeSize, r, vs, vu;
1401 
1402               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1403               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1404               for (r = 0; r < econeSize; ++r) {
1405                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs));
1406                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu));
1407                 if (vs >= 0) continue;
1408                 PetscCheck(vu >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep-2);
1409                 hasUnsplit   = PETSC_TRUE;
1410               }
1411             }
1412           }
1413         }
1414         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1415         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1416         /* Face support */
1417         PetscInt vals[2];
1418 
1419         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1420         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1421         else                 vals[1] = -vals[0];
1422         PetscCheck(vals[0]*vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1423 
1424         for (s = 0; s < 2; ++s) {
1425           if (s >= supportSize) {
1426             if (vals[s] < 0) {
1427               /* Ghost old face:   Replace negative side cell with cohesive cell */
1428               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1429             } else {
1430               /* Ghost new face:   Replace positive side cell with cohesive cell */
1431               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1432             }
1433           } else {
1434             if (vals[s] < 0) {
1435               /* Split old face:   Replace negative side cell with cohesive cell */
1436               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1437             } else {
1438               /* Split new face:   Replace positive side cell with cohesive cell */
1439               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1440             }
1441           }
1442         }
1443         /* Get orientation for cohesive face using the positive side cell */
1444         {
1445           const PetscInt *ncone, *nconeO;
1446           PetscInt        nconeSize, nc, ocell;
1447           PetscBool       flip = PETSC_FALSE;
1448 
1449           if (supportSize > 1) {ocell = vals[0] < 0 ? support[1] : support[0];}
1450           else                 {ocell = support[0]; flip = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;}
1451           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1452           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1453           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1454           for (nc = 0; nc < nconeSize; ++nc) {
1455             if (ncone[nc] == oldp) {
1456               coneONew[0] = flip ? -(nconeO[nc]+1) : nconeO[nc];
1457               break;
1458             }
1459           }
1460           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1461         }
1462         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1463         {
1464           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct)/2;
1465           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1466         }
1467         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1468 
1469         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1470         coneNew[1]  = splitp;
1471         coneONew[1] = coneONew[0];
1472         for (q = 0; q < coneSize; ++q) {
1473           /* Hybrid faces must follow order from oriented end face */
1474           const PetscInt qa = arr[q*2+0];
1475           const PetscInt qo = arr[q*2+1];
1476           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1477 
1478           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1479           if (v < 0) {
1480             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1481             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1482           } else {
1483             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1484           }
1485           coneONew[2+q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1486         }
1487         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1488         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1489         /* Label the hybrid cells on the boundary of the split */
1490         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1491       } else if (dep == 0) {
1492         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1493 
1494         /* Split old vertex: Edges in old split faces and new cohesive edge */
1495         for (e = 0, qn = 0; e < supportSize; ++e) {
1496           PetscInt val;
1497 
1498           PetscCall(DMLabelGetValue(label, support[e], &val));
1499           if ((val == 1) || (val == (shift + 1))) {
1500             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1501           }
1502         }
1503         supportNew[qn] = hybedge;
1504         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1505         /* Split new vertex: Edges in new split faces and new cohesive edge */
1506         for (e = 0, qp = 0; e < supportSize; ++e) {
1507           PetscInt val, edge;
1508 
1509           PetscCall(DMLabelGetValue(label, support[e], &val));
1510           if (val == 1) {
1511             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1512             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1513             supportNew[qp++] = edge + pMaxNew[dep+1];
1514           } else if (val == -(shift + 1)) {
1515             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1516           }
1517         }
1518         supportNew[qp] = hybedge;
1519         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1520         /* Hybrid edge:    Old and new split vertex */
1521         coneNew[0] = newp;
1522         coneNew[1] = splitp;
1523         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1524         for (e = 0, qf = 0; e < supportSize; ++e) {
1525           PetscInt val, edge;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1530             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1531             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1532           }
1533         }
1534         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1535       } else if (dep == dim-2) {
1536         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1537 
1538         /* Split old edge:   old vertices in cone so no change */
1539         /* Split new edge:   new vertices in cone */
1540         for (q = 0; q < coneSize; ++q) {
1541           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1542           if (v < 0) {
1543             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1544             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1545             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1546           } else {
1547             coneNew[q] = v + pMaxNew[dep-1];
1548           }
1549         }
1550         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1551         /* Split old edge: Faces in positive side cells and old split faces */
1552         for (e = 0, q = 0; e < supportSize; ++e) {
1553           PetscInt val;
1554 
1555           PetscCall(DMLabelGetValue(label, support[e], &val));
1556           if (val == dim-1) {
1557             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1558           } else if (val == (shift + dim-1)) {
1559             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1560           }
1561         }
1562         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1563         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1564         /* Split new edge: Faces in negative side cells and new split faces */
1565         for (e = 0, q = 0; e < supportSize; ++e) {
1566           PetscInt val, face;
1567 
1568           PetscCall(DMLabelGetValue(label, support[e], &val));
1569           if (val == dim-1) {
1570             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1571             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1572             supportNew[q++] = face + pMaxNew[dep+1];
1573           } else if (val == -(shift + dim-1)) {
1574             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1575           }
1576         }
1577         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1578         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1579         /* Hybrid face */
1580         coneNew[0] = newp;
1581         coneNew[1] = splitp;
1582         for (v = 0; v < coneSize; ++v) {
1583           PetscInt vertex;
1584           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex));
1585           if (vertex < 0) {
1586             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex));
1587             PetscCheck(vertex >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep-1);
1588             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1589           } else {
1590             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1591           }
1592         }
1593         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1594         for (e = 0, qf = 0; e < supportSize; ++e) {
1595           PetscInt val, face;
1596 
1597           PetscCall(DMLabelGetValue(label, support[e], &val));
1598           if (val == dim-1) {
1599             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1600             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1601             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1602           }
1603         }
1604         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1605       }
1606     }
1607   }
1608   for (dep = 0; dep <= depth; ++dep) {
1609     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1610       const PetscInt  oldp   = unsplitPoints[dep][p];
1611       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1612       const PetscInt *cone, *support;
1613       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1614 
1615       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1616       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1617       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1618       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1619       if (dep == 0) {
1620         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1621 
1622         /* Unsplit vertex */
1623         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1624         for (s = 0, q = 0; s < supportSize; ++s) {
1625           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1626           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1627           if (e >= 0) {
1628             supportNew[q++] = e + pMaxNew[dep+1];
1629           }
1630         }
1631         supportNew[q++] = hybedge;
1632         supportNew[q++] = hybedge;
1633         PetscCheck(q == supportSizeNew,comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1634         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1635         /* Hybrid edge */
1636         coneNew[0] = newp;
1637         coneNew[1] = newp;
1638         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1639         for (e = 0, qf = 0; e < supportSize; ++e) {
1640           PetscInt val, edge;
1641 
1642           PetscCall(DMLabelGetValue(label, support[e], &val));
1643           if (val == 1) {
1644             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1645             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1646             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1647           } else if  (val ==  (shift2 + 1)) {
1648             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge));
1649             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1650             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1651           }
1652         }
1653         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1654       } else if (dep == dim-2) {
1655         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1656 
1657         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1658         for (f = 0, qf = 0; f < supportSize; ++f) {
1659           PetscInt val, face;
1660 
1661           PetscCall(DMLabelGetValue(label, support[f], &val));
1662           if (val == dim-1) {
1663             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1664             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1665             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1666             supportNew[qf++] = face + pMaxNew[dep+1];
1667           } else {
1668             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1669           }
1670         }
1671         supportNew[qf++] = hybface;
1672         supportNew[qf++] = hybface;
1673         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1674         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1675         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1676         /* Add hybrid face */
1677         coneNew[0] = newp;
1678         coneNew[1] = newp;
1679         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1680         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1681         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1682         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1683         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1684         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1685         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1686         for (f = 0, qf = 0; f < supportSize; ++f) {
1687           PetscInt val, face;
1688 
1689           PetscCall(DMLabelGetValue(label, support[f], &val));
1690           if (val == dim-1) {
1691             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1692             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1693           }
1694         }
1695         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1696         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1697         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1698       }
1699     }
1700   }
1701   /* Step 6b: Replace split points in negative side cones */
1702   for (sp = 0; sp < numSP; ++sp) {
1703     PetscInt        dep = values[sp];
1704     IS              pIS;
1705     PetscInt        numPoints;
1706     const PetscInt *points;
1707 
1708     if (dep >= 0) continue;
1709     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1710     if (!pIS) continue;
1711     dep  = -dep - shift;
1712     PetscCall(ISGetLocalSize(pIS, &numPoints));
1713     PetscCall(ISGetIndices(pIS, &points));
1714     for (p = 0; p < numPoints; ++p) {
1715       const PetscInt  oldp = points[p];
1716       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1717       const PetscInt *cone;
1718       PetscInt        coneSize, c;
1719       /* PetscBool       replaced = PETSC_FALSE; */
1720 
1721       /* Negative edge: replace split vertex */
1722       /* Negative cell: replace split face */
1723       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1724       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1725       for (c = 0; c < coneSize; ++c) {
1726         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1727         PetscInt       csplitp, cp, val;
1728 
1729         PetscCall(DMLabelGetValue(label, coldp, &val));
1730         if (val == dep-1) {
1731           PetscCall(PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp));
1732           PetscCheck(cp >= 0,comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep-1);
1733           csplitp  = pMaxNew[dep-1] + cp;
1734           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1735           /* replaced = PETSC_TRUE; */
1736         }
1737       }
1738       /* Cells with only a vertex or edge on the submesh have no replacement */
1739       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1740     }
1741     PetscCall(ISRestoreIndices(pIS, &points));
1742     PetscCall(ISDestroy(&pIS));
1743   }
1744   /* Step 7: Coordinates */
1745   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1746   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1747   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1748   PetscCall(VecGetArray(coordinates, &coords));
1749   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1750     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1751     const PetscInt splitp = pMaxNew[0] + v;
1752     PetscInt       dof, off, soff, d;
1753 
1754     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1755     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1756     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1757     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1758   }
1759   PetscCall(VecRestoreArray(coordinates, &coords));
1760   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1761   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1762   /*   TODO We need to associate the ghost points with the correct replica */
1763   /* Step 9: Labels */
1764   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1765   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1766   PetscCall(DMGetNumLabels(sdm, &numLabels));
1767   for (dep = 0; dep <= depth; ++dep) {
1768     for (p = 0; p < numSplitPoints[dep]; ++p) {
1769       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1770       const PetscInt splitp = pMaxNew[dep] + p;
1771       PetscInt       l;
1772 
1773       if (splitLabel) {
1774         const PetscInt val = 100 + dep;
1775 
1776         PetscCall(DMLabelSetValue(splitLabel, newp,    val));
1777         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1778       }
1779       for (l = 0; l < numLabels; ++l) {
1780         DMLabel     mlabel;
1781         const char *lname;
1782         PetscInt    val;
1783         PetscBool   isDepth;
1784 
1785         PetscCall(DMGetLabelName(sdm, l, &lname));
1786         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1787         if (isDepth) continue;
1788         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1789         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1790         if (val >= 0) {
1791           PetscCall(DMLabelSetValue(mlabel, splitp, val));
1792         }
1793       }
1794     }
1795   }
1796   for (sp = 0; sp < numSP; ++sp) {
1797     const PetscInt dep = values[sp];
1798 
1799     if ((dep < 0) || (dep > depth)) continue;
1800     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1801     PetscCall(ISDestroy(&splitIS[dep]));
1802     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1803     PetscCall(ISDestroy(&unsplitIS[dep]));
1804   }
1805   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1806   PetscCall(ISDestroy(&ghostIS));
1807   if (label) {
1808     PetscCall(ISRestoreIndices(valueIS, &values));
1809     PetscCall(ISDestroy(&valueIS));
1810   }
1811   for (d = 0; d <= depth; ++d) {
1812     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1813     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1814   }
1815   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1816   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1817   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1818   PetscFunctionReturn(0);
1819 }
1820 
1821 /*@C
1822   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1823 
1824   Collective on dm
1825 
1826   Input Parameters:
1827 + dm - The original DM
1828 - label - The label specifying the boundary faces (this could be auto-generated)
1829 
1830   Output Parameters:
1831 + splitLabel - The label containing the split points, or NULL if no output is desired
1832 - dmSplit - The new DM
1833 
1834   Level: developer
1835 
1836 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1837 @*/
1838 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1839 {
1840   DM             sdm;
1841   PetscInt       dim;
1842 
1843   PetscFunctionBegin;
1844   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1845   PetscValidPointer(dmSplit, 4);
1846   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1847   PetscCall(DMSetType(sdm, DMPLEX));
1848   PetscCall(DMGetDimension(dm, &dim));
1849   PetscCall(DMSetDimension(sdm, dim));
1850   switch (dim) {
1851   case 2:
1852   case 3:
1853     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1854     break;
1855   default:
1856     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1857   }
1858   *dmSplit = sdm;
1859   PetscFunctionReturn(0);
1860 }
1861 
1862 /* Returns the side of the surface for a given cell with a face on the surface */
1863 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1864 {
1865   const PetscInt *cone, *ornt;
1866   PetscInt        dim, coneSize, c;
1867 
1868   PetscFunctionBegin;
1869   *pos = PETSC_TRUE;
1870   PetscCall(DMGetDimension(dm, &dim));
1871   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1872   PetscCall(DMPlexGetCone(dm, cell, &cone));
1873   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1874   for (c = 0; c < coneSize; ++c) {
1875     if (cone[c] == face) {
1876       PetscInt o = ornt[c];
1877 
1878       if (subdm) {
1879         const PetscInt *subcone, *subornt;
1880         PetscInt        subpoint, subface, subconeSize, sc;
1881 
1882         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1883         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1884         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1885         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1886         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1887         for (sc = 0; sc < subconeSize; ++sc) {
1888           if (subcone[sc] == subface) {
1889             o = subornt[0];
1890             break;
1891           }
1892         }
1893         PetscCheck(sc < subconeSize,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1894       }
1895       if (o >= 0) *pos = PETSC_TRUE;
1896       else        *pos = PETSC_FALSE;
1897       break;
1898     }
1899   }
1900   PetscCheck(c != coneSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1901   PetscFunctionReturn(0);
1902 }
1903 
1904 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1905 {
1906   IS              facePosIS, faceNegIS, dimIS;
1907   const PetscInt *points;
1908   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1909 
1910   PetscFunctionBegin;
1911   PetscCall(DMGetDimension(dm, &dim));
1912   /* If any faces touching the fault divide cells on either side, split them */
1913   PetscCall(DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS));
1914   PetscCall(DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS));
1915   if (!facePosIS || !faceNegIS) {
1916     PetscCall(ISDestroy(&facePosIS));
1917     PetscCall(ISDestroy(&faceNegIS));
1918     PetscFunctionReturn(0);
1919   }
1920   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1921   PetscCall(ISDestroy(&facePosIS));
1922   PetscCall(ISDestroy(&faceNegIS));
1923   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1924   PetscCall(ISGetIndices(dimIS, &points));
1925   for (p = 0; p < numPoints; ++p) {
1926     const PetscInt  point = points[p];
1927     const PetscInt *support;
1928     PetscInt        supportSize, valA, valB;
1929 
1930     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1931     if (supportSize != 2) continue;
1932     PetscCall(DMPlexGetSupport(dm, point, &support));
1933     PetscCall(DMLabelGetValue(label, support[0], &valA));
1934     PetscCall(DMLabelGetValue(label, support[1], &valB));
1935     if ((valA == -1) || (valB == -1)) continue;
1936     if (valA*valB > 0) continue;
1937     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1938     {
1939       PetscInt *closure = NULL;
1940       PetscBool split   = PETSC_FALSE;
1941       PetscInt  closureSize, cl;
1942 
1943       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1944       for (cl = 0; cl < closureSize*2; cl += 2) {
1945         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1946         if ((valA >= 0) && (valA <= dim)) {split = PETSC_TRUE; break;}
1947       }
1948       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1949       if (!split) continue;
1950     }
1951     /* Split the face */
1952     PetscCall(DMLabelGetValue(label, point, &valA));
1953     PetscCall(DMLabelClearValue(label, point, valA));
1954     PetscCall(DMLabelSetValue(label, point, dim-1));
1955     /* Label its closure:
1956       unmarked: label as unsplit
1957       incident: relabel as split
1958       split:    do nothing
1959     */
1960     {
1961       PetscInt *closure = NULL;
1962       PetscInt  closureSize, cl, dep;
1963 
1964       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1965       for (cl = 0; cl < closureSize*2; cl += 2) {
1966         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1967         if (valA == -1) { /* Mark as unsplit */
1968           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1969           PetscCall(DMLabelSetValue(label, closure[cl], shift2+dep));
1970         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1971           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1972           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1973           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1974         }
1975       }
1976       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1977     }
1978   }
1979   PetscCall(ISRestoreIndices(dimIS, &points));
1980   PetscCall(ISDestroy(&dimIS));
1981   PetscFunctionReturn(0);
1982 }
1983 
1984 /*@
1985   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1986   to complete the surface
1987 
1988   Input Parameters:
1989 + dm     - The DM
1990 . label  - A DMLabel marking the surface
1991 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1992 . bvalue - Value of DMLabel marking the vertices on the boundary
1993 . flip   - Flag to flip the submesh normal and replace points on the other side
1994 - subdm  - The subDM associated with the label, or NULL
1995 
1996   Output Parameter:
1997 . label - A DMLabel marking all surface points
1998 
1999   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2000 
2001   Level: developer
2002 
2003 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2004 @*/
2005 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2006 {
2007   DMLabel         depthLabel;
2008   IS              dimIS, subpointIS = NULL;
2009   const PetscInt *points, *subpoints;
2010   const PetscInt  rev   = flip ? -1 : 1;
2011   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2012 
2013   PetscFunctionBegin;
2014   PetscCall(DMPlexGetDepth(dm, &depth));
2015   PetscCall(DMGetDimension(dm, &dim));
2016   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2017   if (subdm) {
2018     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2019     if (subpointIS) {
2020       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2021       PetscCall(ISGetIndices(subpointIS, &subpoints));
2022     }
2023   }
2024   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2025   PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2026   if (!dimIS) goto divide;
2027   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2028   PetscCall(ISGetIndices(dimIS, &points));
2029   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2030     const PetscInt *support;
2031     PetscInt        supportSize, s;
2032 
2033     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2034 #if 0
2035     if (supportSize != 2) {
2036       const PetscInt *lp;
2037       PetscInt        Nlp, pind;
2038 
2039       /* Check that for a cell with a single support face, that face is in the SF */
2040       /*   THis check only works for the remote side. We would need root side information */
2041       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2042       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2043       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2044     }
2045 #endif
2046     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2047     for (s = 0; s < supportSize; ++s) {
2048       const PetscInt *cone;
2049       PetscInt        coneSize, c;
2050       PetscBool       pos;
2051 
2052       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2053       if (pos) PetscCall(DMLabelSetValue(label, support[s],  rev*(shift+dim)));
2054       else     PetscCall(DMLabelSetValue(label, support[s], -rev*(shift+dim)));
2055       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2056       /* Put faces touching the fault in the label */
2057       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2058       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2059       for (c = 0; c < coneSize; ++c) {
2060         const PetscInt point = cone[c];
2061 
2062         PetscCall(DMLabelGetValue(label, point, &val));
2063         if (val == -1) {
2064           PetscInt *closure = NULL;
2065           PetscInt  closureSize, cl;
2066 
2067           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2068           for (cl = 0; cl < closureSize*2; cl += 2) {
2069             const PetscInt clp  = closure[cl];
2070             PetscInt       bval = -1;
2071 
2072             PetscCall(DMLabelGetValue(label, clp, &val));
2073             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2074             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
2075               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1)));
2076               break;
2077             }
2078           }
2079           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2080         }
2081       }
2082     }
2083   }
2084   PetscCall(ISRestoreIndices(dimIS, &points));
2085   PetscCall(ISDestroy(&dimIS));
2086   /* Mark boundary points as unsplit */
2087   if (blabel) {
2088     IS bdIS;
2089 
2090     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2091     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2092     PetscCall(ISGetIndices(bdIS, &points));
2093     for (p = 0; p < numPoints; ++p) {
2094       const PetscInt point = points[p];
2095       PetscInt       val, bval;
2096 
2097       PetscCall(DMLabelGetValue(blabel, point, &bval));
2098       if (bval >= 0) {
2099         PetscCall(DMLabelGetValue(label, point, &val));
2100         if ((val < 0) || (val > dim)) {
2101           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2102           PetscCall(DMLabelClearValue(blabel, point, bval));
2103         }
2104       }
2105     }
2106     for (p = 0; p < numPoints; ++p) {
2107       const PetscInt point = points[p];
2108       PetscInt       val, bval;
2109 
2110       PetscCall(DMLabelGetValue(blabel, point, &bval));
2111       if (bval >= 0) {
2112         const PetscInt *cone,    *support;
2113         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2114 
2115         /* Mark as unsplit */
2116         PetscCall(DMLabelGetValue(label, point, &val));
2117         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2118         PetscCall(DMLabelClearValue(label, point, val));
2119         PetscCall(DMLabelSetValue(label, point, shift2+val));
2120         /* Check for cross-edge
2121              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2122         if (val != 0) continue;
2123         PetscCall(DMPlexGetSupport(dm, point, &support));
2124         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2125         for (s = 0; s < supportSize; ++s) {
2126           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2127           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2128           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2129           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2130           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2131           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2132           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2133         }
2134       }
2135     }
2136     PetscCall(ISRestoreIndices(bdIS, &points));
2137     PetscCall(ISDestroy(&bdIS));
2138   }
2139   /* Mark ghost fault cells */
2140   {
2141     PetscSF         sf;
2142     const PetscInt *leaves;
2143     PetscInt         Nl, l;
2144 
2145     PetscCall(DMGetPointSF(dm, &sf));
2146     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2147     PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2148     if (!dimIS) goto divide;
2149     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2150     PetscCall(ISGetIndices(dimIS, &points));
2151     if (Nl > 0) {
2152       for (p = 0; p < numPoints; ++p) {
2153         const PetscInt point = points[p];
2154         PetscInt       val;
2155 
2156         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2157         if (l >= 0) {
2158           PetscInt *closure = NULL;
2159           PetscInt  closureSize, cl;
2160 
2161           PetscCall(DMLabelGetValue(label, point, &val));
2162           PetscCheck((val == dim-1) || (val == shift2+dim-1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2163           PetscCall(DMLabelClearValue(label, point, val));
2164           PetscCall(DMLabelSetValue(label, point, shift3+val));
2165           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2166           for (cl = 2; cl < closureSize*2; cl += 2) {
2167             const PetscInt clp  = closure[cl];
2168 
2169             PetscCall(DMLabelGetValue(label, clp, &val));
2170             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2171             PetscCall(DMLabelClearValue(label, clp, val));
2172             PetscCall(DMLabelSetValue(label, clp, shift3+val));
2173           }
2174           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2175         }
2176       }
2177     }
2178     PetscCall(ISRestoreIndices(dimIS, &points));
2179     PetscCall(ISDestroy(&dimIS));
2180   }
2181   divide:
2182   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2183   PetscCall(DMPlexLabelFaultHalo(dm, label));
2184   PetscCall(CheckFaultEdge_Private(dm, label));
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 /* Check that no cell have all vertices on the fault */
2189 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2190 {
2191   IS              subpointIS;
2192   const PetscInt *dmpoints;
2193   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2194 
2195   PetscFunctionBegin;
2196   if (!label) PetscFunctionReturn(0);
2197   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2198   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2199   if (!subpointIS) PetscFunctionReturn(0);
2200   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2201   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2202   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2203   for (c = cStart; c < cEnd; ++c) {
2204     PetscBool invalidCell = PETSC_TRUE;
2205     PetscInt *closure     = NULL;
2206     PetscInt  closureSize, cl;
2207 
2208     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2209     for (cl = 0; cl < closureSize*2; cl += 2) {
2210       PetscInt value = 0;
2211 
2212       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2213       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2214       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2215     }
2216     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2217     if (invalidCell) {
2218       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2219       PetscCall(ISDestroy(&subpointIS));
2220       PetscCall(DMDestroy(&subdm));
2221       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2222     }
2223   }
2224   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2225   PetscFunctionReturn(0);
2226 }
2227 
2228 /*@
2229   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2230 
2231   Collective on dm
2232 
2233   Input Parameters:
2234 + dm - The original DM
2235 . label - The label specifying the interface vertices
2236 . bdlabel - The optional label specifying the interface boundary vertices
2237 - bdvalue - Value of optional label specifying the interface boundary vertices
2238 
2239   Output Parameters:
2240 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2241 . splitLabel - The label containing the split points, or NULL if no output is desired
2242 . dmInterface - The new interface DM, or NULL
2243 - dmHybrid - The new DM with cohesive cells
2244 
2245   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2246   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2247   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2248   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2249   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2250 
2251   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2252   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2253   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2254 
2255   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2256   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2257 
2258   Level: developer
2259 
2260 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2261 @*/
2262 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2263 {
2264   DM             idm;
2265   DMLabel        subpointMap, hlabel, slabel = NULL;
2266   PetscInt       dim;
2267 
2268   PetscFunctionBegin;
2269   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2270   if (bdlabel) PetscValidPointer(bdlabel, 3);
2271   if (hybridLabel) PetscValidPointer(hybridLabel, 4);
2272   if (splitLabel)  PetscValidPointer(splitLabel, 5);
2273   if (dmInterface) PetscValidPointer(dmInterface, 6);
2274   PetscValidPointer(dmHybrid, 7);
2275   PetscCall(DMGetDimension(dm, &dim));
2276   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2277   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2278   PetscCall(DMPlexOrient(idm));
2279   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2280   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2281   PetscCall(DMLabelClearStratum(hlabel, dim));
2282   if (splitLabel) {
2283     const char *name;
2284     char        sname[PETSC_MAX_PATH_LEN];
2285 
2286     PetscCall(PetscObjectGetName((PetscObject) hlabel, &name));
2287     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2288     PetscCall(PetscStrcat(sname, " split"));
2289     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2290   }
2291   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2292   if (dmInterface) {*dmInterface = idm;}
2293   else             PetscCall(DMDestroy(&idm));
2294   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2295   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2296   if (hybridLabel) *hybridLabel = hlabel;
2297   else             PetscCall(DMLabelDestroy(&hlabel));
2298   if (splitLabel)  *splitLabel  = slabel;
2299   {
2300     DM      cdm;
2301     DMLabel ctLabel;
2302 
2303     /* We need to somehow share the celltype label with the coordinate dm */
2304     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2305     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2306     PetscCall(DMSetLabel(cdm, ctLabel));
2307   }
2308   PetscFunctionReturn(0);
2309 }
2310 
2311 /* Here we need the explicit assumption that:
2312 
2313      For any marked cell, the marked vertices constitute a single face
2314 */
2315 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2316 {
2317   IS               subvertexIS = NULL;
2318   const PetscInt  *subvertices;
2319   PetscInt        *pStart, *pEnd, pSize;
2320   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2321 
2322   PetscFunctionBegin;
2323   *numFaces = 0;
2324   *nFV      = 0;
2325   PetscCall(DMPlexGetDepth(dm, &depth));
2326   PetscCall(DMGetDimension(dm, &dim));
2327   pSize = PetscMax(depth, dim) + 1;
2328   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2329   for (d = 0; d <= depth; ++d) {
2330     PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth-d, &pStart[d], &pEnd[d]));
2331   }
2332   /* Loop over initial vertices and mark all faces in the collective star() */
2333   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2334   if (subvertexIS) {
2335     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2336     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2337   }
2338   for (v = 0; v < numSubVerticesInitial; ++v) {
2339     const PetscInt vertex = subvertices[v];
2340     PetscInt      *star   = NULL;
2341     PetscInt       starSize, s, numCells = 0, c;
2342 
2343     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2344     for (s = 0; s < starSize*2; s += 2) {
2345       const PetscInt point = star[s];
2346       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2347     }
2348     for (c = 0; c < numCells; ++c) {
2349       const PetscInt cell    = star[c];
2350       PetscInt      *closure = NULL;
2351       PetscInt       closureSize, cl;
2352       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2353 
2354       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2355       if (cellLoc == 2) continue;
2356       PetscCheck(cellLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2357       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2358       for (cl = 0; cl < closureSize*2; cl += 2) {
2359         const PetscInt point = closure[cl];
2360         PetscInt       vertexLoc;
2361 
2362         if ((point >= pStart[0]) && (point < pEnd[0])) {
2363           ++numCorners;
2364           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2365           if (vertexLoc == value) closure[faceSize++] = point;
2366         }
2367       }
2368       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2369       PetscCheck(faceSize <= *nFV,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2370       if (faceSize == *nFV) {
2371         const PetscInt *cells = NULL;
2372         PetscInt        numCells, nc;
2373 
2374         ++(*numFaces);
2375         for (cl = 0; cl < faceSize; ++cl) {
2376           PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2377         }
2378         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2379         for (nc = 0; nc < numCells; ++nc) {
2380           PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2381         }
2382         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2383       }
2384       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2385     }
2386     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2387   }
2388   if (subvertexIS) {
2389     PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2390   }
2391   PetscCall(ISDestroy(&subvertexIS));
2392   PetscCall(PetscFree2(pStart, pEnd));
2393   PetscFunctionReturn(0);
2394 }
2395 
2396 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2397 {
2398   IS               subvertexIS = NULL;
2399   const PetscInt  *subvertices;
2400   PetscInt        *pStart, *pEnd;
2401   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2402 
2403   PetscFunctionBegin;
2404   PetscCall(DMGetDimension(dm, &dim));
2405   PetscCall(PetscMalloc2(dim+1, &pStart, dim+1, &pEnd));
2406   for (d = 0; d <= dim; ++d) {
2407     PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim-d, &pStart[d], &pEnd[d]));
2408   }
2409   /* Loop over initial vertices and mark all faces in the collective star() */
2410   if (vertexLabel) {
2411     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2412     if (subvertexIS) {
2413       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2414       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2415     }
2416   }
2417   for (v = 0; v < numSubVerticesInitial; ++v) {
2418     const PetscInt vertex = subvertices[v];
2419     PetscInt      *star   = NULL;
2420     PetscInt       starSize, s, numFaces = 0, f;
2421 
2422     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2423     for (s = 0; s < starSize*2; s += 2) {
2424       const PetscInt point = star[s];
2425       PetscInt       faceLoc;
2426 
2427       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2428         if (markedFaces) {
2429           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2430           if (faceLoc < 0) continue;
2431         }
2432         star[numFaces++] = point;
2433       }
2434     }
2435     for (f = 0; f < numFaces; ++f) {
2436       const PetscInt face    = star[f];
2437       PetscInt      *closure = NULL;
2438       PetscInt       closureSize, c;
2439       PetscInt       faceLoc;
2440 
2441       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2442       if (faceLoc == dim-1) continue;
2443       PetscCheck(faceLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2444       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2445       for (c = 0; c < closureSize*2; c += 2) {
2446         const PetscInt point = closure[c];
2447         PetscInt       vertexLoc;
2448 
2449         if ((point >= pStart[0]) && (point < pEnd[0])) {
2450           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2451           if (vertexLoc != value) break;
2452         }
2453       }
2454       if (c == closureSize*2) {
2455         const PetscInt *support;
2456         PetscInt        supportSize, s;
2457 
2458         for (c = 0; c < closureSize*2; c += 2) {
2459           const PetscInt point = closure[c];
2460 
2461           for (d = 0; d < dim; ++d) {
2462             if ((point >= pStart[d]) && (point < pEnd[d])) {
2463               PetscCall(DMLabelSetValue(subpointMap, point, d));
2464               break;
2465             }
2466           }
2467         }
2468         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2469         PetscCall(DMPlexGetSupport(dm, face, &support));
2470         for (s = 0; s < supportSize; ++s) {
2471           PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2472         }
2473       }
2474       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2475     }
2476     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2477   }
2478   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2479   PetscCall(ISDestroy(&subvertexIS));
2480   PetscCall(PetscFree2(pStart, pEnd));
2481   PetscFunctionReturn(0);
2482 }
2483 
2484 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2485 {
2486   DMLabel         label = NULL;
2487   const PetscInt *cone;
2488   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2489 
2490   PetscFunctionBegin;
2491   *numFaces = 0;
2492   *nFV = 0;
2493   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2494   *subCells = NULL;
2495   PetscCall(DMGetDimension(dm, &dim));
2496   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2497   if (cMax < 0) PetscFunctionReturn(0);
2498   if (label) {
2499     for (c = cMax; c < cEnd; ++c) {
2500       PetscInt val;
2501 
2502       PetscCall(DMLabelGetValue(label, c, &val));
2503       if (val == value) {
2504         ++(*numFaces);
2505         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2506       }
2507     }
2508   } else {
2509     *numFaces = cEnd - cMax;
2510     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2511   }
2512   PetscCall(PetscMalloc1(*numFaces *2, subCells));
2513   if (!(*numFaces)) PetscFunctionReturn(0);
2514   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2515   for (c = cMax; c < cEnd; ++c) {
2516     const PetscInt *cells;
2517     PetscInt        numCells;
2518 
2519     if (label) {
2520       PetscInt val;
2521 
2522       PetscCall(DMLabelGetValue(label, c, &val));
2523       if (val != value) continue;
2524     }
2525     PetscCall(DMPlexGetCone(dm, c, &cone));
2526     for (p = 0; p < *nFV; ++p) {
2527       PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2528     }
2529     /* Negative face */
2530     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2531     /* Not true in parallel
2532     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2533     for (p = 0; p < numCells; ++p) {
2534       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2535       (*subCells)[subc++] = cells[p];
2536     }
2537     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2538     /* Positive face is not included */
2539   }
2540   PetscFunctionReturn(0);
2541 }
2542 
2543 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2544 {
2545   PetscInt      *pStart, *pEnd;
2546   PetscInt       dim, cMax, cEnd, c, d;
2547 
2548   PetscFunctionBegin;
2549   PetscCall(DMGetDimension(dm, &dim));
2550   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2551   if (cMax < 0) PetscFunctionReturn(0);
2552   PetscCall(PetscMalloc2(dim+1,&pStart,dim+1,&pEnd));
2553   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2554   for (c = cMax; c < cEnd; ++c) {
2555     const PetscInt *cone;
2556     PetscInt       *closure = NULL;
2557     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2558 
2559     if (label) {
2560       PetscCall(DMLabelGetValue(label, c, &val));
2561       if (val != value) continue;
2562     }
2563     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2564     PetscCall(DMPlexGetCone(dm, c, &cone));
2565     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2566     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2567     /* Negative face */
2568     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2569     for (cl = 0; cl < closureSize*2; cl += 2) {
2570       const PetscInt point = closure[cl];
2571 
2572       for (d = 0; d <= dim; ++d) {
2573         if ((point >= pStart[d]) && (point < pEnd[d])) {
2574           PetscCall(DMLabelSetValue(subpointMap, point, d));
2575           break;
2576         }
2577       }
2578     }
2579     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2580     /* Cells -- positive face is not included */
2581     for (cl = 0; cl < 1; ++cl) {
2582       const PetscInt *support;
2583       PetscInt        supportSize, s;
2584 
2585       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2586       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2587       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2588       for (s = 0; s < supportSize; ++s) {
2589         PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2590       }
2591     }
2592   }
2593   PetscCall(PetscFree2(pStart, pEnd));
2594   PetscFunctionReturn(0);
2595 }
2596 
2597 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2598 {
2599   MPI_Comm       comm;
2600   PetscBool      posOrient = PETSC_FALSE;
2601   const PetscInt debug     = 0;
2602   PetscInt       cellDim, faceSize, f;
2603 
2604   PetscFunctionBegin;
2605   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2606   PetscCall(DMGetDimension(dm, &cellDim));
2607   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2608 
2609   if (cellDim == 1 && numCorners == 2) {
2610     /* Triangle */
2611     faceSize  = numCorners-1;
2612     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2613   } else if (cellDim == 2 && numCorners == 3) {
2614     /* Triangle */
2615     faceSize  = numCorners-1;
2616     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2617   } else if (cellDim == 3 && numCorners == 4) {
2618     /* Tetrahedron */
2619     faceSize  = numCorners-1;
2620     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2621   } else if (cellDim == 1 && numCorners == 3) {
2622     /* Quadratic line */
2623     faceSize  = 1;
2624     posOrient = PETSC_TRUE;
2625   } else if (cellDim == 2 && numCorners == 4) {
2626     /* Quads */
2627     faceSize = 2;
2628     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2629       posOrient = PETSC_TRUE;
2630     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2631       posOrient = PETSC_TRUE;
2632     } else {
2633       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2634         posOrient = PETSC_FALSE;
2635       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2636     }
2637   } else if (cellDim == 2 && numCorners == 6) {
2638     /* Quadratic triangle (I hate this) */
2639     /* Edges are determined by the first 2 vertices (corners of edges) */
2640     const PetscInt faceSizeTri = 3;
2641     PetscInt       sortedIndices[3], i, iFace;
2642     PetscBool      found                    = PETSC_FALSE;
2643     PetscInt       faceVerticesTriSorted[9] = {
2644       0, 3,  4, /* bottom */
2645       1, 4,  5, /* right */
2646       2, 3,  5, /* left */
2647     };
2648     PetscInt       faceVerticesTri[9] = {
2649       0, 3,  4, /* bottom */
2650       1, 4,  5, /* right */
2651       2, 5,  3, /* left */
2652     };
2653 
2654     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2655     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2656     for (iFace = 0; iFace < 3; ++iFace) {
2657       const PetscInt ii = iFace*faceSizeTri;
2658       PetscInt       fVertex, cVertex;
2659 
2660       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2661           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2662         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2663           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2664             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2665               faceVertices[fVertex] = origVertices[cVertex];
2666               break;
2667             }
2668           }
2669         }
2670         found = PETSC_TRUE;
2671         break;
2672       }
2673     }
2674     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2675     if (posOriented) *posOriented = PETSC_TRUE;
2676     PetscFunctionReturn(0);
2677   } else if (cellDim == 2 && numCorners == 9) {
2678     /* Quadratic quad (I hate this) */
2679     /* Edges are determined by the first 2 vertices (corners of edges) */
2680     const PetscInt faceSizeQuad = 3;
2681     PetscInt       sortedIndices[3], i, iFace;
2682     PetscBool      found                      = PETSC_FALSE;
2683     PetscInt       faceVerticesQuadSorted[12] = {
2684       0, 1,  4, /* bottom */
2685       1, 2,  5, /* right */
2686       2, 3,  6, /* top */
2687       0, 3,  7, /* left */
2688     };
2689     PetscInt       faceVerticesQuad[12] = {
2690       0, 1,  4, /* bottom */
2691       1, 2,  5, /* right */
2692       2, 3,  6, /* top */
2693       3, 0,  7, /* left */
2694     };
2695 
2696     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2697     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2698     for (iFace = 0; iFace < 4; ++iFace) {
2699       const PetscInt ii = iFace*faceSizeQuad;
2700       PetscInt       fVertex, cVertex;
2701 
2702       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2703           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2704         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2705           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2706             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2707               faceVertices[fVertex] = origVertices[cVertex];
2708               break;
2709             }
2710           }
2711         }
2712         found = PETSC_TRUE;
2713         break;
2714       }
2715     }
2716     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2717     if (posOriented) *posOriented = PETSC_TRUE;
2718     PetscFunctionReturn(0);
2719   } else if (cellDim == 3 && numCorners == 8) {
2720     /* Hexes
2721        A hex is two oriented quads with the normal of the first
2722        pointing up at the second.
2723 
2724           7---6
2725          /|  /|
2726         4---5 |
2727         | 1-|-2
2728         |/  |/
2729         0---3
2730 
2731         Faces are determined by the first 4 vertices (corners of faces) */
2732     const PetscInt faceSizeHex = 4;
2733     PetscInt       sortedIndices[4], i, iFace;
2734     PetscBool      found                     = PETSC_FALSE;
2735     PetscInt       faceVerticesHexSorted[24] = {
2736       0, 1, 2, 3,  /* bottom */
2737       4, 5, 6, 7,  /* top */
2738       0, 3, 4, 5,  /* front */
2739       2, 3, 5, 6,  /* right */
2740       1, 2, 6, 7,  /* back */
2741       0, 1, 4, 7,  /* left */
2742     };
2743     PetscInt       faceVerticesHex[24] = {
2744       1, 2, 3, 0,  /* bottom */
2745       4, 5, 6, 7,  /* top */
2746       0, 3, 5, 4,  /* front */
2747       3, 2, 6, 5,  /* right */
2748       2, 1, 7, 6,  /* back */
2749       1, 0, 4, 7,  /* left */
2750     };
2751 
2752     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2753     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2754     for (iFace = 0; iFace < 6; ++iFace) {
2755       const PetscInt ii = iFace*faceSizeHex;
2756       PetscInt       fVertex, cVertex;
2757 
2758       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2759           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2760           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2761           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2762         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2763           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2764             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2765               faceVertices[fVertex] = origVertices[cVertex];
2766               break;
2767             }
2768           }
2769         }
2770         found = PETSC_TRUE;
2771         break;
2772       }
2773     }
2774     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2775     if (posOriented) *posOriented = PETSC_TRUE;
2776     PetscFunctionReturn(0);
2777   } else if (cellDim == 3 && numCorners == 10) {
2778     /* Quadratic tet */
2779     /* Faces are determined by the first 3 vertices (corners of faces) */
2780     const PetscInt faceSizeTet = 6;
2781     PetscInt       sortedIndices[6], i, iFace;
2782     PetscBool      found                     = PETSC_FALSE;
2783     PetscInt       faceVerticesTetSorted[24] = {
2784       0, 1, 2,  6, 7, 8, /* bottom */
2785       0, 3, 4,  6, 7, 9,  /* front */
2786       1, 4, 5,  7, 8, 9,  /* right */
2787       2, 3, 5,  6, 8, 9,  /* left */
2788     };
2789     PetscInt       faceVerticesTet[24] = {
2790       0, 1, 2,  6, 7, 8, /* bottom */
2791       0, 4, 3,  6, 7, 9,  /* front */
2792       1, 5, 4,  7, 8, 9,  /* right */
2793       2, 3, 5,  8, 6, 9,  /* left */
2794     };
2795 
2796     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2797     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2798     for (iFace=0; iFace < 4; ++iFace) {
2799       const PetscInt ii = iFace*faceSizeTet;
2800       PetscInt       fVertex, cVertex;
2801 
2802       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2803           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2804           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2805           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2806         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2807           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2808             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2809               faceVertices[fVertex] = origVertices[cVertex];
2810               break;
2811             }
2812           }
2813         }
2814         found = PETSC_TRUE;
2815         break;
2816       }
2817     }
2818     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2819     if (posOriented) *posOriented = PETSC_TRUE;
2820     PetscFunctionReturn(0);
2821   } else if (cellDim == 3 && numCorners == 27) {
2822     /* Quadratic hexes (I hate this)
2823        A hex is two oriented quads with the normal of the first
2824        pointing up at the second.
2825 
2826          7---6
2827         /|  /|
2828        4---5 |
2829        | 3-|-2
2830        |/  |/
2831        0---1
2832 
2833        Faces are determined by the first 4 vertices (corners of faces) */
2834     const PetscInt faceSizeQuadHex = 9;
2835     PetscInt       sortedIndices[9], i, iFace;
2836     PetscBool      found                         = PETSC_FALSE;
2837     PetscInt       faceVerticesQuadHexSorted[54] = {
2838       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2839       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2840       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2841       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2842       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2843       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2844     };
2845     PetscInt       faceVerticesQuadHex[54] = {
2846       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2847       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2848       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2849       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2850       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2851       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2852     };
2853 
2854     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2855     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2856     for (iFace = 0; iFace < 6; ++iFace) {
2857       const PetscInt ii = iFace*faceSizeQuadHex;
2858       PetscInt       fVertex, cVertex;
2859 
2860       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2861           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2862           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2863           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2864         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2865           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2866             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2867               faceVertices[fVertex] = origVertices[cVertex];
2868               break;
2869             }
2870           }
2871         }
2872         found = PETSC_TRUE;
2873         break;
2874       }
2875     }
2876     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2877     if (posOriented) *posOriented = PETSC_TRUE;
2878     PetscFunctionReturn(0);
2879   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2880   if (!posOrient) {
2881     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2882     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2883   } else {
2884     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2885     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2886   }
2887   if (posOriented) *posOriented = posOrient;
2888   PetscFunctionReturn(0);
2889 }
2890 
2891 /*@
2892   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2893   in faceVertices. The orientation is such that the face normal points out of the cell
2894 
2895   Not collective
2896 
2897   Input Parameters:
2898 + dm           - The original mesh
2899 . cell         - The cell mesh point
2900 . faceSize     - The number of vertices on the face
2901 . face         - The face vertices
2902 . numCorners   - The number of vertices on the cell
2903 . indices      - Local numbering of face vertices in cell cone
2904 - origVertices - Original face vertices
2905 
2906   Output Parameters:
2907 + faceVertices - The face vertices properly oriented
2908 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2909 
2910   Level: developer
2911 
2912 .seealso: `DMPlexGetCone()`
2913 @*/
2914 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2915 {
2916   const PetscInt *cone = NULL;
2917   PetscInt        coneSize, v, f, v2;
2918   PetscInt        oppositeVertex = -1;
2919 
2920   PetscFunctionBegin;
2921   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2922   PetscCall(DMPlexGetCone(dm, cell, &cone));
2923   for (v = 0, v2 = 0; v < coneSize; ++v) {
2924     PetscBool found = PETSC_FALSE;
2925 
2926     for (f = 0; f < faceSize; ++f) {
2927       if (face[f] == cone[v]) {
2928         found = PETSC_TRUE; break;
2929       }
2930     }
2931     if (found) {
2932       indices[v2]      = v;
2933       origVertices[v2] = cone[v];
2934       ++v2;
2935     } else {
2936       oppositeVertex = v;
2937     }
2938   }
2939   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2940   PetscFunctionReturn(0);
2941 }
2942 
2943 /*
2944   DMPlexInsertFace_Internal - Puts a face into the mesh
2945 
2946   Not collective
2947 
2948   Input Parameters:
2949   + dm              - The DMPlex
2950   . numFaceVertex   - The number of vertices in the face
2951   . faceVertices    - The vertices in the face for dm
2952   . subfaceVertices - The vertices in the face for subdm
2953   . numCorners      - The number of vertices in the cell
2954   . cell            - A cell in dm containing the face
2955   . subcell         - A cell in subdm containing the face
2956   . firstFace       - First face in the mesh
2957   - newFacePoint    - Next face in the mesh
2958 
2959   Output Parameters:
2960   . newFacePoint - Contains next face point number on input, updated on output
2961 
2962   Level: developer
2963 */
2964 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2965 {
2966   MPI_Comm        comm;
2967   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2968   const PetscInt *faces;
2969   PetscInt        numFaces, coneSize;
2970 
2971   PetscFunctionBegin;
2972   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2973   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2974   PetscCheck(coneSize == 1,comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2975 #if 0
2976   /* Cannot use this because support() has not been constructed yet */
2977   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2978 #else
2979   {
2980     PetscInt f;
2981 
2982     numFaces = 0;
2983     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
2984     for (f = firstFace; f < *newFacePoint; ++f) {
2985       PetscInt dof, off, d;
2986 
2987       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2988       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2989       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2990       for (d = 0; d < dof; ++d) {
2991         const PetscInt p = submesh->cones[off+d];
2992         PetscInt       v;
2993 
2994         for (v = 0; v < numFaceVertices; ++v) {
2995           if (subfaceVertices[v] == p) break;
2996         }
2997         if (v == numFaceVertices) break;
2998       }
2999       if (d == dof) {
3000         numFaces               = 1;
3001         ((PetscInt*) faces)[0] = f;
3002       }
3003     }
3004   }
3005 #endif
3006   PetscCheck(numFaces <= 1,comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3007   else if (numFaces == 1) {
3008     /* Add the other cell neighbor for this face */
3009     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3010   } else {
3011     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3012     PetscBool posOriented;
3013 
3014     PetscCall(DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3015     origVertices        = &orientedVertices[numFaceVertices];
3016     indices             = &orientedVertices[numFaceVertices*2];
3017     orientedSubVertices = &orientedVertices[numFaceVertices*3];
3018     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3019     /* TODO: I know that routine should return a permutation, not the indices */
3020     for (v = 0; v < numFaceVertices; ++v) {
3021       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3022       for (ov = 0; ov < numFaceVertices; ++ov) {
3023         if (orientedVertices[ov] == vertex) {
3024           orientedSubVertices[ov] = subvertex;
3025           break;
3026         }
3027       }
3028       PetscCheck(ov != numFaceVertices,comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3029     }
3030     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3031     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3032     PetscCall(DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3033     ++(*newFacePoint);
3034   }
3035 #if 0
3036   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3037 #else
3038   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
3039 #endif
3040   PetscFunctionReturn(0);
3041 }
3042 
3043 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3044 {
3045   MPI_Comm        comm;
3046   DMLabel         subpointMap;
3047   IS              subvertexIS,  subcellIS;
3048   const PetscInt *subVertices, *subCells;
3049   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3050   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3051   PetscInt        vStart, vEnd, c, f;
3052 
3053   PetscFunctionBegin;
3054   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3055   /* Create subpointMap which marks the submesh */
3056   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3057   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3058   PetscCall(DMLabelDestroy(&subpointMap));
3059   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3060   /* Setup chart */
3061   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3062   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3063   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3064   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3065   /* Set cone sizes */
3066   firstSubVertex = numSubCells;
3067   firstSubFace   = numSubCells+numSubVertices;
3068   newFacePoint   = firstSubFace;
3069   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3070   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3071   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3072   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3073   for (c = 0; c < numSubCells; ++c) {
3074     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3075   }
3076   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3077     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3078   }
3079   PetscCall(DMSetUp(subdm));
3080   /* Create face cones */
3081   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3082   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3083   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3084   for (c = 0; c < numSubCells; ++c) {
3085     const PetscInt cell    = subCells[c];
3086     const PetscInt subcell = c;
3087     PetscInt      *closure = NULL;
3088     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3089 
3090     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3091     for (cl = 0; cl < closureSize*2; cl += 2) {
3092       const PetscInt point = closure[cl];
3093       PetscInt       subVertex;
3094 
3095       if ((point >= vStart) && (point < vEnd)) {
3096         ++numCorners;
3097         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3098         if (subVertex >= 0) {
3099           closure[faceSize] = point;
3100           subface[faceSize] = firstSubVertex+subVertex;
3101           ++faceSize;
3102         }
3103       }
3104     }
3105     PetscCheck(faceSize <= nFV,comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3106     if (faceSize == nFV) {
3107       PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3108     }
3109     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3110   }
3111   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3112   PetscCall(DMPlexSymmetrize(subdm));
3113   PetscCall(DMPlexStratify(subdm));
3114   /* Build coordinates */
3115   {
3116     PetscSection coordSection, subCoordSection;
3117     Vec          coordinates, subCoordinates;
3118     PetscScalar *coords, *subCoords;
3119     PetscInt     numComp, coordSize, v;
3120     const char  *name;
3121 
3122     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3123     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3124     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3125     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3126     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3127     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3128     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3129     for (v = 0; v < numSubVertices; ++v) {
3130       const PetscInt vertex    = subVertices[v];
3131       const PetscInt subvertex = firstSubVertex+v;
3132       PetscInt       dof;
3133 
3134       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3135       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3136       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3137     }
3138     PetscCall(PetscSectionSetUp(subCoordSection));
3139     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3140     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3141     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3142     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3143     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3144     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3145     if (coordSize) {
3146       PetscCall(VecGetArray(coordinates,    &coords));
3147       PetscCall(VecGetArray(subCoordinates, &subCoords));
3148       for (v = 0; v < numSubVertices; ++v) {
3149         const PetscInt vertex    = subVertices[v];
3150         const PetscInt subvertex = firstSubVertex+v;
3151         PetscInt       dof, off, sdof, soff, d;
3152 
3153         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3154         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3155         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3156         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3157         PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3158         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3159       }
3160       PetscCall(VecRestoreArray(coordinates,    &coords));
3161       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3162     }
3163     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3164     PetscCall(VecDestroy(&subCoordinates));
3165   }
3166   /* Cleanup */
3167   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3168   PetscCall(ISDestroy(&subvertexIS));
3169   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3170   PetscCall(ISDestroy(&subcellIS));
3171   PetscFunctionReturn(0);
3172 }
3173 
3174 /* TODO: Fix this to properly propogate up error conditions it may find */
3175 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3176 {
3177   PetscInt       subPoint;
3178   PetscErrorCode ierr;
3179 
3180   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr) return -1;
3181   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
3182 }
3183 
3184 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3185 {
3186   PetscInt       Nl, l, d;
3187 
3188   PetscFunctionBegin;
3189   PetscCall(DMGetNumLabels(dm, &Nl));
3190   for (l = 0; l < Nl; ++l) {
3191     DMLabel         label, newlabel;
3192     const char     *lname;
3193     PetscBool       isDepth, isDim, isCelltype, isVTK;
3194     IS              valueIS;
3195     const PetscInt *values;
3196     PetscInt        Nv, v;
3197 
3198     PetscCall(DMGetLabelName(dm, l, &lname));
3199     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3200     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3201     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3202     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3203     if (isDepth || isDim || isCelltype || isVTK) continue;
3204     PetscCall(DMCreateLabel(subdm, lname));
3205     PetscCall(DMGetLabel(dm, lname, &label));
3206     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3207     PetscCall(DMLabelGetDefaultValue(label, &v));
3208     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3209     PetscCall(DMLabelGetValueIS(label, &valueIS));
3210     PetscCall(ISGetLocalSize(valueIS, &Nv));
3211     PetscCall(ISGetIndices(valueIS, &values));
3212     for (v = 0; v < Nv; ++v) {
3213       IS              pointIS;
3214       const PetscInt *points;
3215       PetscInt        Np, p;
3216 
3217       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3218       PetscCall(ISGetLocalSize(pointIS, &Np));
3219       PetscCall(ISGetIndices(pointIS, &points));
3220       for (p = 0; p < Np; ++p) {
3221         const PetscInt point = points[p];
3222         PetscInt       subp;
3223 
3224         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3225         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3226         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3227       }
3228       PetscCall(ISRestoreIndices(pointIS, &points));
3229       PetscCall(ISDestroy(&pointIS));
3230     }
3231     PetscCall(ISRestoreIndices(valueIS, &values));
3232     PetscCall(ISDestroy(&valueIS));
3233   }
3234   PetscFunctionReturn(0);
3235 }
3236 
3237 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3238 {
3239   MPI_Comm         comm;
3240   DMLabel          subpointMap;
3241   IS              *subpointIS;
3242   const PetscInt **subpoints;
3243   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3244   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3245   PetscMPIInt      rank;
3246 
3247   PetscFunctionBegin;
3248   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3249   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3250   /* Create subpointMap which marks the submesh */
3251   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3252   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3253   if (cellHeight) {
3254     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3255     else            PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3256   } else {
3257     DMLabel         depth;
3258     IS              pointIS;
3259     const PetscInt *points;
3260     PetscInt        numPoints=0;
3261 
3262     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3263     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3264     if (pointIS) {
3265       PetscCall(ISGetIndices(pointIS, &points));
3266       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3267     }
3268     for (p = 0; p < numPoints; ++p) {
3269       PetscInt *closure = NULL;
3270       PetscInt  closureSize, c, pdim;
3271 
3272       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3273       for (c = 0; c < closureSize*2; c += 2) {
3274         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3275         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3276       }
3277       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3278     }
3279     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3280     PetscCall(ISDestroy(&pointIS));
3281   }
3282   /* Setup chart */
3283   PetscCall(DMGetDimension(dm, &dim));
3284   PetscCall(PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints));
3285   for (d = 0; d <= dim; ++d) {
3286     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3287     totSubPoints += numSubPoints[d];
3288   }
3289   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3290   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3291   /* Set cone sizes */
3292   firstSubPoint[dim] = 0;
3293   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3294   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3295   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3296   for (d = 0; d <= dim; ++d) {
3297     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3298     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3299   }
3300   /* We do not want this label automatically computed, instead we compute it here */
3301   PetscCall(DMCreateLabel(subdm, "celltype"));
3302   for (d = 0; d <= dim; ++d) {
3303     for (p = 0; p < numSubPoints[d]; ++p) {
3304       const PetscInt  point    = subpoints[d][p];
3305       const PetscInt  subpoint = firstSubPoint[d] + p;
3306       const PetscInt *cone;
3307       PetscInt        coneSize;
3308 
3309       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3310       if (cellHeight && (d == dim)) {
3311         PetscInt coneSizeNew, c, val;
3312 
3313         PetscCall(DMPlexGetCone(dm, point, &cone));
3314         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3315           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3316           if (val >= 0) coneSizeNew++;
3317         }
3318         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3319         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3320       } else {
3321         DMPolytopeType  ct;
3322 
3323         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3324         PetscCall(DMPlexGetCellType(dm, point, &ct));
3325         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3326       }
3327     }
3328   }
3329   PetscCall(DMLabelDestroy(&subpointMap));
3330   PetscCall(DMSetUp(subdm));
3331   /* Set cones */
3332   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3333   PetscCall(PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew));
3334   for (d = 0; d <= dim; ++d) {
3335     for (p = 0; p < numSubPoints[d]; ++p) {
3336       const PetscInt  point    = subpoints[d][p];
3337       const PetscInt  subpoint = firstSubPoint[d] + p;
3338       const PetscInt *cone, *ornt;
3339       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3340 
3341       if (d == dim-1) {
3342         const PetscInt *support, *cone, *ornt;
3343         PetscInt        supportSize, coneSize, s, subc;
3344 
3345         PetscCall(DMPlexGetSupport(dm, point, &support));
3346         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3347         for (s = 0; s < supportSize; ++s) {
3348           PetscBool isHybrid;
3349 
3350           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3351           if (!isHybrid) continue;
3352           PetscCall(PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc));
3353           if (subc >= 0) {
3354             const PetscInt ccell = subpoints[d+1][subc];
3355 
3356             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3357             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3358             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3359             for (c = 0; c < coneSize; ++c) {
3360               if (cone[c] == point) {
3361                 fornt = ornt[c];
3362                 break;
3363               }
3364             }
3365             break;
3366           }
3367         }
3368       }
3369       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3370       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3371       PetscCall(DMPlexGetCone(dm, point, &cone));
3372       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3373       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3374         PetscCall(PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc));
3375         if (subc >= 0) {
3376           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3377           orntNew[coneSizeNew] = ornt[c];
3378           ++coneSizeNew;
3379         }
3380       }
3381       PetscCheck(coneSizeNew == subconeSize,comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3382       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3383       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3384       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3385     }
3386   }
3387   PetscCall(PetscFree2(coneNew,orntNew));
3388   PetscCall(DMPlexSymmetrize(subdm));
3389   PetscCall(DMPlexStratify(subdm));
3390   /* Build coordinates */
3391   {
3392     PetscSection coordSection, subCoordSection;
3393     Vec          coordinates, subCoordinates;
3394     PetscScalar *coords, *subCoords;
3395     PetscInt     cdim, numComp, coordSize;
3396     const char  *name;
3397 
3398     PetscCall(DMGetCoordinateDim(dm, &cdim));
3399     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3400     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3401     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3402     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3403     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3404     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3405     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]));
3406     for (v = 0; v < numSubPoints[0]; ++v) {
3407       const PetscInt vertex    = subpoints[0][v];
3408       const PetscInt subvertex = firstSubPoint[0]+v;
3409       PetscInt       dof;
3410 
3411       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3412       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3413       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3414     }
3415     PetscCall(PetscSectionSetUp(subCoordSection));
3416     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3417     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3418     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3419     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3420     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3421     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3422     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3423     PetscCall(VecGetArray(coordinates,    &coords));
3424     PetscCall(VecGetArray(subCoordinates, &subCoords));
3425     for (v = 0; v < numSubPoints[0]; ++v) {
3426       const PetscInt vertex    = subpoints[0][v];
3427       const PetscInt subvertex = firstSubPoint[0]+v;
3428       PetscInt dof, off, sdof, soff, d;
3429 
3430       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3431       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3432       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3433       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3434       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3435       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3436     }
3437     PetscCall(VecRestoreArray(coordinates,    &coords));
3438     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3439     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3440     PetscCall(VecDestroy(&subCoordinates));
3441   }
3442   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3443   {
3444     PetscSF            sfPoint, sfPointSub;
3445     IS                 subpIS;
3446     const PetscSFNode *remotePoints;
3447     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3448     const PetscInt    *localPoints, *subpoints;
3449     PetscInt          *slocalPoints;
3450     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3451     PetscMPIInt        rank;
3452 
3453     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3454     PetscCall(DMGetPointSF(dm, &sfPoint));
3455     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3456     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3457     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3458     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3459     if (subpIS) {
3460       PetscCall(ISGetIndices(subpIS, &subpoints));
3461       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3462     }
3463     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3464     if (numRoots >= 0) {
3465       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3466       for (p = 0; p < pEnd-pStart; ++p) {
3467         newLocalPoints[p].rank  = -2;
3468         newLocalPoints[p].index = -2;
3469       }
3470       /* Set subleaves */
3471       for (l = 0; l < numLeaves; ++l) {
3472         const PetscInt point    = localPoints[l];
3473         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3474 
3475         if (subpoint < 0) continue;
3476         newLocalPoints[point-pStart].rank  = rank;
3477         newLocalPoints[point-pStart].index = subpoint;
3478         ++numSubleaves;
3479       }
3480       /* Must put in owned subpoints */
3481       for (p = pStart; p < pEnd; ++p) {
3482         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3483 
3484         if (subpoint < 0) {
3485           newOwners[p-pStart].rank  = -3;
3486           newOwners[p-pStart].index = -3;
3487         } else {
3488           newOwners[p-pStart].rank  = rank;
3489           newOwners[p-pStart].index = subpoint;
3490         }
3491       }
3492       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3493       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3494       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3495       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3496       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3497       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3498       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3499         const PetscInt point    = localPoints[l];
3500         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3501 
3502         if (subpoint < 0) continue;
3503         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3504         slocalPoints[sl]        = subpoint;
3505         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3506         sremotePoints[sl].index = newLocalPoints[point].index;
3507         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3508         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3509         ++sl;
3510       }
3511       PetscCheck(sl + ll == numSubleaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3512       PetscCall(PetscFree2(newLocalPoints,newOwners));
3513       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3514     }
3515     if (subpIS) {
3516       PetscCall(ISRestoreIndices(subpIS, &subpoints));
3517     }
3518   }
3519   /* Filter labels */
3520   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3521   /* Cleanup */
3522   for (d = 0; d <= dim; ++d) {
3523     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3524     PetscCall(ISDestroy(&subpointIS[d]));
3525   }
3526   PetscCall(PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints));
3527   PetscFunctionReturn(0);
3528 }
3529 
3530 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3531 {
3532   PetscFunctionBegin;
3533   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3534   PetscFunctionReturn(0);
3535 }
3536 
3537 /*@
3538   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3539 
3540   Input Parameters:
3541 + dm           - The original mesh
3542 . vertexLabel  - The DMLabel marking points contained in the surface
3543 . value        - The label value to use
3544 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3545 
3546   Output Parameter:
3547 . subdm - The surface mesh
3548 
3549   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3550 
3551   Level: developer
3552 
3553 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3554 @*/
3555 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3556 {
3557   DMPlexInterpolatedFlag interpolated;
3558   PetscInt       dim, cdim;
3559 
3560   PetscFunctionBegin;
3561   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3562   PetscValidPointer(subdm, 5);
3563   PetscCall(DMGetDimension(dm, &dim));
3564   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3565   PetscCall(DMSetType(*subdm, DMPLEX));
3566   PetscCall(DMSetDimension(*subdm, dim-1));
3567   PetscCall(DMGetCoordinateDim(dm, &cdim));
3568   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3569   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3570   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3571   if (interpolated) {
3572     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3573   } else {
3574     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3575   }
3576   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3577   PetscFunctionReturn(0);
3578 }
3579 
3580 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3581 {
3582   MPI_Comm        comm;
3583   DMLabel         subpointMap;
3584   IS              subvertexIS;
3585   const PetscInt *subVertices;
3586   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3587   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3588   PetscInt        c, f;
3589 
3590   PetscFunctionBegin;
3591   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3592   /* Create subpointMap which marks the submesh */
3593   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3594   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3595   PetscCall(DMLabelDestroy(&subpointMap));
3596   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3597   /* Setup chart */
3598   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3599   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3600   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3601   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3602   /* Set cone sizes */
3603   firstSubVertex = numSubCells;
3604   firstSubFace   = numSubCells+numSubVertices;
3605   newFacePoint   = firstSubFace;
3606   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3607   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3608   for (c = 0; c < numSubCells; ++c) {
3609     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3610   }
3611   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3612     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3613   }
3614   PetscCall(DMSetUp(subdm));
3615   /* Create face cones */
3616   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3617   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3618   for (c = 0; c < numSubCells; ++c) {
3619     const PetscInt  cell    = subCells[c];
3620     const PetscInt  subcell = c;
3621     const PetscInt *cone, *cells;
3622     PetscBool       isHybrid;
3623     PetscInt        numCells, subVertex, p, v;
3624 
3625     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3626     if (!isHybrid) continue;
3627     PetscCall(DMPlexGetCone(dm, cell, &cone));
3628     for (v = 0; v < nFV; ++v) {
3629       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3630       subface[v] = firstSubVertex+subVertex;
3631     }
3632     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3633     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3634     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3635     /* Not true in parallel
3636     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3637     for (p = 0; p < numCells; ++p) {
3638       PetscInt  negsubcell;
3639       PetscBool isHybrid;
3640 
3641       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3642       if (isHybrid) continue;
3643       /* I know this is a crap search */
3644       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3645         if (subCells[negsubcell] == cells[p]) break;
3646       }
3647       PetscCheck(negsubcell != numSubCells,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3648       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3649     }
3650     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3651     ++newFacePoint;
3652   }
3653   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3654   PetscCall(DMPlexSymmetrize(subdm));
3655   PetscCall(DMPlexStratify(subdm));
3656   /* Build coordinates */
3657   {
3658     PetscSection coordSection, subCoordSection;
3659     Vec          coordinates, subCoordinates;
3660     PetscScalar *coords, *subCoords;
3661     PetscInt     cdim, numComp, coordSize, v;
3662     const char  *name;
3663 
3664     PetscCall(DMGetCoordinateDim(dm, &cdim));
3665     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3666     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3667     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3668     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3669     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3670     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3671     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3672     for (v = 0; v < numSubVertices; ++v) {
3673       const PetscInt vertex    = subVertices[v];
3674       const PetscInt subvertex = firstSubVertex+v;
3675       PetscInt       dof;
3676 
3677       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3678       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3679       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3680     }
3681     PetscCall(PetscSectionSetUp(subCoordSection));
3682     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3683     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3684     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3685     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3686     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3687     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3688     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3689     PetscCall(VecGetArray(coordinates,    &coords));
3690     PetscCall(VecGetArray(subCoordinates, &subCoords));
3691     for (v = 0; v < numSubVertices; ++v) {
3692       const PetscInt vertex    = subVertices[v];
3693       const PetscInt subvertex = firstSubVertex+v;
3694       PetscInt       dof, off, sdof, soff, d;
3695 
3696       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3697       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3698       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3699       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3700       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3701       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3702     }
3703     PetscCall(VecRestoreArray(coordinates,    &coords));
3704     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3705     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3706     PetscCall(VecDestroy(&subCoordinates));
3707   }
3708   /* Build SF */
3709   CHKMEMQ;
3710   {
3711     PetscSF            sfPoint, sfPointSub;
3712     const PetscSFNode *remotePoints;
3713     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3714     const PetscInt    *localPoints;
3715     PetscInt          *slocalPoints;
3716     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3717     PetscMPIInt        rank;
3718 
3719     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3720     PetscCall(DMGetPointSF(dm, &sfPoint));
3721     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3722     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3723     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3724     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3725     if (numRoots >= 0) {
3726       /* Only vertices should be shared */
3727       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3728       for (p = 0; p < pEnd-pStart; ++p) {
3729         newLocalPoints[p].rank  = -2;
3730         newLocalPoints[p].index = -2;
3731       }
3732       /* Set subleaves */
3733       for (l = 0; l < numLeaves; ++l) {
3734         const PetscInt point    = localPoints[l];
3735         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3736 
3737         PetscCheck(!(point < vStart) || !(point >= vEnd),PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3738         if (subPoint < 0) continue;
3739         newLocalPoints[point-pStart].rank  = rank;
3740         newLocalPoints[point-pStart].index = subPoint;
3741         ++numSubLeaves;
3742       }
3743       /* Must put in owned subpoints */
3744       for (p = pStart; p < pEnd; ++p) {
3745         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3746 
3747         if (subPoint < 0) {
3748           newOwners[p-pStart].rank  = -3;
3749           newOwners[p-pStart].index = -3;
3750         } else {
3751           newOwners[p-pStart].rank  = rank;
3752           newOwners[p-pStart].index = subPoint;
3753         }
3754       }
3755       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3756       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3757       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3758       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3759       PetscCall(PetscMalloc1(numSubLeaves,    &slocalPoints));
3760       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3761       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3762         const PetscInt point    = localPoints[l];
3763         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3764 
3765         if (subPoint < 0) continue;
3766         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3767         slocalPoints[sl]        = subPoint;
3768         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3769         sremotePoints[sl].index = newLocalPoints[point].index;
3770         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3771         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3772         ++sl;
3773       }
3774       PetscCall(PetscFree2(newLocalPoints,newOwners));
3775       PetscCheck(sl + ll == numSubLeaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3776       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3777     }
3778   }
3779   CHKMEMQ;
3780   /* Cleanup */
3781   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3782   PetscCall(ISDestroy(&subvertexIS));
3783   PetscCall(PetscFree(subCells));
3784   PetscFunctionReturn(0);
3785 }
3786 
3787 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3788 {
3789   DMLabel        label = NULL;
3790 
3791   PetscFunctionBegin;
3792   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3793   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3794   PetscFunctionReturn(0);
3795 }
3796 
3797 /*@C
3798   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3799 
3800   Input Parameters:
3801 + dm          - The original mesh
3802 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3803 . label       - A label name, or NULL
3804 - value  - A label value
3805 
3806   Output Parameter:
3807 . subdm - The surface mesh
3808 
3809   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3810 
3811   Level: developer
3812 
3813 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3814 @*/
3815 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3816 {
3817   PetscInt       dim, cdim, depth;
3818 
3819   PetscFunctionBegin;
3820   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3821   PetscValidPointer(subdm, 5);
3822   PetscCall(DMGetDimension(dm, &dim));
3823   PetscCall(DMPlexGetDepth(dm, &depth));
3824   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3825   PetscCall(DMSetType(*subdm, DMPLEX));
3826   PetscCall(DMSetDimension(*subdm, dim-1));
3827   PetscCall(DMGetCoordinateDim(dm, &cdim));
3828   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3829   if (depth == dim) {
3830     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3831   } else {
3832     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3833   }
3834   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3835   PetscFunctionReturn(0);
3836 }
3837 
3838 /*@
3839   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3840 
3841   Input Parameters:
3842 + dm        - The original mesh
3843 . cellLabel - The DMLabel marking cells contained in the new mesh
3844 - value     - The label value to use
3845 
3846   Output Parameter:
3847 . subdm - The new mesh
3848 
3849   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3850 
3851   Level: developer
3852 
3853 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3854 @*/
3855 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3856 {
3857   PetscInt       dim;
3858 
3859   PetscFunctionBegin;
3860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3861   PetscValidPointer(subdm, 4);
3862   PetscCall(DMGetDimension(dm, &dim));
3863   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), subdm));
3864   PetscCall(DMSetType(*subdm, DMPLEX));
3865   PetscCall(DMSetDimension(*subdm, dim));
3866   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3867   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3868   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3869   PetscFunctionReturn(0);
3870 }
3871 
3872 /*@
3873   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3874 
3875   Input Parameter:
3876 . dm - The submesh DM
3877 
3878   Output Parameter:
3879 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3880 
3881   Level: developer
3882 
3883 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3884 @*/
3885 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3886 {
3887   PetscFunctionBegin;
3888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3889   PetscValidPointer(subpointMap, 2);
3890   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3891   PetscFunctionReturn(0);
3892 }
3893 
3894 /*@
3895   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3896 
3897   Input Parameters:
3898 + dm - The submesh DM
3899 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3900 
3901   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3902 
3903   Level: developer
3904 
3905 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3906 @*/
3907 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3908 {
3909   DM_Plex       *mesh = (DM_Plex *) dm->data;
3910   DMLabel        tmp;
3911 
3912   PetscFunctionBegin;
3913   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3914   tmp  = mesh->subpointMap;
3915   mesh->subpointMap = subpointMap;
3916   PetscCall(PetscObjectReference((PetscObject) mesh->subpointMap));
3917   PetscCall(DMLabelDestroy(&tmp));
3918   PetscFunctionReturn(0);
3919 }
3920 
3921 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3922 {
3923   DMLabel        spmap;
3924   PetscInt       depth, d;
3925 
3926   PetscFunctionBegin;
3927   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3928   PetscCall(DMPlexGetDepth(dm, &depth));
3929   if (spmap && depth >= 0) {
3930     DM_Plex  *mesh = (DM_Plex *) dm->data;
3931     PetscInt *points, *depths;
3932     PetscInt  pStart, pEnd, p, off;
3933 
3934     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3935     PetscCheck(!pStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3936     PetscCall(PetscMalloc1(pEnd, &points));
3937     PetscCall(DMGetWorkArray(dm, depth+1, MPIU_INT, &depths));
3938     depths[0] = depth;
3939     depths[1] = 0;
3940     for (d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3941     for (d = 0, off = 0; d <= depth; ++d) {
3942       const PetscInt dep = depths[d];
3943       PetscInt       depStart, depEnd, n;
3944 
3945       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3946       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3947       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3948         PetscCheck(n == depEnd-depStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd-depStart);
3949       } else {
3950         if (!n) {
3951           if (d == 0) {
3952             /* Missing cells */
3953             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3954           } else {
3955             /* Missing faces */
3956             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3957           }
3958         }
3959       }
3960       if (n) {
3961         IS              is;
3962         const PetscInt *opoints;
3963 
3964         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3965         PetscCall(ISGetIndices(is, &opoints));
3966         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3967         PetscCall(ISRestoreIndices(is, &opoints));
3968         PetscCall(ISDestroy(&is));
3969       }
3970     }
3971     PetscCall(DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths));
3972     PetscCheck(off == pEnd,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3973     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3974     PetscCall(PetscObjectStateGet((PetscObject) spmap, &mesh->subpointState));
3975   }
3976   PetscFunctionReturn(0);
3977 }
3978 
3979 /*@
3980   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3981 
3982   Input Parameter:
3983 . dm - The submesh DM
3984 
3985   Output Parameter:
3986 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3987 
3988   Note: This IS is guaranteed to be sorted by the construction of the submesh
3989 
3990   Level: developer
3991 
3992 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3993 @*/
3994 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
3995 {
3996   DM_Plex         *mesh = (DM_Plex *) dm->data;
3997   DMLabel          spmap;
3998   PetscObjectState state;
3999 
4000   PetscFunctionBegin;
4001   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4002   PetscValidPointer(subpointIS, 2);
4003   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4004   PetscCall(PetscObjectStateGet((PetscObject) spmap, &state));
4005   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4006   *subpointIS = mesh->subpointIS;
4007   PetscFunctionReturn(0);
4008 }
4009 
4010 /*@
4011   DMGetEnclosureRelation - Get the relationship between dmA and dmB
4012 
4013   Input Parameters:
4014 + dmA - The first DM
4015 - dmB - The second DM
4016 
4017   Output Parameter:
4018 . rel - The relation of dmA to dmB
4019 
4020   Level: intermediate
4021 
4022 .seealso: `DMGetEnclosurePoint()`
4023 @*/
4024 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4025 {
4026   DM             plexA, plexB, sdm;
4027   DMLabel        spmap;
4028   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4029 
4030   PetscFunctionBegin;
4031   PetscValidPointer(rel, 3);
4032   *rel = DM_ENC_NONE;
4033   if (!dmA || !dmB) PetscFunctionReturn(0);
4034   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4035   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4036   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
4037   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4038   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4039   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4040   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4041   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4042     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4043   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4044     *rel = DM_ENC_EQUALITY;
4045     goto end;
4046   }
4047   NpA = pEndA - pStartA;
4048   NpB = pEndB - pStartB;
4049   if (NpA == NpB) goto end;
4050   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4051   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4052   if (!spmap) goto end;
4053   /* TODO Check the space mapped to by subpointMap is same size as dm */
4054   if (NpA > NpB) {
4055     *rel = DM_ENC_SUPERMESH;
4056   } else {
4057     *rel = DM_ENC_SUBMESH;
4058   }
4059   end:
4060   PetscCall(DMDestroy(&plexA));
4061   PetscCall(DMDestroy(&plexB));
4062   PetscFunctionReturn(0);
4063 }
4064 
4065 /*@
4066   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4067 
4068   Input Parameters:
4069 + dmA   - The first DM
4070 . dmB   - The second DM
4071 . etype - The type of enclosure relation that dmA has to dmB
4072 - pB    - A point of dmB
4073 
4074   Output Parameter:
4075 . pA    - The corresponding point of dmA
4076 
4077   Level: intermediate
4078 
4079 .seealso: `DMGetEnclosureRelation()`
4080 @*/
4081 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4082 {
4083   DM              sdm;
4084   IS              subpointIS;
4085   const PetscInt *subpoints;
4086   PetscInt        numSubpoints;
4087 
4088   PetscFunctionBegin;
4089   /* TODO Cache the IS, making it look like an index */
4090   switch (etype) {
4091     case DM_ENC_SUPERMESH:
4092     sdm  = dmB;
4093     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4094     PetscCall(ISGetIndices(subpointIS, &subpoints));
4095     *pA  = subpoints[pB];
4096     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4097     break;
4098     case DM_ENC_SUBMESH:
4099     sdm  = dmA;
4100     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4101     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4102     PetscCall(ISGetIndices(subpointIS, &subpoints));
4103     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4104     if (*pA < 0) {
4105       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4106       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4107       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4108     }
4109     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4110     break;
4111     case DM_ENC_EQUALITY:
4112     case DM_ENC_NONE:
4113     *pA = pB;break;
4114     case DM_ENC_UNKNOWN:
4115     {
4116       DMEnclosureType enc;
4117 
4118       PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4119       PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4120     }
4121     break;
4122     default: SETERRQ(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
4123   }
4124   PetscFunctionReturn(0);
4125 }
4126