xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 7fdeb8b9be692273ce99bf256278a44482a052ca)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petscsf.h>
3 
4 #undef __FUNCT__
5 #define __FUNCT__ "DMPlexMarkBoundaryFaces"
6 /*@
7   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
8 
9   Not Collective
10 
11   Input Parameter:
12 . dm - The original DM
13 
14   Output Parameter:
15 . label - The DMLabel marking boundary faces with value 1
16 
17   Level: developer
18 
19 .seealso: DMLabelCreate(), DMPlexCreateLabel()
20 @*/
21 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
22 {
23   PetscInt       fStart, fEnd, f;
24   PetscErrorCode ierr;
25 
26   PetscFunctionBegin;
27   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
28   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
29   for (f = fStart; f < fEnd; ++f) {
30     PetscInt supportSize;
31 
32     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
33     if (supportSize == 1) {
34       ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);
35     }
36   }
37   PetscFunctionReturn(0);
38 }
39 
40 #undef __FUNCT__
41 #define __FUNCT__ "DMPlexLabelComplete"
42 /*@
43   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
44 
45   Input Parameters:
46 + dm - The DM
47 - label - A DMLabel marking the surface points
48 
49   Output Parameter:
50 . label - A DMLabel marking all surface points in the transitive closure
51 
52   Level: developer
53 
54 .seealso: DMPlexLabelCohesiveComplete()
55 @*/
56 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
57 {
58   IS              valueIS;
59   const PetscInt *values;
60   PetscInt        numValues, v;
61   PetscErrorCode  ierr;
62 
63   PetscFunctionBegin;
64   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
65   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
66   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
67   for (v = 0; v < numValues; ++v) {
68     IS              pointIS;
69     const PetscInt *points;
70     PetscInt        numPoints, p;
71 
72     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
73     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
74     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
75     for (p = 0; p < numPoints; ++p) {
76       PetscInt *closure = NULL;
77       PetscInt  closureSize, c;
78 
79       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
80       for (c = 0; c < closureSize*2; c += 2) {
81         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
82       }
83       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
84     }
85     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
86     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
87   }
88   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
89   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
90   PetscFunctionReturn(0);
91 }
92 
93 #undef __FUNCT__
94 #define __FUNCT__ "DMPlexShiftPoint_Internal"
95 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthMax[], PetscInt depthEnd[], PetscInt depthShift[])
96 {
97   if (depth < 0) return p;
98   /* Normal Cells                 */ if (p < depthMax[depth])                return p;
99   /* Hybrid Cells+Normal Vertices */ if (p < depthMax[0])                    return p + depthShift[depth];
100   /* Hybrid Vertices+Normal Faces */ if (depth < 2 || p < depthMax[depth-1]) return p + depthShift[depth] + depthShift[0];
101   /* Hybrid Faces+Normal Edges    */ if (depth < 3 || p < depthMax[depth-2]) return p + depthShift[depth] + depthShift[0] + depthShift[depth-1];
102   /* Hybrid Edges                 */                                         return p + depthShift[depth] + depthShift[0] + depthShift[depth-1] + depthShift[depth-2];
103 }
104 
105 #undef __FUNCT__
106 #define __FUNCT__ "DMPlexShiftSizes_Internal"
107 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
108 {
109   PetscInt      *depthMax, *depthEnd;
110   PetscInt       depth = 0, d, pStart, pEnd, p;
111   PetscErrorCode ierr;
112 
113   PetscFunctionBegin;
114   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
115   if (depth < 0) PetscFunctionReturn(0);
116   ierr = PetscMalloc2(depth+1,&depthMax,depth+1,&depthEnd);CHKERRQ(ierr);
117   /* Step 1: Expand chart */
118   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
119   ierr = DMPlexGetHybridBounds(dm, &depthMax[depth], depth > 0 ? &depthMax[depth-1] : NULL, &depthMax[1], &depthMax[0]);CHKERRQ(ierr);
120   for (d = 0; d <= depth; ++d) {
121     pEnd += depthShift[d];
122     ierr  = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
123     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
124   }
125   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
126   /* Step 2: Set cone and support sizes */
127   for (d = 0; d <= depth; ++d) {
128     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
129     for (p = pStart; p < pEnd; ++p) {
130       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthMax, depthEnd, depthShift);
131       PetscInt size;
132 
133       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
134       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
135       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
136       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
137     }
138   }
139   ierr = PetscFree2(depthMax,depthEnd);CHKERRQ(ierr);
140   PetscFunctionReturn(0);
141 }
142 
143 #undef __FUNCT__
144 #define __FUNCT__ "DMPlexShiftPoints_Internal"
145 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
146 {
147   PetscInt      *depthEnd, *depthMax, *newpoints;
148   PetscInt       depth = 0, d, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
149   PetscErrorCode ierr;
150 
151   PetscFunctionBegin;
152   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
153   if (depth < 0) PetscFunctionReturn(0);
154   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
155   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
156   ierr = PetscMalloc3(depth+1,&depthEnd,depth+1,&depthMax,PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
157   ierr = DMPlexGetHybridBounds(dm, &depthMax[depth], depth > 0 ? &depthMax[depth-1] : NULL, &depthMax[1], &depthMax[0]);CHKERRQ(ierr);
158   for (d = 0; d <= depth; ++d) {
159     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
160     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
161   }
162   /* Step 5: Set cones and supports */
163   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
164   for (p = pStart; p < pEnd; ++p) {
165     const PetscInt *points = NULL, *orientations = NULL;
166     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthMax, depthEnd, depthShift);
167 
168     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
169     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
170     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
171     for (i = 0; i < size; ++i) {
172       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthMax, depthEnd, depthShift);
173     }
174     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
175     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
176     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
177     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
178     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
179     for (i = 0; i < size; ++i) {
180       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthMax, depthEnd, depthShift);
181     }
182     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
183     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
184   }
185   ierr = PetscFree3(depthEnd,depthMax,newpoints);CHKERRQ(ierr);
186   PetscFunctionReturn(0);
187 }
188 
189 #undef __FUNCT__
190 #define __FUNCT__ "DMPlexShiftCoordinates_Internal"
191 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
192 {
193   PetscSection   coordSection, newCoordSection;
194   Vec            coordinates, newCoordinates;
195   PetscScalar   *coords, *newCoords;
196   PetscInt      *depthEnd, coordSize;
197   PetscInt       dim, depth = 0, d, vStart, vEnd, vStartNew, vEndNew, v;
198   PetscErrorCode ierr;
199 
200   PetscFunctionBegin;
201   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
202   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
203   ierr = PetscMalloc1((depth+1), &depthEnd);CHKERRQ(ierr);
204   for (d = 0; d <= depth; ++d) {
205     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
206   }
207   /* Step 8: Convert coordinates */
208   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
209   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
210   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
211   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
212   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
213   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
214   ierr = PetscSectionSetChart(newCoordSection, vStartNew, vEndNew);CHKERRQ(ierr);
215   for (v = vStartNew; v < vEndNew; ++v) {
216     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
217     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
218   }
219   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
220   ierr = DMSetCoordinateSection(dmNew, newCoordSection);CHKERRQ(ierr);
221   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
222   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &newCoordinates);CHKERRQ(ierr);
223   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
224   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
225   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
226   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
227   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
228   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
229   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
230   for (v = vStart; v < vEnd; ++v) {
231     PetscInt dof, off, noff, d;
232 
233     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
234     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
235     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthEnd, depthEnd, depthShift), &noff);CHKERRQ(ierr);
236     for (d = 0; d < dof; ++d) {
237       newCoords[noff+d] = coords[off+d];
238     }
239   }
240   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
241   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
242   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
243   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
244   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
245   PetscFunctionReturn(0);
246 }
247 
248 #undef __FUNCT__
249 #define __FUNCT__ "DMPlexShiftSF_Internal"
250 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
251 {
252   PetscInt          *depthMax, *depthEnd;
253   PetscInt           depth = 0, d;
254   PetscSF            sfPoint, sfPointNew;
255   const PetscSFNode *remotePoints;
256   PetscSFNode       *gremotePoints;
257   const PetscInt    *localPoints;
258   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
259   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
260   PetscErrorCode     ierr;
261 
262   PetscFunctionBegin;
263   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
264   ierr = PetscMalloc2(depth+1,&depthMax,depth+1,&depthEnd);CHKERRQ(ierr);
265   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
266   for (d = 0; d <= depth; ++d) {
267     totShift += depthShift[d];
268     ierr      = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
269     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
270   }
271   /* Step 9: Convert pointSF */
272   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
273   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
274   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
275   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
276   if (numRoots >= 0) {
277     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
278     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthMax, depthEnd, depthShift);
279     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
280     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
281     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
282     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
283     for (l = 0; l < numLeaves; ++l) {
284       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthMax, depthEnd, depthShift);
285       gremotePoints[l].rank  = remotePoints[l].rank;
286       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
287     }
288     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
289     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
290   }
291   ierr = PetscFree2(depthMax,depthEnd);CHKERRQ(ierr);
292   PetscFunctionReturn(0);
293 }
294 
295 #undef __FUNCT__
296 #define __FUNCT__ "DMPlexShiftLabels_Internal"
297 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
298 {
299   PetscSF            sfPoint;
300   DMLabel            vtkLabel, ghostLabel;
301   PetscInt          *depthMax, *depthEnd;
302   const PetscSFNode *leafRemote;
303   const PetscInt    *leafLocal;
304   PetscInt           depth = 0, d, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
305   PetscMPIInt        rank;
306   PetscErrorCode     ierr;
307 
308   PetscFunctionBegin;
309   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
310   ierr = PetscMalloc2(depth+1,&depthMax,depth+1,&depthEnd);CHKERRQ(ierr);
311   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
312   for (d = 0; d <= depth; ++d) {
313     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
314     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
315   }
316   /* Step 10: Convert labels */
317   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
318   for (l = 0; l < numLabels; ++l) {
319     DMLabel         label, newlabel;
320     const char     *lname;
321     PetscBool       isDepth;
322     IS              valueIS;
323     const PetscInt *values;
324     PetscInt        numValues, val;
325 
326     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
327     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
328     if (isDepth) continue;
329     ierr = DMPlexCreateLabel(dmNew, lname);CHKERRQ(ierr);
330     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
331     ierr = DMPlexGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
332     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
333     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
334     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
335     for (val = 0; val < numValues; ++val) {
336       IS              pointIS;
337       const PetscInt *points;
338       PetscInt        numPoints, p;
339 
340       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
341       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
342       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
343       for (p = 0; p < numPoints; ++p) {
344         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthMax, depthEnd, depthShift);
345 
346         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
347       }
348       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
349       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
350     }
351     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
352     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
353   }
354   ierr = PetscFree2(depthMax,depthEnd);CHKERRQ(ierr);
355   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
356   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
357   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
358   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
359   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
360   ierr = DMPlexCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
361   ierr = DMPlexCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
362   ierr = DMPlexGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
363   ierr = DMPlexGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
364   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
365     for (; c < leafLocal[l] && c < cEnd; ++c) {
366       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
367     }
368     if (leafLocal[l] >= cEnd) break;
369     if (leafRemote[l].rank == rank) {
370       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
371     } else {
372       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
373     }
374   }
375   for (; c < cEnd; ++c) {
376     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
377   }
378   if (0) {
379     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
380     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
381     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
382   }
383   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
384   for (f = fStart; f < fEnd; ++f) {
385     PetscInt numCells;
386 
387     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
388     if (numCells < 2) {
389       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
390     } else {
391       const PetscInt *cells = NULL;
392       PetscInt        vA, vB;
393 
394       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
395       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
396       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
397       if (!vA && !vB) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
398     }
399   }
400   if (0) {
401     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
402     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
403     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
404   }
405   PetscFunctionReturn(0);
406 }
407 
408 #undef __FUNCT__
409 #define __FUNCT__ "DMPlexConstructGhostCells_Internal"
410 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
411 {
412   IS              valueIS;
413   const PetscInt *values;
414   PetscInt       *depthShift;
415   PetscInt        depth = 0, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
416   PetscErrorCode  ierr;
417 
418   PetscFunctionBegin;
419   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
420   /* Count ghost cells */
421   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
422   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
423   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
424   *numGhostCells = 0;
425   for (fs = 0; fs < numFS; ++fs) {
426     IS              faceIS;
427     const PetscInt *faces;
428     PetscInt        numFaces, f, numBdFaces = 0;
429 
430     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
431     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
432     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
433     for (f = 0; f < numFaces; ++f) {
434       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
435     }
436     *numGhostCells += numBdFaces;
437     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
438   }
439   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
440   ierr = PetscMalloc1((depth+1), &depthShift);CHKERRQ(ierr);
441   ierr = PetscMemzero(depthShift, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
442   if (depth >= 0) depthShift[depth] = *numGhostCells;
443   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
444   /* Step 3: Set cone/support sizes for new points */
445   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
446   for (c = cEnd; c < cEnd + *numGhostCells; ++c) {
447     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
448   }
449   for (fs = 0; fs < numFS; ++fs) {
450     IS              faceIS;
451     const PetscInt *faces;
452     PetscInt        numFaces, f;
453 
454     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
455     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
456     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
457     for (f = 0; f < numFaces; ++f) {
458       PetscInt size;
459 
460       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
461       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
462       if (size != 1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
463       ierr = DMPlexSetSupportSize(gdm, faces[f] + *numGhostCells, 2);CHKERRQ(ierr);
464     }
465     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
466     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
467   }
468   /* Step 4: Setup ghosted DM */
469   ierr = DMSetUp(gdm);CHKERRQ(ierr);
470   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
471   /* Step 6: Set cones and supports for new points */
472   ghostCell = cEnd;
473   for (fs = 0; fs < numFS; ++fs) {
474     IS              faceIS;
475     const PetscInt *faces;
476     PetscInt        numFaces, f;
477 
478     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
479     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
480     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
481     for (f = 0; f < numFaces; ++f) {
482       PetscInt newFace = faces[f] + *numGhostCells;
483 
484       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
485       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
486       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
487       ++ghostCell;
488     }
489     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
490     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
491   }
492   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
493   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
494   /* Step 7: Stratify */
495   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
496   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
497   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
498   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
499   ierr = PetscFree(depthShift);CHKERRQ(ierr);
500   PetscFunctionReturn(0);
501 }
502 
503 #undef __FUNCT__
504 #define __FUNCT__ "DMPlexConstructGhostCells"
505 /*@C
506   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
507 
508   Collective on dm
509 
510   Input Parameters:
511 + dm - The original DM
512 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
513 
514   Output Parameters:
515 + numGhostCells - The number of ghost cells added to the DM
516 - dmGhosted - The new DM
517 
518   Note: If no label exists of that name, one will be created marking all boundary faces
519 
520   Level: developer
521 
522 .seealso: DMCreate()
523 @*/
524 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
525 {
526   DM             gdm;
527   DMLabel        label;
528   const char    *name = labelName ? labelName : "Face Sets";
529   PetscInt       dim;
530   PetscErrorCode ierr;
531 
532   PetscFunctionBegin;
533   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
534   PetscValidPointer(numGhostCells, 3);
535   PetscValidPointer(dmGhosted, 4);
536   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
537   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
538   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
539   ierr = DMPlexSetDimension(gdm, dim);CHKERRQ(ierr);
540   ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
541   if (!label) {
542     /* Get label for boundary faces */
543     ierr = DMPlexCreateLabel(dm, name);CHKERRQ(ierr);
544     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
545     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
546   }
547   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
548   ierr = DMSetFromOptions(gdm);CHKERRQ(ierr);
549   *dmGhosted = gdm;
550   PetscFunctionReturn(0);
551 }
552 
553 #undef __FUNCT__
554 #define __FUNCT__ "DMPlexConstructCohesiveCells_Internal"
555 /*
556   We are adding three kinds of points here:
557     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
558     Non-replicated: Points which exist on the fault, but are not replicated
559     Hybrid:         Entirely new points, such as cohesive cells
560 
561   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
562   each depth so that the new split/hybrid points can be inserted as a block.
563 */
564 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
565 {
566   MPI_Comm         comm;
567   IS               valueIS;
568   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
569   const PetscInt  *values;          /* List of depths for which we have replicated points */
570   IS              *splitIS;
571   IS              *unsplitIS;
572   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
573   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
574   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
575   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
576   const PetscInt **splitPoints;        /* Replicated points for each depth */
577   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
578   PetscSection     coordSection;
579   Vec              coordinates;
580   PetscScalar     *coords;
581   PetscInt         depths[4];          /* Depths in the order that plex points are numbered */
582   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
583   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
584   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
585   PetscInt        *depthOffset;        /* Prefix sums of depthShift */
586   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
587   PetscInt        *coneNew, *coneONew, *supportNew;
588   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
589   PetscErrorCode   ierr;
590 
591   PetscFunctionBegin;
592   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
593   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
594   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
595   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
596   depths[0] = depth;
597   depths[1] = 0;
598   depths[2] = depth-1;
599   depths[3] = 1;
600   /* Count split points and add cohesive cells */
601   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
602   ierr = PetscMalloc6(depth+1,&depthMax,depth+1,&depthEnd,depth+1,&depthShift,depth+1,&depthOffset,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
603   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
604   ierr = PetscMemzero(depthShift,  (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
605   ierr = PetscMemzero(depthOffset, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
606   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
607   for (d = 0; d <= depth; ++d) {
608     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
609     depthEnd[d]           = pMaxNew[d];
610     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
611     numSplitPoints[d]     = 0;
612     numUnsplitPoints[d]   = 0;
613     numHybridPoints[d]    = 0;
614     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
615     splitPoints[d]        = NULL;
616     unsplitPoints[d]      = NULL;
617     splitIS[d]            = NULL;
618     unsplitIS[d]          = NULL;
619   }
620   if (label) {
621     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
622     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
623     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
624   }
625   for (sp = 0; sp < numSP; ++sp) {
626     const PetscInt dep = values[sp];
627 
628     if ((dep < 0) || (dep > depth)) continue;
629     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
630     if (splitIS[dep]) {
631       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
632       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
633     }
634     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
635     if (unsplitIS[dep]) {
636       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
637       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
638     }
639   }
640   /* Calculate number of hybrid points */
641   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
642   for (d = 0; d <= depth; ++d) depthShift[d]          = numSplitPoints[d] + numHybridPoints[d];
643   for (d = 1; d <= depth; ++d) depthOffset[depths[d]] = depthOffset[depths[d-1]] + depthShift[depths[d-1]];
644   for (d = 0; d <= depth; ++d) pMaxNew[d]            += depthOffset[d] - numHybridPointsOld[d];
645   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
646   /* Step 3: Set cone/support sizes for new points */
647   for (dep = 0; dep <= depth; ++dep) {
648     for (p = 0; p < numSplitPoints[dep]; ++p) {
649       const PetscInt  oldp   = splitPoints[dep][p];
650       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
651       const PetscInt  splitp = p    + pMaxNew[dep];
652       const PetscInt *support;
653       PetscInt        coneSize, supportSize, qf, qn, qp, e;
654 
655       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
656       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
657       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
658       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
659       if (dep == depth-1) {
660         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
661 
662         /* Add cohesive cells, they are prisms */
663         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
664       } else if (dep == 0) {
665         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
666 
667         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
668         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
669           PetscInt val;
670 
671           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
672           if (val == 1) ++qf;
673           if ((val == 1) || (val ==  (shift + 1))) ++qn;
674           if ((val == 1) || (val == -(shift + 1))) ++qp;
675         }
676         /* Split old vertex: Edges into original vertex and new cohesive edge */
677         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
678         /* Split new vertex: Edges into split vertex and new cohesive edge */
679         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
680         /* Add hybrid edge */
681         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
682         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
683       } else if (dep == dim-2) {
684         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
685 
686         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
687         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
688           PetscInt val;
689 
690           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
691           if (val == dim-1) ++qf;
692           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
693           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
694         }
695         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
696         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
697         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
698         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
699         /* Add hybrid face */
700         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
701         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
702       }
703     }
704   }
705   for (dep = 0; dep <= depth; ++dep) {
706     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
707       const PetscInt  oldp   = unsplitPoints[dep][p];
708       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
709       const PetscInt *support;
710       PetscInt        coneSize, supportSize, qf, e, s;
711 
712       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
713       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
714       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
715       if (dep == 0) {
716         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
717 
718         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
719         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
720           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
721           if (e >= 0) ++qf;
722         }
723         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
724         /* Add hybrid edge */
725         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
726         for (e = 0, qf = 0; e < supportSize; ++e) {
727           PetscInt val;
728 
729           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
730           /* Split and unsplit edges produce hybrid faces */
731           if (val == 1) ++qf;
732           if (val == (shift2 + 1)) ++qf;
733         }
734         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
735       } else if (dep == dim-2) {
736         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
737         PetscInt       val;
738 
739         for (e = 0, qf = 0; e < supportSize; ++e) {
740           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
741           if (val == dim-1) qf += 2;
742           else              ++qf;
743         }
744         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
745         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
746         /* Add hybrid face */
747         for (e = 0, qf = 0; e < supportSize; ++e) {
748           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
749           if (val == dim-1) ++qf;
750         }
751         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
752         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
753       }
754     }
755   }
756   /* Step 4: Setup split DM */
757   ierr = DMSetUp(sdm);CHKERRQ(ierr);
758   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
759   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
760   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
761   /* Step 6: Set cones and supports for new points */
762   for (dep = 0; dep <= depth; ++dep) {
763     for (p = 0; p < numSplitPoints[dep]; ++p) {
764       const PetscInt  oldp   = splitPoints[dep][p];
765       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
766       const PetscInt  splitp = p    + pMaxNew[dep];
767       const PetscInt *cone, *support, *ornt;
768       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
769 
770       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
771       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
772       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
773       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
774       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
775       if (dep == depth-1) {
776         PetscBool       hasUnsplit = PETSC_FALSE;
777         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
778         const PetscInt *supportF;
779 
780         /* Split face:       copy in old face to new face to start */
781         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
782         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
783         /* Split old face:   old vertices/edges in cone so no change */
784         /* Split new face:   new vertices/edges in cone */
785         for (q = 0; q < coneSize; ++q) {
786           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
787           if (v < 0) {
788             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
789             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
790             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthMax, depthEnd, depthShift) /*cone[q] + depthOffset[dep-1]*/;
791             hasUnsplit   = PETSC_TRUE;
792           } else {
793             coneNew[2+q] = v + pMaxNew[dep-1];
794             if (dep > 1) {
795               const PetscInt *econe;
796               PetscInt        econeSize, r, vs, vu;
797 
798               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
799               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
800               for (r = 0; r < econeSize; ++r) {
801                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
802                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
803                 if (vs >= 0) continue;
804                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
805                 hasUnsplit   = PETSC_TRUE;
806               }
807             }
808           }
809         }
810         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
811         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
812         /* Face support */
813         for (s = 0; s < supportSize; ++s) {
814           PetscInt val;
815 
816           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
817           if (val < 0) {
818             /* Split old face:   Replace negative side cell with cohesive cell */
819              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
820           } else {
821             /* Split new face:   Replace positive side cell with cohesive cell */
822             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
823             /* Get orientation for cohesive face */
824             {
825               const PetscInt *ncone, *nconeO;
826               PetscInt        nconeSize, nc;
827 
828               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
829               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
830               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
831               for (nc = 0; nc < nconeSize; ++nc) {
832                 if (ncone[nc] == oldp) {
833                   coneONew[0] = nconeO[nc];
834                   break;
835                 }
836               }
837               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
838             }
839           }
840         }
841         /* Cohesive cell:    Old and new split face, then new cohesive faces */
842         coneNew[0]  = newp;   /* Extracted negative side orientation above */
843         coneNew[1]  = splitp;
844         coneONew[1] = coneONew[0];
845         for (q = 0; q < coneSize; ++q) {
846           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
847           if (v < 0) {
848             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
849             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
850             coneONew[2+q] = 0;
851           } else {
852             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
853           }
854           coneONew[2+q] = 0;
855         }
856         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
857         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
858         /* Label the hybrid cells on the boundary of the split */
859         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
860       } else if (dep == 0) {
861         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
862 
863         /* Split old vertex: Edges in old split faces and new cohesive edge */
864         for (e = 0, qn = 0; e < supportSize; ++e) {
865           PetscInt val;
866 
867           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
868           if ((val == 1) || (val == (shift + 1))) {
869             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
870           }
871         }
872         supportNew[qn] = hybedge;
873         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
874         /* Split new vertex: Edges in new split faces and new cohesive edge */
875         for (e = 0, qp = 0; e < supportSize; ++e) {
876           PetscInt val, edge;
877 
878           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
879           if (val == 1) {
880             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
881             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
882             supportNew[qp++] = edge + pMaxNew[dep+1];
883           } else if (val == -(shift + 1)) {
884             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
885           }
886         }
887         supportNew[qp] = hybedge;
888         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
889         /* Hybrid edge:    Old and new split vertex */
890         coneNew[0] = newp;
891         coneNew[1] = splitp;
892         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
893         for (e = 0, qf = 0; e < supportSize; ++e) {
894           PetscInt val, edge;
895 
896           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
897           if (val == 1) {
898             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
899             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
900             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
901           }
902         }
903         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
904       } else if (dep == dim-2) {
905         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
906 
907         /* Split old edge:   old vertices in cone so no change */
908         /* Split new edge:   new vertices in cone */
909         for (q = 0; q < coneSize; ++q) {
910           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
911           if (v < 0) {
912             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
913             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
914             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthMax, depthEnd, depthShift) /*cone[q] + depthOffset[dep-1]*/;
915           } else {
916             coneNew[q] = v + pMaxNew[dep-1];
917           }
918         }
919         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
920         /* Split old edge: Faces in positive side cells and old split faces */
921         for (e = 0, q = 0; e < supportSize; ++e) {
922           PetscInt val;
923 
924           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
925           if (val == dim-1) {
926             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
927           } else if (val == (shift + dim-1)) {
928             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
929           }
930         }
931         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
932         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
933         /* Split new edge: Faces in negative side cells and new split faces */
934         for (e = 0, q = 0; e < supportSize; ++e) {
935           PetscInt val, face;
936 
937           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
938           if (val == dim-1) {
939             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
940             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
941             supportNew[q++] = face + pMaxNew[dep+1];
942           } else if (val == -(shift + dim-1)) {
943             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
944           }
945         }
946         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
947         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
948         /* Hybrid face */
949         coneNew[0] = newp;
950         coneNew[1] = splitp;
951         for (v = 0; v < coneSize; ++v) {
952           PetscInt vertex;
953           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
954           if (vertex < 0) {
955             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
956             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
957             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
958           } else {
959             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
960           }
961         }
962         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
963         for (e = 0, qf = 0; e < supportSize; ++e) {
964           PetscInt val, face;
965 
966           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
967           if (val == dim-1) {
968             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
969             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
970             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
971           }
972         }
973         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
974       }
975     }
976   }
977   for (dep = 0; dep <= depth; ++dep) {
978     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
979       const PetscInt  oldp   = unsplitPoints[dep][p];
980       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
981       const PetscInt *cone, *support, *ornt;
982       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
983 
984       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
985       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
986       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
987       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
988       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
989       if (dep == 0) {
990         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
991 
992         /* Unsplit vertex */
993         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
994         for (s = 0, q = 0; s < supportSize; ++s) {
995           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthMax, depthEnd, depthShift) /*support[s] + depthOffset[dep+1]*/;
996           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
997           if (e >= 0) {
998             supportNew[q++] = e + pMaxNew[dep+1];
999           }
1000         }
1001         supportNew[q++] = hybedge;
1002         supportNew[q++] = hybedge;
1003         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1004         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1005         /* Hybrid edge */
1006         coneNew[0] = newp;
1007         coneNew[1] = newp;
1008         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1009         for (e = 0, qf = 0; e < supportSize; ++e) {
1010           PetscInt val, edge;
1011 
1012           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1013           if (val == 1) {
1014             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1015             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1016             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1017           } else if  (val ==  (shift2 + 1)) {
1018             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1019             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1020             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1021           }
1022         }
1023         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1024       } else if (dep == dim-2) {
1025         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1026 
1027         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1028         for (f = 0, qf = 0; f < supportSize; ++f) {
1029           PetscInt val, face;
1030 
1031           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1032           if (val == dim-1) {
1033             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1034             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1035             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthMax, depthEnd, depthShift) /*support[f] + depthOffset[dep+1]*/;
1036             supportNew[qf++] = face + pMaxNew[dep+1];
1037           } else {
1038             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthMax, depthEnd, depthShift) /*support[f] + depthOffset[dep+1]*/;
1039           }
1040         }
1041         supportNew[qf++] = hybface;
1042         supportNew[qf++] = hybface;
1043         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1044         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1045         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1046         /* Add hybrid face */
1047         coneNew[0] = newp;
1048         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1049         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1050         coneNew[1] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1051         coneNew[2] = newp;
1052         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1053         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1054         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1055         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1056         for (f = 0, qf = 0; f < supportSize; ++f) {
1057           PetscInt val, face;
1058 
1059           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1060           if (val == dim-1) {
1061             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1062             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1063           }
1064         }
1065         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1066         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1067         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1068       }
1069     }
1070   }
1071   /* Step 6b: Replace split points in negative side cones */
1072   for (sp = 0; sp < numSP; ++sp) {
1073     PetscInt        dep = values[sp];
1074     IS              pIS;
1075     PetscInt        numPoints;
1076     const PetscInt *points;
1077 
1078     if (dep >= 0) continue;
1079     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1080     if (!pIS) continue;
1081     dep  = -dep - shift;
1082     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1083     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1084     for (p = 0; p < numPoints; ++p) {
1085       const PetscInt  oldp = points[p];
1086       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*depthOffset[dep] + oldp*/;
1087       const PetscInt *cone;
1088       PetscInt        coneSize, c;
1089       /* PetscBool       replaced = PETSC_FALSE; */
1090 
1091       /* Negative edge: replace split vertex */
1092       /* Negative cell: replace split face */
1093       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1094       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1095       for (c = 0; c < coneSize; ++c) {
1096         const PetscInt coldp = cone[c] - depthOffset[dep-1];
1097         PetscInt       csplitp, cp, val;
1098 
1099         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1100         if (val == dep-1) {
1101           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1102           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1103           csplitp  = pMaxNew[dep-1] + cp;
1104           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1105           /* replaced = PETSC_TRUE; */
1106         }
1107       }
1108       /* Cells with only a vertex or edge on the submesh have no replacement */
1109       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1110     }
1111     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1112     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1113   }
1114   /* Step 7: Stratify */
1115   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1116   /* Step 8: Coordinates */
1117   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1118   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1119   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1120   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1121   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1122     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthMax, depthEnd, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1123     const PetscInt splitp = pMaxNew[0] + v;
1124     PetscInt       dof, off, soff, d;
1125 
1126     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1127     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1128     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1129     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1130   }
1131   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1132   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1133   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1134   /* Step 10: Labels */
1135   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1136   ierr = DMPlexGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1137   for (dep = 0; dep <= depth; ++dep) {
1138     for (p = 0; p < numSplitPoints[dep]; ++p) {
1139       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthMax, depthEnd, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1140       const PetscInt splitp = pMaxNew[dep] + p;
1141       PetscInt       l;
1142 
1143       for (l = 0; l < numLabels; ++l) {
1144         DMLabel     mlabel;
1145         const char *lname;
1146         PetscInt    val;
1147         PetscBool   isDepth;
1148 
1149         ierr = DMPlexGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1150         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1151         if (isDepth) continue;
1152         ierr = DMPlexGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1153         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1154         if (val >= 0) {
1155           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1156 #if 0
1157           /* Do not put cohesive edges into the label */
1158           if (dep == 0) {
1159             const PetscInt cedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1160             ierr = DMLabelSetValue(mlabel, cedge, val);CHKERRQ(ierr);
1161           } else if (dep == dim-2) {
1162             const PetscInt cface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1163             ierr = DMLabelSetValue(mlabel, cface, val);CHKERRQ(ierr);
1164           }
1165           /* Do not put cohesive faces into the label */
1166 #endif
1167         }
1168       }
1169     }
1170   }
1171   for (sp = 0; sp < numSP; ++sp) {
1172     const PetscInt dep = values[sp];
1173 
1174     if ((dep < 0) || (dep > depth)) continue;
1175     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1176     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1177     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1178     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1179   }
1180   if (label) {
1181     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1182     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1183   }
1184   for (d = 0; d <= depth; ++d) {
1185     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1186     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1187   }
1188   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1189   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1190   ierr = PetscFree6(depthMax, depthEnd, depthShift, depthOffset, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1191   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1192   PetscFunctionReturn(0);
1193 }
1194 
1195 #undef __FUNCT__
1196 #define __FUNCT__ "DMPlexConstructCohesiveCells"
1197 /*@C
1198   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1199 
1200   Collective on dm
1201 
1202   Input Parameters:
1203 + dm - The original DM
1204 - label - The label specifying the boundary faces (this could be auto-generated)
1205 
1206   Output Parameters:
1207 - dmSplit - The new DM
1208 
1209   Level: developer
1210 
1211 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1212 @*/
1213 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1214 {
1215   DM             sdm;
1216   PetscInt       dim;
1217   PetscErrorCode ierr;
1218 
1219   PetscFunctionBegin;
1220   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1221   PetscValidPointer(dmSplit, 3);
1222   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1223   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1224   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1225   ierr = DMPlexSetDimension(sdm, dim);CHKERRQ(ierr);
1226   switch (dim) {
1227   case 2:
1228   case 3:
1229     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1230     break;
1231   default:
1232     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1233   }
1234   *dmSplit = sdm;
1235   PetscFunctionReturn(0);
1236 }
1237 
1238 #undef __FUNCT__
1239 #define __FUNCT__ "DMPlexLabelCohesiveComplete"
1240 /*@
1241   DMPlexLabelCohesiveComplete - Starting with a label marking vertices on an internal surface, we add all other mesh pieces
1242   to complete the surface
1243 
1244   Input Parameters:
1245 + dm - The DM
1246 . label - A DMLabel marking the surface vertices
1247 . flip  - Flag to flip the submesh normal and replace points on the other side
1248 - subdm - The subDM associated with the label, or NULL
1249 
1250   Output Parameter:
1251 . label - A DMLabel marking all surface points
1252 
1253   Level: developer
1254 
1255 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1256 @*/
1257 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, PetscBool flip, DM subdm)
1258 {
1259   DMLabel         depthLabel;
1260   IS              dimIS, subpointIS, facePosIS, faceNegIS;
1261   const PetscInt *points, *subpoints;
1262   const PetscInt  rev   = flip ? -1 : 1;
1263   PetscInt        shift = 100, shift2 = 200, dim, dep, cStart, cEnd, numPoints, numSubpoints, p, val;
1264   PetscErrorCode  ierr;
1265 
1266   PetscFunctionBegin;
1267   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1268   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1269   if (subdm) {
1270     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1271     if (subpointIS) {
1272       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1273       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1274     }
1275   }
1276   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1277   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1278   if (!dimIS) PetscFunctionReturn(0);
1279   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1280   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1281   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1282     const PetscInt *support;
1283     PetscInt        supportSize, s;
1284 
1285     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1286     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1287     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1288     for (s = 0; s < supportSize; ++s) {
1289       const PetscInt *cone, *ornt;
1290       PetscInt        coneSize, c;
1291       PetscBool       pos = PETSC_TRUE;
1292 
1293       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1294       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1295       ierr = DMPlexGetConeOrientation(dm, support[s], &ornt);CHKERRQ(ierr);
1296       for (c = 0; c < coneSize; ++c) {
1297         if (cone[c] == points[p]) {
1298           PetscInt o = ornt[c];
1299 
1300           if (subdm) {
1301             const PetscInt *subcone, *subornt;
1302             PetscInt        subpoint, subface, subconeSize, sc;
1303 
1304             ierr = PetscFindInt(support[s], numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1305             ierr = PetscFindInt(points[p],  numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1306             ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1307             ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1308             ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1309             for (sc = 0; sc < subconeSize; ++sc) {
1310               if (subcone[sc] == subface) {
1311                 o = subornt[0];
1312                 break;
1313               }
1314             }
1315             if (sc >= subconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find point %d in cone for subpoint %d", points[p], subpoint);
1316           }
1317           if (o >= 0) {
1318             ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);
1319             pos  = rev > 0 ? PETSC_TRUE : PETSC_FALSE;
1320           } else {
1321             ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);
1322             pos  = rev > 0 ? PETSC_FALSE : PETSC_TRUE;
1323           }
1324           break;
1325         }
1326       }
1327       if (c == coneSize) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell split face %d support does not have it in the cone", points[p]);
1328       /* Put faces touching the fault in the label */
1329       for (c = 0; c < coneSize; ++c) {
1330         const PetscInt point = cone[c];
1331 
1332         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1333         if (val == -1) {
1334           PetscInt *closure = NULL;
1335           PetscInt  closureSize, cl;
1336 
1337           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1338           for (cl = 0; cl < closureSize*2; cl += 2) {
1339             const PetscInt clp = closure[cl];
1340 
1341             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1342             if ((val >= 0) && (val < dim-1)) {
1343               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1344               break;
1345             }
1346           }
1347           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1348         }
1349       }
1350     }
1351   }
1352   if (subdm) {
1353     if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1354     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1355   }
1356   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1357   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1358   /* Search for other cells/faces/edges connected to the fault by a vertex */
1359   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1360   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1361   if (!dimIS) PetscFunctionReturn(0);
1362   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1363   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1364   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1365     PetscInt *star = NULL;
1366     PetscInt  starSize, s;
1367     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1368 
1369     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1370     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1371     while (again) {
1372       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1373       again = 0;
1374       for (s = 0; s < starSize*2; s += 2) {
1375         const PetscInt  point = star[s];
1376         const PetscInt *cone;
1377         PetscInt        coneSize, c;
1378 
1379         if ((point < cStart) || (point >= cEnd)) continue;
1380         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1381         if (val != -1) continue;
1382         again = again == 1 ? 1 : 2;
1383         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1384         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1385         for (c = 0; c < coneSize; ++c) {
1386           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1387           if (val != -1) {
1388             const PetscInt *ccone;
1389             PetscInt        cconeSize, cc, side;
1390 
1391             if (abs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1392             if (val > 0) side =  1;
1393             else         side = -1;
1394             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1395             /* Mark cell faces which touch the fault */
1396             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1397             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1398             for (cc = 0; cc < cconeSize; ++cc) {
1399               PetscInt *closure = NULL;
1400               PetscInt  closureSize, cl;
1401 
1402               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1403               if (val != -1) continue;
1404               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1405               for (cl = 0; cl < closureSize*2; cl += 2) {
1406                 const PetscInt clp = closure[cl];
1407 
1408                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1409                 if (val == -1) continue;
1410                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1411                 break;
1412               }
1413               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1414             }
1415             again = 1;
1416             break;
1417           }
1418         }
1419       }
1420     }
1421     /* Classify the rest by cell membership */
1422     for (s = 0; s < starSize*2; s += 2) {
1423       const PetscInt point = star[s];
1424 
1425       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1426       if (val == -1) {
1427         PetscInt  *sstar = NULL;
1428         PetscInt   sstarSize, ss;
1429         PetscBool  marked = PETSC_FALSE;
1430 
1431         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1432         for (ss = 0; ss < sstarSize*2; ss += 2) {
1433           const PetscInt spoint = sstar[ss];
1434 
1435           if ((spoint < cStart) || (spoint >= cEnd)) continue;
1436           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1437           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1438           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1439           if (val > 0) {
1440             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1441           } else {
1442             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1443           }
1444           marked = PETSC_TRUE;
1445           break;
1446         }
1447         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1448         if (!marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1449       }
1450     }
1451     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1452   }
1453   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1454   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1455   /* If any faces touching the fault divide cells on either side, split them */
1456   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1457   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1458   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1459   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1460   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1461   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1462   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1463   for (p = 0; p < numPoints; ++p) {
1464     const PetscInt  point = points[p];
1465     const PetscInt *support;
1466     PetscInt        supportSize, valA, valB;
1467 
1468     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1469     if (supportSize != 2) continue;
1470     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1471     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1472     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1473     if ((valA == -1) || (valB == -1)) continue;
1474     if (valA*valB > 0) continue;
1475     /* Split the face */
1476     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1477     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1478     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1479     /* Label its closure:
1480       unmarked: label as unsplit
1481       incident: relabel as split
1482       split:    do nothing
1483     */
1484     {
1485       PetscInt *closure = NULL;
1486       PetscInt  closureSize, cl;
1487 
1488       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1489       for (cl = 0; cl < closureSize*2; cl += 2) {
1490         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1491         if (valA == -1) { /* Mark as unsplit */
1492           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1493           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1494         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1495           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1496           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1497           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1498         }
1499       }
1500       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1501     }
1502   }
1503   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1504   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1505   PetscFunctionReturn(0);
1506 }
1507 
1508 #undef __FUNCT__
1509 #define __FUNCT__ "DMPlexCreateHybridMesh"
1510 /*@C
1511   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1512 
1513   Collective on dm
1514 
1515   Input Parameters:
1516 + dm - The original DM
1517 - labelName - The label specifying the interface vertices
1518 
1519   Output Parameters:
1520 + hybridLabel - The label fully marking the interface
1521 - dmHybrid - The new DM
1522 
1523   Level: developer
1524 
1525 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1526 @*/
1527 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1528 {
1529   DM             idm;
1530   DMLabel        subpointMap, hlabel;
1531   PetscInt       dim;
1532   PetscErrorCode ierr;
1533 
1534   PetscFunctionBegin;
1535   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1536   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1537   PetscValidPointer(dmHybrid, 4);
1538   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1539   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1540   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1541   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1542   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1543   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1544   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, PETSC_FALSE, idm);CHKERRQ(ierr);
1545   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1546   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1547   if (hybridLabel) *hybridLabel = hlabel;
1548   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1549   PetscFunctionReturn(0);
1550 }
1551 
1552 #undef __FUNCT__
1553 #define __FUNCT__ "DMPlexMarkSubmesh_Uninterpolated"
1554 /* Here we need the explicit assumption that:
1555 
1556      For any marked cell, the marked vertices constitute a single face
1557 */
1558 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1559 {
1560   IS               subvertexIS = NULL;
1561   const PetscInt  *subvertices;
1562   PetscInt        *pStart, *pEnd, *pMax, pSize;
1563   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1564   PetscErrorCode   ierr;
1565 
1566   PetscFunctionBegin;
1567   *numFaces = 0;
1568   *nFV      = 0;
1569   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1570   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1571   pSize = PetscMax(depth, dim) + 1;
1572   ierr = PetscMalloc3(pSize,&pStart,pSize,&pEnd,pSize,&pMax);CHKERRQ(ierr);
1573   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1574   for (d = 0; d <= depth; ++d) {
1575     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1576     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1577   }
1578   /* Loop over initial vertices and mark all faces in the collective star() */
1579   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1580   if (subvertexIS) {
1581     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1582     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1583   }
1584   for (v = 0; v < numSubVerticesInitial; ++v) {
1585     const PetscInt vertex = subvertices[v];
1586     PetscInt      *star   = NULL;
1587     PetscInt       starSize, s, numCells = 0, c;
1588 
1589     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1590     for (s = 0; s < starSize*2; s += 2) {
1591       const PetscInt point = star[s];
1592       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1593     }
1594     for (c = 0; c < numCells; ++c) {
1595       const PetscInt cell    = star[c];
1596       PetscInt      *closure = NULL;
1597       PetscInt       closureSize, cl;
1598       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1599 
1600       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1601       if (cellLoc == 2) continue;
1602       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1603       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1604       for (cl = 0; cl < closureSize*2; cl += 2) {
1605         const PetscInt point = closure[cl];
1606         PetscInt       vertexLoc;
1607 
1608         if ((point >= pStart[0]) && (point < pEnd[0])) {
1609           ++numCorners;
1610           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1611           if (vertexLoc == value) closure[faceSize++] = point;
1612         }
1613       }
1614       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1615       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1616       if (faceSize == *nFV) {
1617         const PetscInt *cells = NULL;
1618         PetscInt        numCells, nc;
1619 
1620         ++(*numFaces);
1621         for (cl = 0; cl < faceSize; ++cl) {
1622           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1623         }
1624         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1625         for (nc = 0; nc < numCells; ++nc) {
1626           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1627         }
1628         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1629       }
1630       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1631     }
1632     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1633   }
1634   if (subvertexIS) {
1635     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1636   }
1637   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1638   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1639   PetscFunctionReturn(0);
1640 }
1641 
1642 #undef __FUNCT__
1643 #define __FUNCT__ "DMPlexMarkSubmesh_Interpolated"
1644 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1645 {
1646   IS               subvertexIS;
1647   const PetscInt  *subvertices;
1648   PetscInt        *pStart, *pEnd, *pMax;
1649   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1650   PetscErrorCode   ierr;
1651 
1652   PetscFunctionBegin;
1653   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1654   ierr = PetscMalloc3(dim+1,&pStart,dim+1,&pEnd,dim+1,&pMax);CHKERRQ(ierr);
1655   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1656   for (d = 0; d <= dim; ++d) {
1657     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1658     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1659   }
1660   /* Loop over initial vertices and mark all faces in the collective star() */
1661   ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1662   if (subvertexIS) {
1663     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1664     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1665   }
1666   for (v = 0; v < numSubVerticesInitial; ++v) {
1667     const PetscInt vertex = subvertices[v];
1668     PetscInt      *star   = NULL;
1669     PetscInt       starSize, s, numFaces = 0, f;
1670 
1671     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1672     for (s = 0; s < starSize*2; s += 2) {
1673       const PetscInt point = star[s];
1674       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1675     }
1676     for (f = 0; f < numFaces; ++f) {
1677       const PetscInt face    = star[f];
1678       PetscInt      *closure = NULL;
1679       PetscInt       closureSize, c;
1680       PetscInt       faceLoc;
1681 
1682       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
1683       if (faceLoc == dim-1) continue;
1684       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
1685       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1686       for (c = 0; c < closureSize*2; c += 2) {
1687         const PetscInt point = closure[c];
1688         PetscInt       vertexLoc;
1689 
1690         if ((point >= pStart[0]) && (point < pEnd[0])) {
1691           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1692           if (vertexLoc != value) break;
1693         }
1694       }
1695       if (c == closureSize*2) {
1696         const PetscInt *support;
1697         PetscInt        supportSize, s;
1698 
1699         for (c = 0; c < closureSize*2; c += 2) {
1700           const PetscInt point = closure[c];
1701 
1702           for (d = 0; d < dim; ++d) {
1703             if ((point >= pStart[d]) && (point < pEnd[d])) {
1704               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1705               break;
1706             }
1707           }
1708         }
1709         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
1710         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
1711         for (s = 0; s < supportSize; ++s) {
1712           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1713         }
1714       }
1715       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1716     }
1717     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1718   }
1719   if (subvertexIS) {
1720     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1721   }
1722   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1723   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1724   PetscFunctionReturn(0);
1725 }
1726 
1727 #undef __FUNCT__
1728 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Uninterpolated"
1729 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
1730 {
1731   DMLabel         label = NULL;
1732   const PetscInt *cone;
1733   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize;
1734   PetscErrorCode  ierr;
1735 
1736   PetscFunctionBegin;
1737   *numFaces = 0;
1738   *nFV = 0;
1739   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1740   *subCells = NULL;
1741   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1742   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1743   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1744   if (cMax < 0) PetscFunctionReturn(0);
1745   if (label) {
1746     for (c = cMax; c < cEnd; ++c) {
1747       PetscInt val;
1748 
1749       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1750       if (val == value) {
1751         ++(*numFaces);
1752         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1753       }
1754     }
1755   } else {
1756     *numFaces = cEnd - cMax;
1757     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
1758   }
1759   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
1760   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
1761   for (c = cMax; c < cEnd; ++c) {
1762     const PetscInt *cells;
1763     PetscInt        numCells;
1764 
1765     if (label) {
1766       PetscInt val;
1767 
1768       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1769       if (val != value) continue;
1770     }
1771     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1772     for (p = 0; p < *nFV; ++p) {
1773       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
1774     }
1775     /* Negative face */
1776     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1777     /* Not true in parallel
1778     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
1779     for (p = 0; p < numCells; ++p) {
1780       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
1781       (*subCells)[subc++] = cells[p];
1782     }
1783     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1784     /* Positive face is not included */
1785   }
1786   PetscFunctionReturn(0);
1787 }
1788 
1789 #undef __FUNCT__
1790 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Interpolated"
1791 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DMLabel subpointMap, DM subdm)
1792 {
1793   DMLabel        label = NULL;
1794   PetscInt      *pStart, *pEnd;
1795   PetscInt       dim, cMax, cEnd, c, d;
1796   PetscErrorCode ierr;
1797 
1798   PetscFunctionBegin;
1799   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1800   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1801   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1802   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1803   if (cMax < 0) PetscFunctionReturn(0);
1804   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
1805   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
1806   for (c = cMax; c < cEnd; ++c) {
1807     const PetscInt *cone;
1808     PetscInt       *closure = NULL;
1809     PetscInt        fconeSize, coneSize, closureSize, cl, val;
1810 
1811     if (label) {
1812       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1813       if (val != value) continue;
1814     }
1815     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1816     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1817     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
1818     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
1819     /* Negative face */
1820     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1821     for (cl = 0; cl < closureSize*2; cl += 2) {
1822       const PetscInt point = closure[cl];
1823 
1824       for (d = 0; d <= dim; ++d) {
1825         if ((point >= pStart[d]) && (point < pEnd[d])) {
1826           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1827           break;
1828         }
1829       }
1830     }
1831     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1832     /* Cells -- positive face is not included */
1833     for (cl = 0; cl < 1; ++cl) {
1834       const PetscInt *support;
1835       PetscInt        supportSize, s;
1836 
1837       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
1838       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
1839       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
1840       for (s = 0; s < supportSize; ++s) {
1841         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1842       }
1843     }
1844   }
1845   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
1846   PetscFunctionReturn(0);
1847 }
1848 
1849 #undef __FUNCT__
1850 #define __FUNCT__ "DMPlexGetFaceOrientation"
1851 PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
1852 {
1853   MPI_Comm       comm;
1854   PetscBool      posOrient = PETSC_FALSE;
1855   const PetscInt debug     = 0;
1856   PetscInt       cellDim, faceSize, f;
1857   PetscErrorCode ierr;
1858 
1859   PetscFunctionBegin;
1860   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1861   ierr = DMPlexGetDimension(dm, &cellDim);CHKERRQ(ierr);
1862   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
1863 
1864   if (cellDim == 1 && numCorners == 2) {
1865     /* Triangle */
1866     faceSize  = numCorners-1;
1867     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1868   } else if (cellDim == 2 && numCorners == 3) {
1869     /* Triangle */
1870     faceSize  = numCorners-1;
1871     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1872   } else if (cellDim == 3 && numCorners == 4) {
1873     /* Tetrahedron */
1874     faceSize  = numCorners-1;
1875     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
1876   } else if (cellDim == 1 && numCorners == 3) {
1877     /* Quadratic line */
1878     faceSize  = 1;
1879     posOrient = PETSC_TRUE;
1880   } else if (cellDim == 2 && numCorners == 4) {
1881     /* Quads */
1882     faceSize = 2;
1883     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
1884       posOrient = PETSC_TRUE;
1885     } else if ((indices[0] == 3) && (indices[1] == 0)) {
1886       posOrient = PETSC_TRUE;
1887     } else {
1888       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
1889         posOrient = PETSC_FALSE;
1890       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
1891     }
1892   } else if (cellDim == 2 && numCorners == 6) {
1893     /* Quadratic triangle (I hate this) */
1894     /* Edges are determined by the first 2 vertices (corners of edges) */
1895     const PetscInt faceSizeTri = 3;
1896     PetscInt       sortedIndices[3], i, iFace;
1897     PetscBool      found                    = PETSC_FALSE;
1898     PetscInt       faceVerticesTriSorted[9] = {
1899       0, 3,  4, /* bottom */
1900       1, 4,  5, /* right */
1901       2, 3,  5, /* left */
1902     };
1903     PetscInt       faceVerticesTri[9] = {
1904       0, 3,  4, /* bottom */
1905       1, 4,  5, /* right */
1906       2, 5,  3, /* left */
1907     };
1908 
1909     faceSize = faceSizeTri;
1910     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
1911     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
1912     for (iFace = 0; iFace < 3; ++iFace) {
1913       const PetscInt ii = iFace*faceSizeTri;
1914       PetscInt       fVertex, cVertex;
1915 
1916       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
1917           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
1918         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
1919           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
1920             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
1921               faceVertices[fVertex] = origVertices[cVertex];
1922               break;
1923             }
1924           }
1925         }
1926         found = PETSC_TRUE;
1927         break;
1928       }
1929     }
1930     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
1931     if (posOriented) *posOriented = PETSC_TRUE;
1932     PetscFunctionReturn(0);
1933   } else if (cellDim == 2 && numCorners == 9) {
1934     /* Quadratic quad (I hate this) */
1935     /* Edges are determined by the first 2 vertices (corners of edges) */
1936     const PetscInt faceSizeQuad = 3;
1937     PetscInt       sortedIndices[3], i, iFace;
1938     PetscBool      found                      = PETSC_FALSE;
1939     PetscInt       faceVerticesQuadSorted[12] = {
1940       0, 1,  4, /* bottom */
1941       1, 2,  5, /* right */
1942       2, 3,  6, /* top */
1943       0, 3,  7, /* left */
1944     };
1945     PetscInt       faceVerticesQuad[12] = {
1946       0, 1,  4, /* bottom */
1947       1, 2,  5, /* right */
1948       2, 3,  6, /* top */
1949       3, 0,  7, /* left */
1950     };
1951 
1952     faceSize = faceSizeQuad;
1953     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
1954     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
1955     for (iFace = 0; iFace < 4; ++iFace) {
1956       const PetscInt ii = iFace*faceSizeQuad;
1957       PetscInt       fVertex, cVertex;
1958 
1959       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
1960           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
1961         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
1962           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
1963             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
1964               faceVertices[fVertex] = origVertices[cVertex];
1965               break;
1966             }
1967           }
1968         }
1969         found = PETSC_TRUE;
1970         break;
1971       }
1972     }
1973     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
1974     if (posOriented) *posOriented = PETSC_TRUE;
1975     PetscFunctionReturn(0);
1976   } else if (cellDim == 3 && numCorners == 8) {
1977     /* Hexes
1978        A hex is two oriented quads with the normal of the first
1979        pointing up at the second.
1980 
1981           7---6
1982          /|  /|
1983         4---5 |
1984         | 1-|-2
1985         |/  |/
1986         0---3
1987 
1988         Faces are determined by the first 4 vertices (corners of faces) */
1989     const PetscInt faceSizeHex = 4;
1990     PetscInt       sortedIndices[4], i, iFace;
1991     PetscBool      found                     = PETSC_FALSE;
1992     PetscInt       faceVerticesHexSorted[24] = {
1993       0, 1, 2, 3,  /* bottom */
1994       4, 5, 6, 7,  /* top */
1995       0, 3, 4, 5,  /* front */
1996       2, 3, 5, 6,  /* right */
1997       1, 2, 6, 7,  /* back */
1998       0, 1, 4, 7,  /* left */
1999     };
2000     PetscInt       faceVerticesHex[24] = {
2001       1, 2, 3, 0,  /* bottom */
2002       4, 5, 6, 7,  /* top */
2003       0, 3, 5, 4,  /* front */
2004       3, 2, 6, 5,  /* right */
2005       2, 1, 7, 6,  /* back */
2006       1, 0, 4, 7,  /* left */
2007     };
2008 
2009     faceSize = faceSizeHex;
2010     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2011     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2012     for (iFace = 0; iFace < 6; ++iFace) {
2013       const PetscInt ii = iFace*faceSizeHex;
2014       PetscInt       fVertex, cVertex;
2015 
2016       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2017           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2018           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2019           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2020         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2021           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2022             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2023               faceVertices[fVertex] = origVertices[cVertex];
2024               break;
2025             }
2026           }
2027         }
2028         found = PETSC_TRUE;
2029         break;
2030       }
2031     }
2032     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2033     if (posOriented) *posOriented = PETSC_TRUE;
2034     PetscFunctionReturn(0);
2035   } else if (cellDim == 3 && numCorners == 10) {
2036     /* Quadratic tet */
2037     /* Faces are determined by the first 3 vertices (corners of faces) */
2038     const PetscInt faceSizeTet = 6;
2039     PetscInt       sortedIndices[6], i, iFace;
2040     PetscBool      found                     = PETSC_FALSE;
2041     PetscInt       faceVerticesTetSorted[24] = {
2042       0, 1, 2,  6, 7, 8, /* bottom */
2043       0, 3, 4,  6, 7, 9,  /* front */
2044       1, 4, 5,  7, 8, 9,  /* right */
2045       2, 3, 5,  6, 8, 9,  /* left */
2046     };
2047     PetscInt       faceVerticesTet[24] = {
2048       0, 1, 2,  6, 7, 8, /* bottom */
2049       0, 4, 3,  6, 7, 9,  /* front */
2050       1, 5, 4,  7, 8, 9,  /* right */
2051       2, 3, 5,  8, 6, 9,  /* left */
2052     };
2053 
2054     faceSize = faceSizeTet;
2055     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2056     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2057     for (iFace=0; iFace < 4; ++iFace) {
2058       const PetscInt ii = iFace*faceSizeTet;
2059       PetscInt       fVertex, cVertex;
2060 
2061       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2062           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2063           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2064           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2065         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2066           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2067             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2068               faceVertices[fVertex] = origVertices[cVertex];
2069               break;
2070             }
2071           }
2072         }
2073         found = PETSC_TRUE;
2074         break;
2075       }
2076     }
2077     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2078     if (posOriented) *posOriented = PETSC_TRUE;
2079     PetscFunctionReturn(0);
2080   } else if (cellDim == 3 && numCorners == 27) {
2081     /* Quadratic hexes (I hate this)
2082        A hex is two oriented quads with the normal of the first
2083        pointing up at the second.
2084 
2085          7---6
2086         /|  /|
2087        4---5 |
2088        | 3-|-2
2089        |/  |/
2090        0---1
2091 
2092        Faces are determined by the first 4 vertices (corners of faces) */
2093     const PetscInt faceSizeQuadHex = 9;
2094     PetscInt       sortedIndices[9], i, iFace;
2095     PetscBool      found                         = PETSC_FALSE;
2096     PetscInt       faceVerticesQuadHexSorted[54] = {
2097       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2098       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2099       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2100       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2101       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2102       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2103     };
2104     PetscInt       faceVerticesQuadHex[54] = {
2105       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2106       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2107       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2108       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2109       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2110       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2111     };
2112 
2113     faceSize = faceSizeQuadHex;
2114     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2115     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2116     for (iFace = 0; iFace < 6; ++iFace) {
2117       const PetscInt ii = iFace*faceSizeQuadHex;
2118       PetscInt       fVertex, cVertex;
2119 
2120       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2121           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2122           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2123           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2124         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2125           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2126             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2127               faceVertices[fVertex] = origVertices[cVertex];
2128               break;
2129             }
2130           }
2131         }
2132         found = PETSC_TRUE;
2133         break;
2134       }
2135     }
2136     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2137     if (posOriented) *posOriented = PETSC_TRUE;
2138     PetscFunctionReturn(0);
2139   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2140   if (!posOrient) {
2141     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2142     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2143   } else {
2144     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2145     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2146   }
2147   if (posOriented) *posOriented = posOrient;
2148   PetscFunctionReturn(0);
2149 }
2150 
2151 #undef __FUNCT__
2152 #define __FUNCT__ "DMPlexGetOrientedFace"
2153 /*
2154     Given a cell and a face, as a set of vertices,
2155       return the oriented face, as a set of vertices, in faceVertices
2156     The orientation is such that the face normal points out of the cell
2157 */
2158 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2159 {
2160   const PetscInt *cone = NULL;
2161   PetscInt        coneSize, v, f, v2;
2162   PetscInt        oppositeVertex = -1;
2163   PetscErrorCode  ierr;
2164 
2165   PetscFunctionBegin;
2166   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2167   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2168   for (v = 0, v2 = 0; v < coneSize; ++v) {
2169     PetscBool found = PETSC_FALSE;
2170 
2171     for (f = 0; f < faceSize; ++f) {
2172       if (face[f] == cone[v]) {
2173         found = PETSC_TRUE; break;
2174       }
2175     }
2176     if (found) {
2177       indices[v2]      = v;
2178       origVertices[v2] = cone[v];
2179       ++v2;
2180     } else {
2181       oppositeVertex = v;
2182     }
2183   }
2184   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 #undef __FUNCT__
2189 #define __FUNCT__ "DMPlexInsertFace_Internal"
2190 /*
2191   DMPlexInsertFace_Internal - Puts a face into the mesh
2192 
2193   Not collective
2194 
2195   Input Parameters:
2196   + dm              - The DMPlex
2197   . numFaceVertex   - The number of vertices in the face
2198   . faceVertices    - The vertices in the face for dm
2199   . subfaceVertices - The vertices in the face for subdm
2200   . numCorners      - The number of vertices in the cell
2201   . cell            - A cell in dm containing the face
2202   . subcell         - A cell in subdm containing the face
2203   . firstFace       - First face in the mesh
2204   - newFacePoint    - Next face in the mesh
2205 
2206   Output Parameters:
2207   . newFacePoint - Contains next face point number on input, updated on output
2208 
2209   Level: developer
2210 */
2211 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2212 {
2213   MPI_Comm        comm;
2214   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2215   const PetscInt *faces;
2216   PetscInt        numFaces, coneSize;
2217   PetscErrorCode  ierr;
2218 
2219   PetscFunctionBegin;
2220   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2221   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2222   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2223 #if 0
2224   /* Cannot use this because support() has not been constructed yet */
2225   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2226 #else
2227   {
2228     PetscInt f;
2229 
2230     numFaces = 0;
2231     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2232     for (f = firstFace; f < *newFacePoint; ++f) {
2233       PetscInt dof, off, d;
2234 
2235       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2236       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2237       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2238       for (d = 0; d < dof; ++d) {
2239         const PetscInt p = submesh->cones[off+d];
2240         PetscInt       v;
2241 
2242         for (v = 0; v < numFaceVertices; ++v) {
2243           if (subfaceVertices[v] == p) break;
2244         }
2245         if (v == numFaceVertices) break;
2246       }
2247       if (d == dof) {
2248         numFaces               = 1;
2249         ((PetscInt*) faces)[0] = f;
2250       }
2251     }
2252   }
2253 #endif
2254   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2255   else if (numFaces == 1) {
2256     /* Add the other cell neighbor for this face */
2257     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2258   } else {
2259     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2260     PetscBool posOriented;
2261 
2262     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2263     origVertices        = &orientedVertices[numFaceVertices];
2264     indices             = &orientedVertices[numFaceVertices*2];
2265     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2266     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2267     /* TODO: I know that routine should return a permutation, not the indices */
2268     for (v = 0; v < numFaceVertices; ++v) {
2269       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2270       for (ov = 0; ov < numFaceVertices; ++ov) {
2271         if (orientedVertices[ov] == vertex) {
2272           orientedSubVertices[ov] = subvertex;
2273           break;
2274         }
2275       }
2276       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2277     }
2278     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2279     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2280     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2281     ++(*newFacePoint);
2282   }
2283 #if 0
2284   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2285 #else
2286   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2287 #endif
2288   PetscFunctionReturn(0);
2289 }
2290 
2291 #undef __FUNCT__
2292 #define __FUNCT__ "DMPlexCreateSubmesh_Uninterpolated"
2293 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2294 {
2295   MPI_Comm        comm;
2296   DMLabel         subpointMap;
2297   IS              subvertexIS,  subcellIS;
2298   const PetscInt *subVertices, *subCells;
2299   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2300   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2301   PetscInt        vStart, vEnd, c, f;
2302   PetscErrorCode  ierr;
2303 
2304   PetscFunctionBegin;
2305   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2306   /* Create subpointMap which marks the submesh */
2307   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2308   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2309   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2310   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2311   /* Setup chart */
2312   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2313   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2314   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2315   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2316   /* Set cone sizes */
2317   firstSubVertex = numSubCells;
2318   firstSubFace   = numSubCells+numSubVertices;
2319   newFacePoint   = firstSubFace;
2320   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2321   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2322   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2323   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2324   for (c = 0; c < numSubCells; ++c) {
2325     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2326   }
2327   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2328     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2329   }
2330   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2331   /* Create face cones */
2332   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2333   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2334   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2335   for (c = 0; c < numSubCells; ++c) {
2336     const PetscInt cell    = subCells[c];
2337     const PetscInt subcell = c;
2338     PetscInt      *closure = NULL;
2339     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2340 
2341     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2342     for (cl = 0; cl < closureSize*2; cl += 2) {
2343       const PetscInt point = closure[cl];
2344       PetscInt       subVertex;
2345 
2346       if ((point >= vStart) && (point < vEnd)) {
2347         ++numCorners;
2348         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2349         if (subVertex >= 0) {
2350           closure[faceSize] = point;
2351           subface[faceSize] = firstSubVertex+subVertex;
2352           ++faceSize;
2353         }
2354       }
2355     }
2356     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2357     if (faceSize == nFV) {
2358       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2359     }
2360     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2361   }
2362   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2363   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2364   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2365   /* Build coordinates */
2366   {
2367     PetscSection coordSection, subCoordSection;
2368     Vec          coordinates, subCoordinates;
2369     PetscScalar *coords, *subCoords;
2370     PetscInt     numComp, coordSize, v;
2371 
2372     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2373     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2374     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2375     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2376     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2377     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2378     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2379     for (v = 0; v < numSubVertices; ++v) {
2380       const PetscInt vertex    = subVertices[v];
2381       const PetscInt subvertex = firstSubVertex+v;
2382       PetscInt       dof;
2383 
2384       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2385       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2386       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2387     }
2388     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2389     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2390     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2391     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2392     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2393     if (coordSize) {
2394       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2395       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2396       for (v = 0; v < numSubVertices; ++v) {
2397         const PetscInt vertex    = subVertices[v];
2398         const PetscInt subvertex = firstSubVertex+v;
2399         PetscInt       dof, off, sdof, soff, d;
2400 
2401         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2402         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2403         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2404         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2405         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2406         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2407       }
2408       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2409       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2410     }
2411     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2412     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2413   }
2414   /* Cleanup */
2415   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2416   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2417   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2418   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2419   PetscFunctionReturn(0);
2420 }
2421 
2422 #undef __FUNCT__
2423 #define __FUNCT__ "DMPlexCreateSubmesh_Interpolated"
2424 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2425 {
2426   MPI_Comm         comm;
2427   DMLabel          subpointMap;
2428   IS              *subpointIS;
2429   const PetscInt **subpoints;
2430   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *coneONew;
2431   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2432   PetscErrorCode   ierr;
2433 
2434   PetscFunctionBegin;
2435   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2436   /* Create subpointMap which marks the submesh */
2437   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2438   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2439   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2440   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Interpolated(dm, vertexLabel, value, subpointMap, subdm);CHKERRQ(ierr);}
2441   /* Setup chart */
2442   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2443   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
2444   for (d = 0; d <= dim; ++d) {
2445     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2446     totSubPoints += numSubPoints[d];
2447   }
2448   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2449   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2450   /* Set cone sizes */
2451   firstSubPoint[dim] = 0;
2452   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2453   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2454   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2455   for (d = 0; d <= dim; ++d) {
2456     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2457     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2458   }
2459   for (d = 0; d <= dim; ++d) {
2460     for (p = 0; p < numSubPoints[d]; ++p) {
2461       const PetscInt  point    = subpoints[d][p];
2462       const PetscInt  subpoint = firstSubPoint[d] + p;
2463       const PetscInt *cone;
2464       PetscInt        coneSize, coneSizeNew, c, val;
2465 
2466       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2467       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2468       if (d == dim) {
2469         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2470         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2471           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2472           if (val >= 0) coneSizeNew++;
2473         }
2474         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2475       }
2476     }
2477   }
2478   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2479   /* Set cones */
2480   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2481   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&coneONew);CHKERRQ(ierr);
2482   for (d = 0; d <= dim; ++d) {
2483     for (p = 0; p < numSubPoints[d]; ++p) {
2484       const PetscInt  point    = subpoints[d][p];
2485       const PetscInt  subpoint = firstSubPoint[d] + p;
2486       const PetscInt *cone, *ornt;
2487       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2488 
2489       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2490       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2491       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2492       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2493       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2494         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2495         if (subc >= 0) {
2496           coneNew[coneSizeNew]  = firstSubPoint[d-1] + subc;
2497           coneONew[coneSizeNew] = ornt[c];
2498           ++coneSizeNew;
2499         }
2500       }
2501       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2502       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2503       ierr = DMPlexSetConeOrientation(subdm, subpoint, coneONew);CHKERRQ(ierr);
2504     }
2505   }
2506   ierr = PetscFree2(coneNew,coneONew);CHKERRQ(ierr);
2507   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2508   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2509   /* Build coordinates */
2510   {
2511     PetscSection coordSection, subCoordSection;
2512     Vec          coordinates, subCoordinates;
2513     PetscScalar *coords, *subCoords;
2514     PetscInt     numComp, coordSize;
2515 
2516     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2517     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2518     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2519     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2520     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2521     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2522     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2523     for (v = 0; v < numSubPoints[0]; ++v) {
2524       const PetscInt vertex    = subpoints[0][v];
2525       const PetscInt subvertex = firstSubPoint[0]+v;
2526       PetscInt       dof;
2527 
2528       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2529       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2530       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2531     }
2532     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2533     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2534     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2535     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2536     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2537     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2538     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2539     for (v = 0; v < numSubPoints[0]; ++v) {
2540       const PetscInt vertex    = subpoints[0][v];
2541       const PetscInt subvertex = firstSubPoint[0]+v;
2542       PetscInt dof, off, sdof, soff, d;
2543 
2544       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2545       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2546       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2547       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2548       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2549       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2550     }
2551     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2552     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2553     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2554     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2555   }
2556   /* Cleanup */
2557   for (d = 0; d <= dim; ++d) {
2558     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2559     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2560   }
2561   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2562   PetscFunctionReturn(0);
2563 }
2564 
2565 #undef __FUNCT__
2566 #define __FUNCT__ "DMPlexCreateSubmesh"
2567 /*@
2568   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
2569 
2570   Input Parameters:
2571 + dm           - The original mesh
2572 . vertexLabel  - The DMLabel marking vertices contained in the surface
2573 - value        - The label value to use
2574 
2575   Output Parameter:
2576 . subdm - The surface mesh
2577 
2578   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2579 
2580   Level: developer
2581 
2582 .seealso: DMPlexGetSubpointMap(), DMPlexGetLabel(), DMLabelSetValue()
2583 @*/
2584 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
2585 {
2586   PetscInt       dim, depth;
2587   PetscErrorCode ierr;
2588 
2589   PetscFunctionBegin;
2590   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2591   PetscValidPointer(subdm, 3);
2592   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2593   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2594   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2595   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2596   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2597   if (depth == dim) {
2598     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2599   } else {
2600     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2601   }
2602   PetscFunctionReturn(0);
2603 }
2604 
2605 #undef __FUNCT__
2606 #define __FUNCT__ "DMPlexFilterPoint_Internal"
2607 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2608 {
2609   PetscInt       subPoint;
2610   PetscErrorCode ierr;
2611 
2612   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2613   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2614 }
2615 
2616 #undef __FUNCT__
2617 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Uninterpolated"
2618 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
2619 {
2620   MPI_Comm        comm;
2621   DMLabel         subpointMap;
2622   IS              subvertexIS;
2623   const PetscInt *subVertices;
2624   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
2625   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
2626   PetscInt        cMax, c, f;
2627   PetscErrorCode  ierr;
2628 
2629   PetscFunctionBegin;
2630   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
2631   /* Create subpointMap which marks the submesh */
2632   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2633   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2634   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2635   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
2636   /* Setup chart */
2637   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2638   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2639   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2640   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2641   /* Set cone sizes */
2642   firstSubVertex = numSubCells;
2643   firstSubFace   = numSubCells+numSubVertices;
2644   newFacePoint   = firstSubFace;
2645   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2646   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2647   for (c = 0; c < numSubCells; ++c) {
2648     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2649   }
2650   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2651     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2652   }
2653   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2654   /* Create face cones */
2655   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2656   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2657   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2658   for (c = 0; c < numSubCells; ++c) {
2659     const PetscInt  cell    = subCells[c];
2660     const PetscInt  subcell = c;
2661     const PetscInt *cone, *cells;
2662     PetscInt        numCells, subVertex, p, v;
2663 
2664     if (cell < cMax) continue;
2665     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2666     for (v = 0; v < nFV; ++v) {
2667       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2668       subface[v] = firstSubVertex+subVertex;
2669     }
2670     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
2671     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
2672     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2673     /* Not true in parallel
2674     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2675     for (p = 0; p < numCells; ++p) {
2676       PetscInt negsubcell;
2677 
2678       if (cells[p] >= cMax) continue;
2679       /* I know this is a crap search */
2680       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
2681         if (subCells[negsubcell] == cells[p]) break;
2682       }
2683       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
2684       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
2685     }
2686     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2687     ++newFacePoint;
2688   }
2689   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2690   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2691   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2692   /* Build coordinates */
2693   {
2694     PetscSection coordSection, subCoordSection;
2695     Vec          coordinates, subCoordinates;
2696     PetscScalar *coords, *subCoords;
2697     PetscInt     numComp, coordSize, v;
2698 
2699     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2700     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2701     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2702     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2703     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2704     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2705     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2706     for (v = 0; v < numSubVertices; ++v) {
2707       const PetscInt vertex    = subVertices[v];
2708       const PetscInt subvertex = firstSubVertex+v;
2709       PetscInt       dof;
2710 
2711       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2712       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2713       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2714     }
2715     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2716     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2717     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2718     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2719     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2720     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2721     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2722     for (v = 0; v < numSubVertices; ++v) {
2723       const PetscInt vertex    = subVertices[v];
2724       const PetscInt subvertex = firstSubVertex+v;
2725       PetscInt       dof, off, sdof, soff, d;
2726 
2727       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2728       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2729       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2730       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2731       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2732       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2733     }
2734     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2735     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2736     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2737     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2738   }
2739   /* Build SF */
2740   CHKMEMQ;
2741   {
2742     PetscSF            sfPoint, sfPointSub;
2743     const PetscSFNode *remotePoints;
2744     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2745     const PetscInt    *localPoints;
2746     PetscInt          *slocalPoints;
2747     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
2748     PetscMPIInt        rank;
2749 
2750     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2751     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2752     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2753     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2754     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2755     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2756     if (numRoots >= 0) {
2757       /* Only vertices should be shared */
2758       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
2759       for (p = 0; p < pEnd-pStart; ++p) {
2760         newLocalPoints[p].rank  = -2;
2761         newLocalPoints[p].index = -2;
2762       }
2763       /* Set subleaves */
2764       for (l = 0; l < numLeaves; ++l) {
2765         const PetscInt point    = localPoints[l];
2766         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2767 
2768         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
2769         if (subPoint < 0) continue;
2770         newLocalPoints[point-pStart].rank  = rank;
2771         newLocalPoints[point-pStart].index = subPoint;
2772         ++numSubLeaves;
2773       }
2774       /* Must put in owned subpoints */
2775       for (p = pStart; p < pEnd; ++p) {
2776         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
2777 
2778         if (subPoint < 0) {
2779           newOwners[p-pStart].rank  = -3;
2780           newOwners[p-pStart].index = -3;
2781         } else {
2782           newOwners[p-pStart].rank  = rank;
2783           newOwners[p-pStart].index = subPoint;
2784         }
2785       }
2786       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2787       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2788       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2789       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2790       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
2791       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
2792       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
2793         const PetscInt point    = localPoints[l];
2794         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2795 
2796         if (subPoint < 0) continue;
2797         if (newLocalPoints[point].rank == rank) {++ll; continue;}
2798         slocalPoints[sl]        = subPoint;
2799         sremotePoints[sl].rank  = newLocalPoints[point].rank;
2800         sremotePoints[sl].index = newLocalPoints[point].index;
2801         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
2802         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
2803         ++sl;
2804       }
2805       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
2806       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
2807       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
2808     }
2809   }
2810   CHKMEMQ;
2811   /* Cleanup */
2812   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2813   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2814   ierr = PetscFree(subCells);CHKERRQ(ierr);
2815   PetscFunctionReturn(0);
2816 }
2817 
2818 #undef __FUNCT__
2819 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Interpolated"
2820 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char label[], PetscInt value, DM subdm)
2821 {
2822   MPI_Comm         comm;
2823   DMLabel          subpointMap;
2824   IS              *subpointIS;
2825   const PetscInt **subpoints;
2826   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
2827   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2828   PetscErrorCode   ierr;
2829 
2830   PetscFunctionBegin;
2831   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2832   /* Create subpointMap which marks the submesh */
2833   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2834   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2835   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2836   ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);
2837   /* Setup chart */
2838   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2839   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
2840   for (d = 0; d <= dim; ++d) {
2841     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2842     totSubPoints += numSubPoints[d];
2843   }
2844   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2845   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2846   /* Set cone sizes */
2847   firstSubPoint[dim] = 0;
2848   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2849   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2850   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2851   for (d = 0; d <= dim; ++d) {
2852     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2853     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2854   }
2855   for (d = 0; d <= dim; ++d) {
2856     for (p = 0; p < numSubPoints[d]; ++p) {
2857       const PetscInt  point    = subpoints[d][p];
2858       const PetscInt  subpoint = firstSubPoint[d] + p;
2859       const PetscInt *cone;
2860       PetscInt        coneSize, coneSizeNew, c, val;
2861 
2862       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2863       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2864       if (d == dim) {
2865         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2866         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2867           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2868           if (val >= 0) coneSizeNew++;
2869         }
2870         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2871       }
2872     }
2873   }
2874   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2875   /* Set cones */
2876   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2877   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
2878   for (d = 0; d <= dim; ++d) {
2879     for (p = 0; p < numSubPoints[d]; ++p) {
2880       const PetscInt  point    = subpoints[d][p];
2881       const PetscInt  subpoint = firstSubPoint[d] + p;
2882       const PetscInt *cone, *ornt;
2883       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2884 
2885       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2886       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2887       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2888       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2889       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2890         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2891         if (subc >= 0) {
2892           coneNew[coneSizeNew]   = firstSubPoint[d-1] + subc;
2893           orntNew[coneSizeNew++] = ornt[c];
2894         }
2895       }
2896       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2897       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2898       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
2899     }
2900   }
2901   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
2902   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2903   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2904   /* Build coordinates */
2905   {
2906     PetscSection coordSection, subCoordSection;
2907     Vec          coordinates, subCoordinates;
2908     PetscScalar *coords, *subCoords;
2909     PetscInt     numComp, coordSize;
2910 
2911     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2912     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2913     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2914     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2915     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2916     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2917     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2918     for (v = 0; v < numSubPoints[0]; ++v) {
2919       const PetscInt vertex    = subpoints[0][v];
2920       const PetscInt subvertex = firstSubPoint[0]+v;
2921       PetscInt       dof;
2922 
2923       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2924       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2925       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2926     }
2927     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2928     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2929     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2930     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2931     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2932     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2933     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2934     for (v = 0; v < numSubPoints[0]; ++v) {
2935       const PetscInt vertex    = subpoints[0][v];
2936       const PetscInt subvertex = firstSubPoint[0]+v;
2937       PetscInt dof, off, sdof, soff, d;
2938 
2939       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2940       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2941       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2942       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2943       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2944       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2945     }
2946     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2947     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2948     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2949     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2950   }
2951   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
2952   {
2953     PetscSF            sfPoint, sfPointSub;
2954     IS                 subpIS;
2955     const PetscSFNode *remotePoints;
2956     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2957     const PetscInt    *localPoints, *subpoints;
2958     PetscInt          *slocalPoints;
2959     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
2960     PetscMPIInt        rank;
2961 
2962     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2963     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2964     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2965     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2966     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
2967     ierr = DMPlexCreateSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
2968     if (subpIS) {
2969       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
2970       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
2971     }
2972     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2973     if (numRoots >= 0) {
2974       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
2975       for (p = 0; p < pEnd-pStart; ++p) {
2976         newLocalPoints[p].rank  = -2;
2977         newLocalPoints[p].index = -2;
2978       }
2979       /* Set subleaves */
2980       for (l = 0; l < numLeaves; ++l) {
2981         const PetscInt point    = localPoints[l];
2982         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
2983 
2984         if (subpoint < 0) continue;
2985         newLocalPoints[point-pStart].rank  = rank;
2986         newLocalPoints[point-pStart].index = subpoint;
2987         ++numSubleaves;
2988       }
2989       /* Must put in owned subpoints */
2990       for (p = pStart; p < pEnd; ++p) {
2991         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
2992 
2993         if (subpoint < 0) {
2994           newOwners[p-pStart].rank  = -3;
2995           newOwners[p-pStart].index = -3;
2996         } else {
2997           newOwners[p-pStart].rank  = rank;
2998           newOwners[p-pStart].index = subpoint;
2999         }
3000       }
3001       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3002       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3003       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3004       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3005       ierr = PetscMalloc(numSubleaves * sizeof(PetscInt),    &slocalPoints);CHKERRQ(ierr);
3006       ierr = PetscMalloc(numSubleaves * sizeof(PetscSFNode), &sremotePoints);CHKERRQ(ierr);
3007       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3008         const PetscInt point    = localPoints[l];
3009         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3010 
3011         if (subpoint < 0) continue;
3012         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3013         slocalPoints[sl]        = subpoint;
3014         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3015         sremotePoints[sl].index = newLocalPoints[point].index;
3016         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3017         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3018         ++sl;
3019       }
3020       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3021       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3022       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3023     }
3024     if (subpIS) {
3025       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3026       ierr = ISDestroy(&subpIS);CHKERRQ(ierr);
3027     }
3028   }
3029   /* Cleanup */
3030   for (d = 0; d <= dim; ++d) {
3031     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3032     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3033   }
3034   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3035   PetscFunctionReturn(0);
3036 }
3037 
3038 #undef __FUNCT__
3039 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh"
3040 /*
3041   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3042 
3043   Input Parameters:
3044 + dm          - The original mesh
3045 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3046 . label       - A label name, or NULL
3047 - value  - A label value
3048 
3049   Output Parameter:
3050 . subdm - The surface mesh
3051 
3052   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3053 
3054   Level: developer
3055 
3056 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3057 */
3058 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3059 {
3060   PetscInt       dim, depth;
3061   PetscErrorCode ierr;
3062 
3063   PetscFunctionBegin;
3064   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3065   PetscValidPointer(subdm, 5);
3066   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
3067   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3068   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3069   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3070   ierr = DMPlexSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3071   if (depth == dim) {
3072     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3073   } else {
3074     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3075   }
3076   PetscFunctionReturn(0);
3077 }
3078 
3079 #undef __FUNCT__
3080 #define __FUNCT__ "DMPlexGetSubpointMap"
3081 /*@
3082   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3083 
3084   Input Parameter:
3085 . dm - The submesh DM
3086 
3087   Output Parameter:
3088 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3089 
3090   Level: developer
3091 
3092 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3093 @*/
3094 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3095 {
3096   DM_Plex *mesh = (DM_Plex*) dm->data;
3097 
3098   PetscFunctionBegin;
3099   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3100   PetscValidPointer(subpointMap, 2);
3101   *subpointMap = mesh->subpointMap;
3102   PetscFunctionReturn(0);
3103 }
3104 
3105 #undef __FUNCT__
3106 #define __FUNCT__ "DMPlexSetSubpointMap"
3107 /* Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh() */
3108 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3109 {
3110   DM_Plex       *mesh = (DM_Plex *) dm->data;
3111   DMLabel        tmp;
3112   PetscErrorCode ierr;
3113 
3114   PetscFunctionBegin;
3115   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3116   tmp  = mesh->subpointMap;
3117   mesh->subpointMap = subpointMap;
3118   ++mesh->subpointMap->refct;
3119   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3120   PetscFunctionReturn(0);
3121 }
3122 
3123 #undef __FUNCT__
3124 #define __FUNCT__ "DMPlexCreateSubpointIS"
3125 /*@
3126   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
3127 
3128   Input Parameter:
3129 . dm - The submesh DM
3130 
3131   Output Parameter:
3132 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3133 
3134   Note: This is IS is guaranteed to be sorted by the construction of the submesh
3135 
3136   Level: developer
3137 
3138 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3139 @*/
3140 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3141 {
3142   MPI_Comm        comm;
3143   DMLabel         subpointMap;
3144   IS              is;
3145   const PetscInt *opoints;
3146   PetscInt       *points, *depths;
3147   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3148   PetscErrorCode  ierr;
3149 
3150   PetscFunctionBegin;
3151   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3152   PetscValidPointer(subpointIS, 2);
3153   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3154   *subpointIS = NULL;
3155   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3156   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3157   if (subpointMap && depth >= 0) {
3158     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3159     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3160     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3161     depths[0] = depth;
3162     depths[1] = 0;
3163     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3164     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3165     for(d = 0, off = 0; d <= depth; ++d) {
3166       const PetscInt dep = depths[d];
3167 
3168       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3169       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3170       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3171         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3172       } else {
3173         if (!n) {
3174           if (d == 0) {
3175             /* Missing cells */
3176             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3177           } else {
3178             /* Missing faces */
3179             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3180           }
3181         }
3182       }
3183       if (n) {
3184         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3185         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3186         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3187         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3188         ierr = ISDestroy(&is);CHKERRQ(ierr);
3189       }
3190     }
3191     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3192     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3193     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3194   }
3195   PetscFunctionReturn(0);
3196 }
3197