xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 2b26979f2976cb8840d2aed9f03519841c1cab77)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <petscsf.h>
3 
4 #undef __FUNCT__
5 #define __FUNCT__ "DMPlexMarkBoundaryFaces_Internal"
6 PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt cellHeight, DMLabel label)
7 {
8   PetscInt       fStart, fEnd, f;
9   PetscErrorCode ierr;
10 
11   PetscFunctionBegin;
12   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
13   ierr = DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd);CHKERRQ(ierr);
14   for (f = fStart; f < fEnd; ++f) {
15     PetscInt supportSize;
16 
17     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
18     if (supportSize == 1) {ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);}
19   }
20   PetscFunctionReturn(0);
21 }
22 
23 #undef __FUNCT__
24 #define __FUNCT__ "DMPlexMarkBoundaryFaces"
25 /*@
26   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
27 
28   Not Collective
29 
30   Input Parameter:
31 . dm - The original DM
32 
33   Output Parameter:
34 . label - The DMLabel marking boundary faces with value 1
35 
36   Level: developer
37 
38 .seealso: DMLabelCreate(), DMPlexCreateLabel()
39 @*/
40 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
41 {
42   PetscErrorCode ierr;
43 
44   PetscFunctionBegin;
45   ierr = DMPlexMarkBoundaryFaces_Internal(dm, 0, label);CHKERRQ(ierr);
46   PetscFunctionReturn(0);
47 }
48 
49 #undef __FUNCT__
50 #define __FUNCT__ "DMPlexLabelComplete"
51 /*@
52   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
53 
54   Input Parameters:
55 + dm - The DM
56 - label - A DMLabel marking the surface points
57 
58   Output Parameter:
59 . label - A DMLabel marking all surface points in the transitive closure
60 
61   Level: developer
62 
63 .seealso: DMPlexLabelCohesiveComplete()
64 @*/
65 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
66 {
67   IS              valueIS;
68   const PetscInt *values;
69   PetscInt        numValues, v;
70   PetscErrorCode  ierr;
71 
72   PetscFunctionBegin;
73   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
74   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
75   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
76   for (v = 0; v < numValues; ++v) {
77     IS              pointIS;
78     const PetscInt *points;
79     PetscInt        numPoints, p;
80 
81     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
82     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
83     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
84     for (p = 0; p < numPoints; ++p) {
85       PetscInt *closure = NULL;
86       PetscInt  closureSize, c;
87 
88       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
89       for (c = 0; c < closureSize*2; c += 2) {
90         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
91       }
92       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
93     }
94     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
95     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
96   }
97   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
98   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
99   PetscFunctionReturn(0);
100 }
101 
102 #undef __FUNCT__
103 #define __FUNCT__ "DMPlexLabelAddCells"
104 /*@
105   DMPlexLabelAddCells - Starting with a label marking faces on a surface, we add a cell for each face
106 
107   Input Parameters:
108 + dm - The DM
109 - label - A DMLabel marking the surface points
110 
111   Output Parameter:
112 . label - A DMLabel incorporating cells
113 
114   Level: developer
115 
116   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
117 
118 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
119 @*/
120 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
121 {
122   IS              valueIS;
123   const PetscInt *values;
124   PetscInt        numValues, v, cStart, cEnd;
125   PetscErrorCode  ierr;
126 
127   PetscFunctionBegin;
128   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
129   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
130   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
131   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
132   for (v = 0; v < numValues; ++v) {
133     IS              pointIS;
134     const PetscInt *points;
135     PetscInt        numPoints, p;
136 
137     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
138     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
139     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
140     for (p = 0; p < numPoints; ++p) {
141       PetscInt *closure = NULL;
142       PetscInt  closureSize, point;
143 
144       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
145       point = closure[(closureSize-1)*2];
146       if ((point >= cStart) && (point < cEnd)) {ierr = DMLabelSetValue(label, point, values[v]);CHKERRQ(ierr);}
147       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
148     }
149     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
150     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
151   }
152   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
153   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
154   PetscFunctionReturn(0);
155 }
156 
157 #undef __FUNCT__
158 #define __FUNCT__ "DMPlexShiftPoint_Internal"
159 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthMax[], PetscInt depthEnd[], PetscInt depthShift[])
160 {
161   if (depth < 0) return p;
162   /* Normal Cells                 */ if (p < depthMax[depth])                return p;
163   /* Hybrid Cells+Normal Vertices */ if (p < depthMax[0])                    return p + depthShift[depth];
164   /* Hybrid Vertices+Normal Faces */ if (depth < 2 || p < depthMax[depth-1]) return p + depthShift[depth] + depthShift[0];
165   /* Hybrid Faces+Normal Edges    */ if (depth < 3 || p < depthMax[depth-2]) return p + depthShift[depth] + depthShift[0] + depthShift[depth-1];
166   /* Hybrid Edges                 */                                         return p + depthShift[depth] + depthShift[0] + depthShift[depth-1] + depthShift[depth-2];
167 }
168 
169 #undef __FUNCT__
170 #define __FUNCT__ "DMPlexShiftSizes_Internal"
171 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
172 {
173   PetscInt      *depthMax, *depthEnd;
174   PetscInt       depth = 0, d, pStart, pEnd, p;
175   PetscErrorCode ierr;
176 
177   PetscFunctionBegin;
178   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
179   if (depth < 0) PetscFunctionReturn(0);
180   ierr = PetscMalloc2(depth+1,&depthMax,depth+1,&depthEnd);CHKERRQ(ierr);
181   /* Step 1: Expand chart */
182   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
183   ierr = DMPlexGetHybridBounds(dm, &depthMax[depth], depth > 0 ? &depthMax[depth-1] : NULL, &depthMax[1], &depthMax[0]);CHKERRQ(ierr);
184   for (d = 0; d <= depth; ++d) {
185     pEnd += depthShift[d];
186     ierr  = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
187     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
188   }
189   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
190   /* Step 2: Set cone and support sizes */
191   for (d = 0; d <= depth; ++d) {
192     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
193     for (p = pStart; p < pEnd; ++p) {
194       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthMax, depthEnd, depthShift);
195       PetscInt size;
196 
197       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
198       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
199       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
200       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
201     }
202   }
203   ierr = PetscFree2(depthMax,depthEnd);CHKERRQ(ierr);
204   PetscFunctionReturn(0);
205 }
206 
207 #undef __FUNCT__
208 #define __FUNCT__ "DMPlexShiftPoints_Internal"
209 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
210 {
211   PetscInt      *depthEnd, *depthMax, *newpoints;
212   PetscInt       depth = 0, d, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
213   PetscErrorCode ierr;
214 
215   PetscFunctionBegin;
216   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
217   if (depth < 0) PetscFunctionReturn(0);
218   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
219   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
220   ierr = PetscMalloc3(depth+1,&depthEnd,depth+1,&depthMax,PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
221   ierr = DMPlexGetHybridBounds(dm, &depthMax[depth], depth > 0 ? &depthMax[depth-1] : NULL, &depthMax[1], &depthMax[0]);CHKERRQ(ierr);
222   for (d = 0; d <= depth; ++d) {
223     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
224     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
225   }
226   /* Step 5: Set cones and supports */
227   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
228   for (p = pStart; p < pEnd; ++p) {
229     const PetscInt *points = NULL, *orientations = NULL;
230     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthMax, depthEnd, depthShift);
231 
232     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
233     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
234     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
235     for (i = 0; i < size; ++i) {
236       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthMax, depthEnd, depthShift);
237     }
238     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
239     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
240     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
241     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
242     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
243     for (i = 0; i < size; ++i) {
244       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthMax, depthEnd, depthShift);
245     }
246     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
247     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
248   }
249   ierr = PetscFree3(depthEnd,depthMax,newpoints);CHKERRQ(ierr);
250   PetscFunctionReturn(0);
251 }
252 
253 #undef __FUNCT__
254 #define __FUNCT__ "DMPlexShiftCoordinates_Internal"
255 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
256 {
257   PetscSection   coordSection, newCoordSection;
258   Vec            coordinates, newCoordinates;
259   PetscScalar   *coords, *newCoords;
260   PetscInt      *depthEnd, coordSize;
261   PetscInt       dim, depth = 0, d, vStart, vEnd, vStartNew, vEndNew, v;
262   PetscErrorCode ierr;
263 
264   PetscFunctionBegin;
265   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
266   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
267   ierr = PetscMalloc1(depth+1, &depthEnd);CHKERRQ(ierr);
268   for (d = 0; d <= depth; ++d) {
269     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
270   }
271   /* Step 8: Convert coordinates */
272   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
273   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
274   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
275   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
276   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
277   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
278   ierr = PetscSectionSetChart(newCoordSection, vStartNew, vEndNew);CHKERRQ(ierr);
279   for (v = vStartNew; v < vEndNew; ++v) {
280     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
281     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
282   }
283   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
284   ierr = DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
285   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
286   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &newCoordinates);CHKERRQ(ierr);
287   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
288   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
289   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
290   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
291   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
292   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
293   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
294   for (v = vStart; v < vEnd; ++v) {
295     PetscInt dof, off, noff, d;
296 
297     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
298     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
299     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthEnd, depthEnd, depthShift), &noff);CHKERRQ(ierr);
300     for (d = 0; d < dof; ++d) {
301       newCoords[noff+d] = coords[off+d];
302     }
303   }
304   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
305   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
306   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
307   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
308   ierr = PetscFree(depthEnd);CHKERRQ(ierr);
309   PetscFunctionReturn(0);
310 }
311 
312 #undef __FUNCT__
313 #define __FUNCT__ "DMPlexShiftSF_Internal"
314 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
315 {
316   PetscInt          *depthMax, *depthEnd;
317   PetscInt           depth = 0, d;
318   PetscSF            sfPoint, sfPointNew;
319   const PetscSFNode *remotePoints;
320   PetscSFNode       *gremotePoints;
321   const PetscInt    *localPoints;
322   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
323   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
324   PetscErrorCode     ierr;
325 
326   PetscFunctionBegin;
327   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
328   ierr = PetscMalloc2(depth+1,&depthMax,depth+1,&depthEnd);CHKERRQ(ierr);
329   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
330   for (d = 0; d <= depth; ++d) {
331     totShift += depthShift[d];
332     ierr      = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
333     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
334   }
335   /* Step 9: Convert pointSF */
336   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
337   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
338   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
339   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
340   if (numRoots >= 0) {
341     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
342     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthMax, depthEnd, depthShift);
343     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
344     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
345     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
346     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
347     for (l = 0; l < numLeaves; ++l) {
348       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthMax, depthEnd, depthShift);
349       gremotePoints[l].rank  = remotePoints[l].rank;
350       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
351     }
352     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
353     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
354   }
355   ierr = PetscFree2(depthMax,depthEnd);CHKERRQ(ierr);
356   PetscFunctionReturn(0);
357 }
358 
359 #undef __FUNCT__
360 #define __FUNCT__ "DMPlexShiftLabels_Internal"
361 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
362 {
363   PetscSF            sfPoint;
364   DMLabel            vtkLabel, ghostLabel;
365   PetscInt          *depthMax, *depthEnd;
366   const PetscSFNode *leafRemote;
367   const PetscInt    *leafLocal;
368   PetscInt           depth = 0, d, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
369   PetscMPIInt        rank;
370   PetscErrorCode     ierr;
371 
372   PetscFunctionBegin;
373   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
374   ierr = PetscMalloc2(depth+1,&depthMax,depth+1,&depthEnd);CHKERRQ(ierr);
375   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
376   for (d = 0; d <= depth; ++d) {
377     ierr = DMPlexGetDepthStratum(dm, d, NULL, &depthEnd[d]);CHKERRQ(ierr);
378     depthMax[d] = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
379   }
380   /* Step 10: Convert labels */
381   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
382   for (l = 0; l < numLabels; ++l) {
383     DMLabel         label, newlabel;
384     const char     *lname;
385     PetscBool       isDepth;
386     IS              valueIS;
387     const PetscInt *values;
388     PetscInt        numValues, val;
389 
390     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
391     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
392     if (isDepth) continue;
393     ierr = DMPlexCreateLabel(dmNew, lname);CHKERRQ(ierr);
394     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
395     ierr = DMPlexGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
396     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
397     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
398     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
399     for (val = 0; val < numValues; ++val) {
400       IS              pointIS;
401       const PetscInt *points;
402       PetscInt        numPoints, p;
403 
404       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
405       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
406       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
407       for (p = 0; p < numPoints; ++p) {
408         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthMax, depthEnd, depthShift);
409 
410         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
411       }
412       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
413       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
414     }
415     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
416     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
417   }
418   ierr = PetscFree2(depthMax,depthEnd);CHKERRQ(ierr);
419   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
420   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
421   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
422   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
423   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
424   ierr = DMPlexCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
425   ierr = DMPlexCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
426   ierr = DMPlexGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
427   ierr = DMPlexGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
428   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
429     for (; c < leafLocal[l] && c < cEnd; ++c) {
430       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
431     }
432     if (leafLocal[l] >= cEnd) break;
433     if (leafRemote[l].rank == rank) {
434       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
435     } else {
436       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
437     }
438   }
439   for (; c < cEnd; ++c) {
440     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
441   }
442   if (0) {
443     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
444     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
445     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
446   }
447   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
448   for (f = fStart; f < fEnd; ++f) {
449     PetscInt numCells;
450 
451     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
452     if (numCells < 2) {
453       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
454     } else {
455       const PetscInt *cells = NULL;
456       PetscInt        vA, vB;
457 
458       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
459       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
460       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
461       if (!vA && !vB) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
462     }
463   }
464   if (0) {
465     ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
466     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
467     ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
468   }
469   PetscFunctionReturn(0);
470 }
471 
472 #undef __FUNCT__
473 #define __FUNCT__ "DMPlexConstructGhostCells_Internal"
474 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
475 {
476   PetscSF         sf;
477   IS              valueIS;
478   const PetscInt *values, *leaves;
479   PetscInt       *depthShift;
480   PetscInt        depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
481   PetscErrorCode  ierr;
482 
483   PetscFunctionBegin;
484   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
485   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
486   nleaves = PetscMax(0, nleaves);
487   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
488   /* Count ghost cells */
489   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
490   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
491   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
492   Ng   = 0;
493   for (fs = 0; fs < numFS; ++fs) {
494     IS              faceIS;
495     const PetscInt *faces;
496     PetscInt        numFaces, f, numBdFaces = 0;
497 
498     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
499     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
500     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
501     for (f = 0; f < numFaces; ++f) {
502       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
503       if (loc >= 0) continue;
504       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
505     }
506     Ng += numBdFaces;
507     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
508   }
509   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
510   ierr = PetscMalloc1(depth+1, &depthShift);CHKERRQ(ierr);
511   ierr = PetscMemzero(depthShift, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
512   if (depth >= 0) depthShift[depth] = Ng;
513   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
514   /* Step 3: Set cone/support sizes for new points */
515   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
516   ierr = DMPlexSetHybridBounds(gdm, cEnd, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
517   for (c = cEnd; c < cEnd + Ng; ++c) {
518     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
519   }
520   for (fs = 0; fs < numFS; ++fs) {
521     IS              faceIS;
522     const PetscInt *faces;
523     PetscInt        numFaces, f;
524 
525     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
526     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
527     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
528     for (f = 0; f < numFaces; ++f) {
529       PetscInt size;
530 
531       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
532       if (loc >= 0) continue;
533       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
534       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
535       if (size != 1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
536       ierr = DMPlexSetSupportSize(gdm, faces[f] + Ng, 2);CHKERRQ(ierr);
537     }
538     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
539     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
540   }
541   /* Step 4: Setup ghosted DM */
542   ierr = DMSetUp(gdm);CHKERRQ(ierr);
543   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
544   /* Step 6: Set cones and supports for new points */
545   ghostCell = cEnd;
546   for (fs = 0; fs < numFS; ++fs) {
547     IS              faceIS;
548     const PetscInt *faces;
549     PetscInt        numFaces, f;
550 
551     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
552     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
553     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
554     for (f = 0; f < numFaces; ++f) {
555       PetscInt newFace = faces[f] + Ng;
556 
557       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
558       if (loc >= 0) continue;
559       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
560       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
561       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
562       ++ghostCell;
563     }
564     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
565     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
566   }
567   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
568   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
569   /* Step 7: Stratify */
570   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
571   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
572   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
573   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
574   ierr = PetscFree(depthShift);CHKERRQ(ierr);
575   if (numGhostCells) *numGhostCells = Ng;
576   PetscFunctionReturn(0);
577 }
578 
579 #undef __FUNCT__
580 #define __FUNCT__ "DMPlexConstructGhostCells"
581 /*@C
582   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
583 
584   Collective on dm
585 
586   Input Parameters:
587 + dm - The original DM
588 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
589 
590   Output Parameters:
591 + numGhostCells - The number of ghost cells added to the DM
592 - dmGhosted - The new DM
593 
594   Note: If no label exists of that name, one will be created marking all boundary faces
595 
596   Level: developer
597 
598 .seealso: DMCreate()
599 @*/
600 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
601 {
602   DM             gdm;
603   DMLabel        label;
604   const char    *name = labelName ? labelName : "Face Sets";
605   PetscInt       dim;
606   PetscBool      flag;
607   PetscErrorCode ierr;
608 
609   PetscFunctionBegin;
610   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
611   if (numGhostCells) PetscValidPointer(numGhostCells, 3);
612   PetscValidPointer(dmGhosted, 4);
613   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
614   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
615   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
616   ierr = DMSetDimension(gdm, dim);CHKERRQ(ierr);
617   ierr = DMPlexGetAdjacencyUseCone(dm, &flag);CHKERRQ(ierr);
618   ierr = DMPlexSetAdjacencyUseCone(gdm, flag);CHKERRQ(ierr);
619   ierr = DMPlexGetAdjacencyUseClosure(dm, &flag);CHKERRQ(ierr);
620   ierr = DMPlexSetAdjacencyUseClosure(gdm, flag);CHKERRQ(ierr);
621   ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
622   if (!label) {
623     /* Get label for boundary faces */
624     ierr = DMPlexCreateLabel(dm, name);CHKERRQ(ierr);
625     ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
626     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
627   }
628   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
629   ierr = DMPlexCopyBoundary(dm, gdm);CHKERRQ(ierr);
630   *dmGhosted = gdm;
631   PetscFunctionReturn(0);
632 }
633 
634 #undef __FUNCT__
635 #define __FUNCT__ "DMPlexConstructCohesiveCells_Internal"
636 /*
637   We are adding three kinds of points here:
638     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
639     Non-replicated: Points which exist on the fault, but are not replicated
640     Hybrid:         Entirely new points, such as cohesive cells
641 
642   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
643   each depth so that the new split/hybrid points can be inserted as a block.
644 */
645 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
646 {
647   MPI_Comm         comm;
648   IS               valueIS;
649   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
650   const PetscInt  *values;          /* List of depths for which we have replicated points */
651   IS              *splitIS;
652   IS              *unsplitIS;
653   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
654   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
655   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
656   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
657   const PetscInt **splitPoints;        /* Replicated points for each depth */
658   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
659   PetscSection     coordSection;
660   Vec              coordinates;
661   PetscScalar     *coords;
662   PetscInt         depths[4];          /* Depths in the order that plex points are numbered */
663   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
664   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
665   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
666   PetscInt        *depthOffset;        /* Prefix sums of depthShift */
667   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
668   PetscInt        *coneNew, *coneONew, *supportNew;
669   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
670   PetscErrorCode   ierr;
671 
672   PetscFunctionBegin;
673   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
674   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
675   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
676   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
677   depths[0] = depth;
678   depths[1] = 0;
679   depths[2] = depth-1;
680   depths[3] = 1;
681   /* Count split points and add cohesive cells */
682   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
683   ierr = PetscMalloc6(depth+1,&depthMax,depth+1,&depthEnd,depth+1,&depthShift,depth+1,&depthOffset,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
684   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
685   ierr = PetscMemzero(depthShift,  (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
686   ierr = PetscMemzero(depthOffset, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
687   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
688   for (d = 0; d <= depth; ++d) {
689     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
690     depthEnd[d]           = pMaxNew[d];
691     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
692     numSplitPoints[d]     = 0;
693     numUnsplitPoints[d]   = 0;
694     numHybridPoints[d]    = 0;
695     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
696     splitPoints[d]        = NULL;
697     unsplitPoints[d]      = NULL;
698     splitIS[d]            = NULL;
699     unsplitIS[d]          = NULL;
700   }
701   if (label) {
702     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
703     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
704     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
705   }
706   for (sp = 0; sp < numSP; ++sp) {
707     const PetscInt dep = values[sp];
708 
709     if ((dep < 0) || (dep > depth)) continue;
710     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
711     if (splitIS[dep]) {
712       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
713       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
714     }
715     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
716     if (unsplitIS[dep]) {
717       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
718       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
719     }
720   }
721   /* Calculate number of hybrid points */
722   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
723   for (d = 0; d <= depth; ++d) depthShift[d]          = numSplitPoints[d] + numHybridPoints[d];
724   for (d = 1; d <= depth; ++d) depthOffset[depths[d]] = depthOffset[depths[d-1]] + depthShift[depths[d-1]];
725   for (d = 0; d <= depth; ++d) pMaxNew[d]            += depthOffset[d] - numHybridPointsOld[d];
726   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
727   /* Step 3: Set cone/support sizes for new points */
728   for (dep = 0; dep <= depth; ++dep) {
729     for (p = 0; p < numSplitPoints[dep]; ++p) {
730       const PetscInt  oldp   = splitPoints[dep][p];
731       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
732       const PetscInt  splitp = p    + pMaxNew[dep];
733       const PetscInt *support;
734       PetscInt        coneSize, supportSize, qf, qn, qp, e;
735 
736       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
737       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
738       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
739       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
740       if (dep == depth-1) {
741         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
742 
743         /* Add cohesive cells, they are prisms */
744         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
745       } else if (dep == 0) {
746         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
747 
748         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
749         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
750           PetscInt val;
751 
752           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
753           if (val == 1) ++qf;
754           if ((val == 1) || (val ==  (shift + 1))) ++qn;
755           if ((val == 1) || (val == -(shift + 1))) ++qp;
756         }
757         /* Split old vertex: Edges into original vertex and new cohesive edge */
758         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
759         /* Split new vertex: Edges into split vertex and new cohesive edge */
760         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
761         /* Add hybrid edge */
762         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
763         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
764       } else if (dep == dim-2) {
765         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
766 
767         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
768         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
769           PetscInt val;
770 
771           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
772           if (val == dim-1) ++qf;
773           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
774           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
775         }
776         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
777         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
778         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
779         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
780         /* Add hybrid face */
781         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
782         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
783       }
784     }
785   }
786   for (dep = 0; dep <= depth; ++dep) {
787     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
788       const PetscInt  oldp   = unsplitPoints[dep][p];
789       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
790       const PetscInt *support;
791       PetscInt        coneSize, supportSize, qf, e, s;
792 
793       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
794       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
795       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
796       if (dep == 0) {
797         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
798 
799         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
800         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
801           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
802           if (e >= 0) ++qf;
803         }
804         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
805         /* Add hybrid edge */
806         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
807         for (e = 0, qf = 0; e < supportSize; ++e) {
808           PetscInt val;
809 
810           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
811           /* Split and unsplit edges produce hybrid faces */
812           if (val == 1) ++qf;
813           if (val == (shift2 + 1)) ++qf;
814         }
815         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
816       } else if (dep == dim-2) {
817         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
818         PetscInt       val;
819 
820         for (e = 0, qf = 0; e < supportSize; ++e) {
821           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
822           if (val == dim-1) qf += 2;
823           else              ++qf;
824         }
825         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
826         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
827         /* Add hybrid face */
828         for (e = 0, qf = 0; e < supportSize; ++e) {
829           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
830           if (val == dim-1) ++qf;
831         }
832         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
833         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
834       }
835     }
836   }
837   /* Step 4: Setup split DM */
838   ierr = DMSetUp(sdm);CHKERRQ(ierr);
839   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
840   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
841   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
842   /* Step 6: Set cones and supports for new points */
843   for (dep = 0; dep <= depth; ++dep) {
844     for (p = 0; p < numSplitPoints[dep]; ++p) {
845       const PetscInt  oldp   = splitPoints[dep][p];
846       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
847       const PetscInt  splitp = p    + pMaxNew[dep];
848       const PetscInt *cone, *support, *ornt;
849       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
850 
851       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
852       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
853       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
854       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
855       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
856       if (dep == depth-1) {
857         PetscBool       hasUnsplit = PETSC_FALSE;
858         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
859         const PetscInt *supportF;
860 
861         /* Split face:       copy in old face to new face to start */
862         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
863         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
864         /* Split old face:   old vertices/edges in cone so no change */
865         /* Split new face:   new vertices/edges in cone */
866         for (q = 0; q < coneSize; ++q) {
867           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
868           if (v < 0) {
869             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
870             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
871             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthMax, depthEnd, depthShift) /*cone[q] + depthOffset[dep-1]*/;
872             hasUnsplit   = PETSC_TRUE;
873           } else {
874             coneNew[2+q] = v + pMaxNew[dep-1];
875             if (dep > 1) {
876               const PetscInt *econe;
877               PetscInt        econeSize, r, vs, vu;
878 
879               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
880               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
881               for (r = 0; r < econeSize; ++r) {
882                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
883                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
884                 if (vs >= 0) continue;
885                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
886                 hasUnsplit   = PETSC_TRUE;
887               }
888             }
889           }
890         }
891         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
892         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
893         /* Face support */
894         for (s = 0; s < supportSize; ++s) {
895           PetscInt val;
896 
897           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
898           if (val < 0) {
899             /* Split old face:   Replace negative side cell with cohesive cell */
900              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
901           } else {
902             /* Split new face:   Replace positive side cell with cohesive cell */
903             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
904             /* Get orientation for cohesive face */
905             {
906               const PetscInt *ncone, *nconeO;
907               PetscInt        nconeSize, nc;
908 
909               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
910               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
911               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
912               for (nc = 0; nc < nconeSize; ++nc) {
913                 if (ncone[nc] == oldp) {
914                   coneONew[0] = nconeO[nc];
915                   break;
916                 }
917               }
918               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
919             }
920           }
921         }
922         /* Cohesive cell:    Old and new split face, then new cohesive faces */
923         coneNew[0]  = newp;   /* Extracted negative side orientation above */
924         coneNew[1]  = splitp;
925         coneONew[1] = coneONew[0];
926         for (q = 0; q < coneSize; ++q) {
927           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
928           if (v < 0) {
929             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
930             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
931             coneONew[2+q] = 0;
932           } else {
933             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
934           }
935           coneONew[2+q] = 0;
936         }
937         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
938         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
939         /* Label the hybrid cells on the boundary of the split */
940         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
941       } else if (dep == 0) {
942         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
943 
944         /* Split old vertex: Edges in old split faces and new cohesive edge */
945         for (e = 0, qn = 0; e < supportSize; ++e) {
946           PetscInt val;
947 
948           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
949           if ((val == 1) || (val == (shift + 1))) {
950             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
951           }
952         }
953         supportNew[qn] = hybedge;
954         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
955         /* Split new vertex: Edges in new split faces and new cohesive edge */
956         for (e = 0, qp = 0; e < supportSize; ++e) {
957           PetscInt val, edge;
958 
959           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
960           if (val == 1) {
961             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
962             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
963             supportNew[qp++] = edge + pMaxNew[dep+1];
964           } else if (val == -(shift + 1)) {
965             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
966           }
967         }
968         supportNew[qp] = hybedge;
969         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
970         /* Hybrid edge:    Old and new split vertex */
971         coneNew[0] = newp;
972         coneNew[1] = splitp;
973         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
974         for (e = 0, qf = 0; e < supportSize; ++e) {
975           PetscInt val, edge;
976 
977           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
978           if (val == 1) {
979             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
980             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
981             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
982           }
983         }
984         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
985       } else if (dep == dim-2) {
986         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
987 
988         /* Split old edge:   old vertices in cone so no change */
989         /* Split new edge:   new vertices in cone */
990         for (q = 0; q < coneSize; ++q) {
991           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
992           if (v < 0) {
993             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
994             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
995             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthMax, depthEnd, depthShift) /*cone[q] + depthOffset[dep-1]*/;
996           } else {
997             coneNew[q] = v + pMaxNew[dep-1];
998           }
999         }
1000         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
1001         /* Split old edge: Faces in positive side cells and old split faces */
1002         for (e = 0, q = 0; e < supportSize; ++e) {
1003           PetscInt val;
1004 
1005           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1006           if (val == dim-1) {
1007             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
1008           } else if (val == (shift + dim-1)) {
1009             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
1010           }
1011         }
1012         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1013         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1014         /* Split new edge: Faces in negative side cells and new split faces */
1015         for (e = 0, q = 0; e < supportSize; ++e) {
1016           PetscInt val, face;
1017 
1018           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1019           if (val == dim-1) {
1020             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1021             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1022             supportNew[q++] = face + pMaxNew[dep+1];
1023           } else if (val == -(shift + dim-1)) {
1024             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthMax, depthEnd, depthShift) /*support[e] + depthOffset[dep+1]*/;
1025           }
1026         }
1027         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1028         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1029         /* Hybrid face */
1030         coneNew[0] = newp;
1031         coneNew[1] = splitp;
1032         for (v = 0; v < coneSize; ++v) {
1033           PetscInt vertex;
1034           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
1035           if (vertex < 0) {
1036             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
1037             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
1038             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1039           } else {
1040             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1041           }
1042         }
1043         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1044         for (e = 0, qf = 0; e < supportSize; ++e) {
1045           PetscInt val, face;
1046 
1047           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1048           if (val == dim-1) {
1049             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1050             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1051             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1052           }
1053         }
1054         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1055       }
1056     }
1057   }
1058   for (dep = 0; dep <= depth; ++dep) {
1059     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1060       const PetscInt  oldp   = unsplitPoints[dep][p];
1061       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*oldp + depthOffset[dep]*/;
1062       const PetscInt *cone, *support, *ornt;
1063       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1064 
1065       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1066       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1067       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1068       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1069       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1070       if (dep == 0) {
1071         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1072 
1073         /* Unsplit vertex */
1074         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1075         for (s = 0, q = 0; s < supportSize; ++s) {
1076           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthMax, depthEnd, depthShift) /*support[s] + depthOffset[dep+1]*/;
1077           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1078           if (e >= 0) {
1079             supportNew[q++] = e + pMaxNew[dep+1];
1080           }
1081         }
1082         supportNew[q++] = hybedge;
1083         supportNew[q++] = hybedge;
1084         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1085         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1086         /* Hybrid edge */
1087         coneNew[0] = newp;
1088         coneNew[1] = newp;
1089         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1090         for (e = 0, qf = 0; e < supportSize; ++e) {
1091           PetscInt val, edge;
1092 
1093           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1094           if (val == 1) {
1095             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1096             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1097             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1098           } else if  (val ==  (shift2 + 1)) {
1099             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1100             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1101             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1102           }
1103         }
1104         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1105       } else if (dep == dim-2) {
1106         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1107 
1108         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1109         for (f = 0, qf = 0; f < supportSize; ++f) {
1110           PetscInt val, face;
1111 
1112           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1113           if (val == dim-1) {
1114             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1115             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1116             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthMax, depthEnd, depthShift) /*support[f] + depthOffset[dep+1]*/;
1117             supportNew[qf++] = face + pMaxNew[dep+1];
1118           } else {
1119             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthMax, depthEnd, depthShift) /*support[f] + depthOffset[dep+1]*/;
1120           }
1121         }
1122         supportNew[qf++] = hybface;
1123         supportNew[qf++] = hybface;
1124         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1125         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1126         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1127         /* Add hybrid face */
1128         coneNew[0] = newp;
1129         coneNew[1] = newp;
1130         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1131         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1132         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1133         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1134         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1135         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1136         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1137         for (f = 0, qf = 0; f < supportSize; ++f) {
1138           PetscInt val, face;
1139 
1140           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1141           if (val == dim-1) {
1142             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1143             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1144           }
1145         }
1146         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1147         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1148         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1149       }
1150     }
1151   }
1152   /* Step 6b: Replace split points in negative side cones */
1153   for (sp = 0; sp < numSP; ++sp) {
1154     PetscInt        dep = values[sp];
1155     IS              pIS;
1156     PetscInt        numPoints;
1157     const PetscInt *points;
1158 
1159     if (dep >= 0) continue;
1160     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1161     if (!pIS) continue;
1162     dep  = -dep - shift;
1163     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1164     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1165     for (p = 0; p < numPoints; ++p) {
1166       const PetscInt  oldp = points[p];
1167       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthMax, depthEnd, depthShift) /*depthOffset[dep] + oldp*/;
1168       const PetscInt *cone;
1169       PetscInt        coneSize, c;
1170       /* PetscBool       replaced = PETSC_FALSE; */
1171 
1172       /* Negative edge: replace split vertex */
1173       /* Negative cell: replace split face */
1174       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1175       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1176       for (c = 0; c < coneSize; ++c) {
1177         const PetscInt coldp = cone[c] - depthOffset[dep-1];
1178         PetscInt       csplitp, cp, val;
1179 
1180         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1181         if (val == dep-1) {
1182           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1183           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1184           csplitp  = pMaxNew[dep-1] + cp;
1185           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1186           /* replaced = PETSC_TRUE; */
1187         }
1188       }
1189       /* Cells with only a vertex or edge on the submesh have no replacement */
1190       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1191     }
1192     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1193     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1194   }
1195   /* Step 7: Stratify */
1196   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1197   /* Step 8: Coordinates */
1198   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1199   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1200   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1201   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1202   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1203     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthMax, depthEnd, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1204     const PetscInt splitp = pMaxNew[0] + v;
1205     PetscInt       dof, off, soff, d;
1206 
1207     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1208     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1209     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1210     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1211   }
1212   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1213   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1214   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1215   /* Step 10: Labels */
1216   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1217   ierr = DMPlexGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1218   for (dep = 0; dep <= depth; ++dep) {
1219     for (p = 0; p < numSplitPoints[dep]; ++p) {
1220       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthMax, depthEnd, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1221       const PetscInt splitp = pMaxNew[dep] + p;
1222       PetscInt       l;
1223 
1224       for (l = 0; l < numLabels; ++l) {
1225         DMLabel     mlabel;
1226         const char *lname;
1227         PetscInt    val;
1228         PetscBool   isDepth;
1229 
1230         ierr = DMPlexGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1231         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1232         if (isDepth) continue;
1233         ierr = DMPlexGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1234         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1235         if (val >= 0) {
1236           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1237 #if 0
1238           /* Do not put cohesive edges into the label */
1239           if (dep == 0) {
1240             const PetscInt cedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1241             ierr = DMLabelSetValue(mlabel, cedge, val);CHKERRQ(ierr);
1242           } else if (dep == dim-2) {
1243             const PetscInt cface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1244             ierr = DMLabelSetValue(mlabel, cface, val);CHKERRQ(ierr);
1245           }
1246           /* Do not put cohesive faces into the label */
1247 #endif
1248         }
1249       }
1250     }
1251   }
1252   for (sp = 0; sp < numSP; ++sp) {
1253     const PetscInt dep = values[sp];
1254 
1255     if ((dep < 0) || (dep > depth)) continue;
1256     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1257     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1258     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1259     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1260   }
1261   if (label) {
1262     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1263     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1264   }
1265   for (d = 0; d <= depth; ++d) {
1266     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1267     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1268   }
1269   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1270   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1271   ierr = PetscFree6(depthMax, depthEnd, depthShift, depthOffset, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1272   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1273   PetscFunctionReturn(0);
1274 }
1275 
1276 #undef __FUNCT__
1277 #define __FUNCT__ "DMPlexConstructCohesiveCells"
1278 /*@C
1279   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1280 
1281   Collective on dm
1282 
1283   Input Parameters:
1284 + dm - The original DM
1285 - label - The label specifying the boundary faces (this could be auto-generated)
1286 
1287   Output Parameters:
1288 - dmSplit - The new DM
1289 
1290   Level: developer
1291 
1292 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1293 @*/
1294 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1295 {
1296   DM             sdm;
1297   PetscInt       dim;
1298   PetscErrorCode ierr;
1299 
1300   PetscFunctionBegin;
1301   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1302   PetscValidPointer(dmSplit, 3);
1303   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1304   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1305   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1306   ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);
1307   switch (dim) {
1308   case 2:
1309   case 3:
1310     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1311     break;
1312   default:
1313     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1314   }
1315   *dmSplit = sdm;
1316   PetscFunctionReturn(0);
1317 }
1318 
1319 #undef __FUNCT__
1320 #define __FUNCT__ "GetSurfaceSide_Static"
1321 /* Returns the side of the surface for a given cell with a face on the surface */
1322 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1323 {
1324   const PetscInt *cone, *ornt;
1325   PetscInt        dim, coneSize, c;
1326   PetscErrorCode  ierr;
1327 
1328   PetscFunctionBegin;
1329   *pos = PETSC_TRUE;
1330   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1331   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1332   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1333   ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
1334   for (c = 0; c < coneSize; ++c) {
1335     if (cone[c] == face) {
1336       PetscInt o = ornt[c];
1337 
1338       if (subdm) {
1339         const PetscInt *subcone, *subornt;
1340         PetscInt        subpoint, subface, subconeSize, sc;
1341 
1342         ierr = PetscFindInt(cell, numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1343         ierr = PetscFindInt(face, numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1344         ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1345         ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1346         ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1347         for (sc = 0; sc < subconeSize; ++sc) {
1348           if (subcone[sc] == subface) {
1349             o = subornt[0];
1350             break;
1351           }
1352         }
1353         if (sc >= subconeSize) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %d (%d) in cone for subpoint %d (%d)", subface, face, subpoint, cell);
1354       }
1355       if (o >= 0) *pos = PETSC_TRUE;
1356       else        *pos = PETSC_FALSE;
1357       break;
1358     }
1359   }
1360   if (c == coneSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %d in split face %d support does not have it in the cone", cell, face);
1361   PetscFunctionReturn(0);
1362 }
1363 
1364 #undef __FUNCT__
1365 #define __FUNCT__ "DMPlexLabelCohesiveComplete"
1366 /*@
1367   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1368   to complete the surface
1369 
1370   Input Parameters:
1371 + dm     - The DM
1372 . label  - A DMLabel marking the surface
1373 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1374 . flip   - Flag to flip the submesh normal and replace points on the other side
1375 - subdm  - The subDM associated with the label, or NULL
1376 
1377   Output Parameter:
1378 . label - A DMLabel marking all surface points
1379 
1380   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1381 
1382   Level: developer
1383 
1384 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1385 @*/
1386 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
1387 {
1388   DMLabel         depthLabel;
1389   IS              dimIS, subpointIS, facePosIS, faceNegIS, crossEdgeIS = NULL;
1390   const PetscInt *points, *subpoints;
1391   const PetscInt  rev   = flip ? -1 : 1;
1392   PetscInt       *pMax;
1393   PetscInt        shift = 100, shift2 = 200, dim, depth, pSize, dep, cStart, cEnd, cMax, fStart, fEnd, vStart, vEnd, numPoints, numSubpoints, p, val;
1394   PetscErrorCode  ierr;
1395 
1396   PetscFunctionBegin;
1397   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1398   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1399   pSize = PetscMax(depth, dim) + 1;
1400   ierr = PetscMalloc1(pSize,&pMax);CHKERRQ(ierr);
1401   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1402   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1403   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1404   if (subdm) {
1405     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1406     if (subpointIS) {
1407       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1408       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1409     }
1410   }
1411   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1412   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1413   if (!dimIS) {
1414     ierr = PetscFree(pMax);CHKERRQ(ierr);
1415     PetscFunctionReturn(0);
1416   }
1417   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1418   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1419   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1420     const PetscInt *support;
1421     PetscInt        supportSize, s;
1422 
1423     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1424     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1425     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1426     for (s = 0; s < supportSize; ++s) {
1427       const PetscInt *cone;
1428       PetscInt        coneSize, c;
1429       PetscBool       pos;
1430 
1431       ierr = GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos);CHKERRQ(ierr);
1432       if (pos) {ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);}
1433       else     {ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);}
1434       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
1435       /* Put faces touching the fault in the label */
1436       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1437       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1438       for (c = 0; c < coneSize; ++c) {
1439         const PetscInt point = cone[c];
1440 
1441         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1442         if (val == -1) {
1443           PetscInt *closure = NULL;
1444           PetscInt  closureSize, cl;
1445 
1446           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1447           for (cl = 0; cl < closureSize*2; cl += 2) {
1448             const PetscInt clp  = closure[cl];
1449             PetscInt       bval = -1;
1450 
1451             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1452             if (blabel) {ierr = DMLabelGetValue(blabel, clp, &bval);CHKERRQ(ierr);}
1453             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
1454               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1455               break;
1456             }
1457           }
1458           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1459         }
1460       }
1461     }
1462   }
1463   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1464   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1465   if (subdm) {
1466     if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1467     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1468   }
1469   /* Mark boundary points as unsplit */
1470   if (blabel) {
1471     ierr = DMLabelGetStratumIS(blabel, 1, &dimIS);CHKERRQ(ierr);
1472     ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1473     ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1474     for (p = 0; p < numPoints; ++p) {
1475       const PetscInt point = points[p];
1476       PetscInt       val, bval;
1477 
1478       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1479       if (bval >= 0) {
1480         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1481         if ((val < 0) || (val > dim)) {
1482           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
1483           ierr = DMLabelClearValue(blabel, point, bval);CHKERRQ(ierr);
1484         }
1485       }
1486     }
1487     for (p = 0; p < numPoints; ++p) {
1488       const PetscInt point = points[p];
1489       PetscInt       val, bval;
1490 
1491       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1492       if (bval >= 0) {
1493         const PetscInt *cone,    *support;
1494         PetscInt        coneSize, supportSize, s, valA, valB, valE;
1495 
1496         /* Mark as unsplit */
1497         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1498         if ((val < 0) || (val > dim)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d has label value %d, should be part of the fault", point, val);
1499         ierr = DMLabelClearValue(label, point, val);CHKERRQ(ierr);
1500         ierr = DMLabelSetValue(label, point, shift2+val);CHKERRQ(ierr);
1501         /* Check for cross-edge
1502              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
1503         if (val != 0) continue;
1504         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1505         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1506         for (s = 0; s < supportSize; ++s) {
1507           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1508           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1509           if (coneSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %D has %D vertices != 2", support[s], coneSize);
1510           ierr = DMLabelGetValue(blabel, cone[0], &valA);CHKERRQ(ierr);
1511           ierr = DMLabelGetValue(blabel, cone[1], &valB);CHKERRQ(ierr);
1512           ierr = DMLabelGetValue(blabel, support[s], &valE);CHKERRQ(ierr);
1513           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) {ierr = DMLabelSetValue(blabel, support[s], 2);CHKERRQ(ierr);}
1514         }
1515       }
1516     }
1517     ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1518     ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1519   }
1520   /* Search for other cells/faces/edges connected to the fault by a vertex */
1521   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1522   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1523   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1524   cMax = cMax < 0 ? cEnd : cMax;
1525   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
1526   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1527   if (blabel) {ierr = DMLabelGetStratumIS(blabel, 2, &crossEdgeIS);CHKERRQ(ierr);}
1528   if (dimIS && crossEdgeIS) {
1529     IS vertIS = dimIS;
1530 
1531     ierr = ISExpand(vertIS, crossEdgeIS, &dimIS);CHKERRQ(ierr);
1532     ierr = ISDestroy(&crossEdgeIS);CHKERRQ(ierr);
1533     ierr = ISDestroy(&vertIS);CHKERRQ(ierr);
1534   }
1535   if (!dimIS) {
1536     ierr = PetscFree(pMax);CHKERRQ(ierr);
1537     PetscFunctionReturn(0);
1538   }
1539   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1540   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1541   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1542     PetscInt *star = NULL;
1543     PetscInt  starSize, s;
1544     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1545 
1546     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1547     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1548     while (again) {
1549       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1550       again = 0;
1551       for (s = 0; s < starSize*2; s += 2) {
1552         const PetscInt  point = star[s];
1553         const PetscInt *cone;
1554         PetscInt        coneSize, c;
1555 
1556         if ((point < cStart) || (point >= cMax)) continue;
1557         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1558         if (val != -1) continue;
1559         again = again == 1 ? 1 : 2;
1560         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1561         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1562         for (c = 0; c < coneSize; ++c) {
1563           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1564           if (val != -1) {
1565             const PetscInt *ccone;
1566             PetscInt        cconeSize, cc, side;
1567 
1568             if (abs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1569             if (val > 0) side =  1;
1570             else         side = -1;
1571             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1572             /* Mark cell faces which touch the fault */
1573             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1574             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1575             for (cc = 0; cc < cconeSize; ++cc) {
1576               PetscInt *closure = NULL;
1577               PetscInt  closureSize, cl;
1578 
1579               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1580               if (val != -1) continue;
1581               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1582               for (cl = 0; cl < closureSize*2; cl += 2) {
1583                 const PetscInt clp = closure[cl];
1584 
1585                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1586                 if (val == -1) continue;
1587                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1588                 break;
1589               }
1590               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1591             }
1592             again = 1;
1593             break;
1594           }
1595         }
1596       }
1597     }
1598     /* Classify the rest by cell membership */
1599     for (s = 0; s < starSize*2; s += 2) {
1600       const PetscInt point = star[s];
1601 
1602       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1603       if (val == -1) {
1604         PetscInt  *sstar = NULL;
1605         PetscInt   sstarSize, ss;
1606         PetscBool  marked = PETSC_FALSE;
1607 
1608         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1609         for (ss = 0; ss < sstarSize*2; ss += 2) {
1610           const PetscInt spoint = sstar[ss];
1611 
1612           if ((spoint < cStart) || (spoint >= cMax)) continue;
1613           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1614           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1615           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1616           if (val > 0) {
1617             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1618           } else {
1619             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1620           }
1621           marked = PETSC_TRUE;
1622           break;
1623         }
1624         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1625         ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1626         if (point < pMax[dep] && !marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1627       }
1628     }
1629     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1630   }
1631   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1632   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1633   /* If any faces touching the fault divide cells on either side, split them */
1634   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1635   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1636   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1637   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1638   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1639   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1640   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1641   for (p = 0; p < numPoints; ++p) {
1642     const PetscInt  point = points[p];
1643     const PetscInt *support;
1644     PetscInt        supportSize, valA, valB;
1645 
1646     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1647     if (supportSize != 2) continue;
1648     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1649     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1650     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1651     if ((valA == -1) || (valB == -1)) continue;
1652     if (valA*valB > 0) continue;
1653     /* Split the face */
1654     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1655     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1656     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1657     /* Label its closure:
1658       unmarked: label as unsplit
1659       incident: relabel as split
1660       split:    do nothing
1661     */
1662     {
1663       PetscInt *closure = NULL;
1664       PetscInt  closureSize, cl;
1665 
1666       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1667       for (cl = 0; cl < closureSize*2; cl += 2) {
1668         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1669         if (valA == -1) { /* Mark as unsplit */
1670           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1671           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1672         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1673           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1674           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1675           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1676         }
1677       }
1678       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1679     }
1680   }
1681   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1682   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1683   ierr = PetscFree(pMax);CHKERRQ(ierr);
1684   PetscFunctionReturn(0);
1685 }
1686 
1687 #undef __FUNCT__
1688 #define __FUNCT__ "DMPlexCreateHybridMesh"
1689 /*@C
1690   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1691 
1692   Collective on dm
1693 
1694   Input Parameters:
1695 + dm - The original DM
1696 - labelName - The label specifying the interface vertices
1697 
1698   Output Parameters:
1699 + hybridLabel - The label fully marking the interface
1700 - dmHybrid - The new DM
1701 
1702   Level: developer
1703 
1704 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1705 @*/
1706 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1707 {
1708   DM             idm;
1709   DMLabel        subpointMap, hlabel;
1710   PetscInt       dim;
1711   PetscErrorCode ierr;
1712 
1713   PetscFunctionBegin;
1714   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1715   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1716   PetscValidPointer(dmHybrid, 4);
1717   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1718   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1719   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1720   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1721   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1722   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1723   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, NULL, PETSC_FALSE, idm);CHKERRQ(ierr);
1724   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1725   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1726   if (hybridLabel) *hybridLabel = hlabel;
1727   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1728   PetscFunctionReturn(0);
1729 }
1730 
1731 #undef __FUNCT__
1732 #define __FUNCT__ "DMPlexMarkSubmesh_Uninterpolated"
1733 /* Here we need the explicit assumption that:
1734 
1735      For any marked cell, the marked vertices constitute a single face
1736 */
1737 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1738 {
1739   IS               subvertexIS = NULL;
1740   const PetscInt  *subvertices;
1741   PetscInt        *pStart, *pEnd, *pMax, pSize;
1742   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1743   PetscErrorCode   ierr;
1744 
1745   PetscFunctionBegin;
1746   *numFaces = 0;
1747   *nFV      = 0;
1748   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1749   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1750   pSize = PetscMax(depth, dim) + 1;
1751   ierr = PetscMalloc3(pSize,&pStart,pSize,&pEnd,pSize,&pMax);CHKERRQ(ierr);
1752   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1753   for (d = 0; d <= depth; ++d) {
1754     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1755     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1756   }
1757   /* Loop over initial vertices and mark all faces in the collective star() */
1758   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1759   if (subvertexIS) {
1760     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1761     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1762   }
1763   for (v = 0; v < numSubVerticesInitial; ++v) {
1764     const PetscInt vertex = subvertices[v];
1765     PetscInt      *star   = NULL;
1766     PetscInt       starSize, s, numCells = 0, c;
1767 
1768     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1769     for (s = 0; s < starSize*2; s += 2) {
1770       const PetscInt point = star[s];
1771       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1772     }
1773     for (c = 0; c < numCells; ++c) {
1774       const PetscInt cell    = star[c];
1775       PetscInt      *closure = NULL;
1776       PetscInt       closureSize, cl;
1777       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1778 
1779       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1780       if (cellLoc == 2) continue;
1781       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1782       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1783       for (cl = 0; cl < closureSize*2; cl += 2) {
1784         const PetscInt point = closure[cl];
1785         PetscInt       vertexLoc;
1786 
1787         if ((point >= pStart[0]) && (point < pEnd[0])) {
1788           ++numCorners;
1789           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1790           if (vertexLoc == value) closure[faceSize++] = point;
1791         }
1792       }
1793       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1794       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1795       if (faceSize == *nFV) {
1796         const PetscInt *cells = NULL;
1797         PetscInt        numCells, nc;
1798 
1799         ++(*numFaces);
1800         for (cl = 0; cl < faceSize; ++cl) {
1801           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1802         }
1803         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1804         for (nc = 0; nc < numCells; ++nc) {
1805           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1806         }
1807         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1808       }
1809       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1810     }
1811     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1812   }
1813   if (subvertexIS) {
1814     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1815   }
1816   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1817   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1818   PetscFunctionReturn(0);
1819 }
1820 
1821 #undef __FUNCT__
1822 #define __FUNCT__ "DMPlexMarkSubmesh_Interpolated"
1823 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1824 {
1825   IS               subvertexIS;
1826   const PetscInt  *subvertices;
1827   PetscInt        *pStart, *pEnd, *pMax;
1828   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1829   PetscErrorCode   ierr;
1830 
1831   PetscFunctionBegin;
1832   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1833   ierr = PetscMalloc3(dim+1,&pStart,dim+1,&pEnd,dim+1,&pMax);CHKERRQ(ierr);
1834   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1835   for (d = 0; d <= dim; ++d) {
1836     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1837     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1838   }
1839   /* Loop over initial vertices and mark all faces in the collective star() */
1840   ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1841   if (subvertexIS) {
1842     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1843     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1844   }
1845   for (v = 0; v < numSubVerticesInitial; ++v) {
1846     const PetscInt vertex = subvertices[v];
1847     PetscInt      *star   = NULL;
1848     PetscInt       starSize, s, numFaces = 0, f;
1849 
1850     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1851     for (s = 0; s < starSize*2; s += 2) {
1852       const PetscInt point = star[s];
1853       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1854     }
1855     for (f = 0; f < numFaces; ++f) {
1856       const PetscInt face    = star[f];
1857       PetscInt      *closure = NULL;
1858       PetscInt       closureSize, c;
1859       PetscInt       faceLoc;
1860 
1861       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
1862       if (faceLoc == dim-1) continue;
1863       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
1864       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1865       for (c = 0; c < closureSize*2; c += 2) {
1866         const PetscInt point = closure[c];
1867         PetscInt       vertexLoc;
1868 
1869         if ((point >= pStart[0]) && (point < pEnd[0])) {
1870           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1871           if (vertexLoc != value) break;
1872         }
1873       }
1874       if (c == closureSize*2) {
1875         const PetscInt *support;
1876         PetscInt        supportSize, s;
1877 
1878         for (c = 0; c < closureSize*2; c += 2) {
1879           const PetscInt point = closure[c];
1880 
1881           for (d = 0; d < dim; ++d) {
1882             if ((point >= pStart[d]) && (point < pEnd[d])) {
1883               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
1884               break;
1885             }
1886           }
1887         }
1888         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
1889         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
1890         for (s = 0; s < supportSize; ++s) {
1891           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
1892         }
1893       }
1894       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1895     }
1896     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1897   }
1898   if (subvertexIS) {
1899     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1900   }
1901   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1902   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1903   PetscFunctionReturn(0);
1904 }
1905 
1906 #undef __FUNCT__
1907 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Uninterpolated"
1908 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
1909 {
1910   DMLabel         label = NULL;
1911   const PetscInt *cone;
1912   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize;
1913   PetscErrorCode  ierr;
1914 
1915   PetscFunctionBegin;
1916   *numFaces = 0;
1917   *nFV = 0;
1918   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1919   *subCells = NULL;
1920   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1921   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1922   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1923   if (cMax < 0) PetscFunctionReturn(0);
1924   if (label) {
1925     for (c = cMax; c < cEnd; ++c) {
1926       PetscInt val;
1927 
1928       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1929       if (val == value) {
1930         ++(*numFaces);
1931         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1932       }
1933     }
1934   } else {
1935     *numFaces = cEnd - cMax;
1936     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
1937   }
1938   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
1939   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
1940   for (c = cMax; c < cEnd; ++c) {
1941     const PetscInt *cells;
1942     PetscInt        numCells;
1943 
1944     if (label) {
1945       PetscInt val;
1946 
1947       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1948       if (val != value) continue;
1949     }
1950     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1951     for (p = 0; p < *nFV; ++p) {
1952       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
1953     }
1954     /* Negative face */
1955     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1956     /* Not true in parallel
1957     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
1958     for (p = 0; p < numCells; ++p) {
1959       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
1960       (*subCells)[subc++] = cells[p];
1961     }
1962     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
1963     /* Positive face is not included */
1964   }
1965   PetscFunctionReturn(0);
1966 }
1967 
1968 #undef __FUNCT__
1969 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Interpolated"
1970 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DMLabel subpointMap, DM subdm)
1971 {
1972   DMLabel        label = NULL;
1973   PetscInt      *pStart, *pEnd;
1974   PetscInt       dim, cMax, cEnd, c, d;
1975   PetscErrorCode ierr;
1976 
1977   PetscFunctionBegin;
1978   if (labelname) {ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
1979   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1980   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
1981   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1982   if (cMax < 0) PetscFunctionReturn(0);
1983   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
1984   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
1985   for (c = cMax; c < cEnd; ++c) {
1986     const PetscInt *cone;
1987     PetscInt       *closure = NULL;
1988     PetscInt        fconeSize, coneSize, closureSize, cl, val;
1989 
1990     if (label) {
1991       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
1992       if (val != value) continue;
1993     }
1994     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
1995     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
1996     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
1997     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
1998     /* Negative face */
1999     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2000     for (cl = 0; cl < closureSize*2; cl += 2) {
2001       const PetscInt point = closure[cl];
2002 
2003       for (d = 0; d <= dim; ++d) {
2004         if ((point >= pStart[d]) && (point < pEnd[d])) {
2005           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2006           break;
2007         }
2008       }
2009     }
2010     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2011     /* Cells -- positive face is not included */
2012     for (cl = 0; cl < 1; ++cl) {
2013       const PetscInt *support;
2014       PetscInt        supportSize, s;
2015 
2016       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
2017       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2018       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
2019       for (s = 0; s < supportSize; ++s) {
2020         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2021       }
2022     }
2023   }
2024   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2025   PetscFunctionReturn(0);
2026 }
2027 
2028 #undef __FUNCT__
2029 #define __FUNCT__ "DMPlexGetFaceOrientation"
2030 PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2031 {
2032   MPI_Comm       comm;
2033   PetscBool      posOrient = PETSC_FALSE;
2034   const PetscInt debug     = 0;
2035   PetscInt       cellDim, faceSize, f;
2036   PetscErrorCode ierr;
2037 
2038   PetscFunctionBegin;
2039   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2040   ierr = DMGetDimension(dm, &cellDim);CHKERRQ(ierr);
2041   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
2042 
2043   if (cellDim == 1 && numCorners == 2) {
2044     /* Triangle */
2045     faceSize  = numCorners-1;
2046     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2047   } else if (cellDim == 2 && numCorners == 3) {
2048     /* Triangle */
2049     faceSize  = numCorners-1;
2050     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2051   } else if (cellDim == 3 && numCorners == 4) {
2052     /* Tetrahedron */
2053     faceSize  = numCorners-1;
2054     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2055   } else if (cellDim == 1 && numCorners == 3) {
2056     /* Quadratic line */
2057     faceSize  = 1;
2058     posOrient = PETSC_TRUE;
2059   } else if (cellDim == 2 && numCorners == 4) {
2060     /* Quads */
2061     faceSize = 2;
2062     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2063       posOrient = PETSC_TRUE;
2064     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2065       posOrient = PETSC_TRUE;
2066     } else {
2067       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2068         posOrient = PETSC_FALSE;
2069       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2070     }
2071   } else if (cellDim == 2 && numCorners == 6) {
2072     /* Quadratic triangle (I hate this) */
2073     /* Edges are determined by the first 2 vertices (corners of edges) */
2074     const PetscInt faceSizeTri = 3;
2075     PetscInt       sortedIndices[3], i, iFace;
2076     PetscBool      found                    = PETSC_FALSE;
2077     PetscInt       faceVerticesTriSorted[9] = {
2078       0, 3,  4, /* bottom */
2079       1, 4,  5, /* right */
2080       2, 3,  5, /* left */
2081     };
2082     PetscInt       faceVerticesTri[9] = {
2083       0, 3,  4, /* bottom */
2084       1, 4,  5, /* right */
2085       2, 5,  3, /* left */
2086     };
2087 
2088     faceSize = faceSizeTri;
2089     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2090     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
2091     for (iFace = 0; iFace < 3; ++iFace) {
2092       const PetscInt ii = iFace*faceSizeTri;
2093       PetscInt       fVertex, cVertex;
2094 
2095       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2096           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2097         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2098           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2099             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2100               faceVertices[fVertex] = origVertices[cVertex];
2101               break;
2102             }
2103           }
2104         }
2105         found = PETSC_TRUE;
2106         break;
2107       }
2108     }
2109     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2110     if (posOriented) *posOriented = PETSC_TRUE;
2111     PetscFunctionReturn(0);
2112   } else if (cellDim == 2 && numCorners == 9) {
2113     /* Quadratic quad (I hate this) */
2114     /* Edges are determined by the first 2 vertices (corners of edges) */
2115     const PetscInt faceSizeQuad = 3;
2116     PetscInt       sortedIndices[3], i, iFace;
2117     PetscBool      found                      = PETSC_FALSE;
2118     PetscInt       faceVerticesQuadSorted[12] = {
2119       0, 1,  4, /* bottom */
2120       1, 2,  5, /* right */
2121       2, 3,  6, /* top */
2122       0, 3,  7, /* left */
2123     };
2124     PetscInt       faceVerticesQuad[12] = {
2125       0, 1,  4, /* bottom */
2126       1, 2,  5, /* right */
2127       2, 3,  6, /* top */
2128       3, 0,  7, /* left */
2129     };
2130 
2131     faceSize = faceSizeQuad;
2132     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2133     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
2134     for (iFace = 0; iFace < 4; ++iFace) {
2135       const PetscInt ii = iFace*faceSizeQuad;
2136       PetscInt       fVertex, cVertex;
2137 
2138       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2139           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2140         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2141           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2142             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2143               faceVertices[fVertex] = origVertices[cVertex];
2144               break;
2145             }
2146           }
2147         }
2148         found = PETSC_TRUE;
2149         break;
2150       }
2151     }
2152     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2153     if (posOriented) *posOriented = PETSC_TRUE;
2154     PetscFunctionReturn(0);
2155   } else if (cellDim == 3 && numCorners == 8) {
2156     /* Hexes
2157        A hex is two oriented quads with the normal of the first
2158        pointing up at the second.
2159 
2160           7---6
2161          /|  /|
2162         4---5 |
2163         | 1-|-2
2164         |/  |/
2165         0---3
2166 
2167         Faces are determined by the first 4 vertices (corners of faces) */
2168     const PetscInt faceSizeHex = 4;
2169     PetscInt       sortedIndices[4], i, iFace;
2170     PetscBool      found                     = PETSC_FALSE;
2171     PetscInt       faceVerticesHexSorted[24] = {
2172       0, 1, 2, 3,  /* bottom */
2173       4, 5, 6, 7,  /* top */
2174       0, 3, 4, 5,  /* front */
2175       2, 3, 5, 6,  /* right */
2176       1, 2, 6, 7,  /* back */
2177       0, 1, 4, 7,  /* left */
2178     };
2179     PetscInt       faceVerticesHex[24] = {
2180       1, 2, 3, 0,  /* bottom */
2181       4, 5, 6, 7,  /* top */
2182       0, 3, 5, 4,  /* front */
2183       3, 2, 6, 5,  /* right */
2184       2, 1, 7, 6,  /* back */
2185       1, 0, 4, 7,  /* left */
2186     };
2187 
2188     faceSize = faceSizeHex;
2189     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2190     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2191     for (iFace = 0; iFace < 6; ++iFace) {
2192       const PetscInt ii = iFace*faceSizeHex;
2193       PetscInt       fVertex, cVertex;
2194 
2195       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2196           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2197           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2198           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2199         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2200           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2201             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2202               faceVertices[fVertex] = origVertices[cVertex];
2203               break;
2204             }
2205           }
2206         }
2207         found = PETSC_TRUE;
2208         break;
2209       }
2210     }
2211     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2212     if (posOriented) *posOriented = PETSC_TRUE;
2213     PetscFunctionReturn(0);
2214   } else if (cellDim == 3 && numCorners == 10) {
2215     /* Quadratic tet */
2216     /* Faces are determined by the first 3 vertices (corners of faces) */
2217     const PetscInt faceSizeTet = 6;
2218     PetscInt       sortedIndices[6], i, iFace;
2219     PetscBool      found                     = PETSC_FALSE;
2220     PetscInt       faceVerticesTetSorted[24] = {
2221       0, 1, 2,  6, 7, 8, /* bottom */
2222       0, 3, 4,  6, 7, 9,  /* front */
2223       1, 4, 5,  7, 8, 9,  /* right */
2224       2, 3, 5,  6, 8, 9,  /* left */
2225     };
2226     PetscInt       faceVerticesTet[24] = {
2227       0, 1, 2,  6, 7, 8, /* bottom */
2228       0, 4, 3,  6, 7, 9,  /* front */
2229       1, 5, 4,  7, 8, 9,  /* right */
2230       2, 3, 5,  8, 6, 9,  /* left */
2231     };
2232 
2233     faceSize = faceSizeTet;
2234     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2235     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2236     for (iFace=0; iFace < 4; ++iFace) {
2237       const PetscInt ii = iFace*faceSizeTet;
2238       PetscInt       fVertex, cVertex;
2239 
2240       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2241           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2242           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2243           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2244         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2245           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2246             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2247               faceVertices[fVertex] = origVertices[cVertex];
2248               break;
2249             }
2250           }
2251         }
2252         found = PETSC_TRUE;
2253         break;
2254       }
2255     }
2256     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2257     if (posOriented) *posOriented = PETSC_TRUE;
2258     PetscFunctionReturn(0);
2259   } else if (cellDim == 3 && numCorners == 27) {
2260     /* Quadratic hexes (I hate this)
2261        A hex is two oriented quads with the normal of the first
2262        pointing up at the second.
2263 
2264          7---6
2265         /|  /|
2266        4---5 |
2267        | 3-|-2
2268        |/  |/
2269        0---1
2270 
2271        Faces are determined by the first 4 vertices (corners of faces) */
2272     const PetscInt faceSizeQuadHex = 9;
2273     PetscInt       sortedIndices[9], i, iFace;
2274     PetscBool      found                         = PETSC_FALSE;
2275     PetscInt       faceVerticesQuadHexSorted[54] = {
2276       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2277       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2278       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2279       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2280       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2281       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2282     };
2283     PetscInt       faceVerticesQuadHex[54] = {
2284       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2285       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2286       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2287       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2288       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2289       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2290     };
2291 
2292     faceSize = faceSizeQuadHex;
2293     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2294     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2295     for (iFace = 0; iFace < 6; ++iFace) {
2296       const PetscInt ii = iFace*faceSizeQuadHex;
2297       PetscInt       fVertex, cVertex;
2298 
2299       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2300           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2301           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2302           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2303         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2304           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2305             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2306               faceVertices[fVertex] = origVertices[cVertex];
2307               break;
2308             }
2309           }
2310         }
2311         found = PETSC_TRUE;
2312         break;
2313       }
2314     }
2315     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2316     if (posOriented) *posOriented = PETSC_TRUE;
2317     PetscFunctionReturn(0);
2318   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2319   if (!posOrient) {
2320     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2321     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2322   } else {
2323     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2324     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2325   }
2326   if (posOriented) *posOriented = posOrient;
2327   PetscFunctionReturn(0);
2328 }
2329 
2330 #undef __FUNCT__
2331 #define __FUNCT__ "DMPlexGetOrientedFace"
2332 /*
2333     Given a cell and a face, as a set of vertices,
2334       return the oriented face, as a set of vertices, in faceVertices
2335     The orientation is such that the face normal points out of the cell
2336 */
2337 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2338 {
2339   const PetscInt *cone = NULL;
2340   PetscInt        coneSize, v, f, v2;
2341   PetscInt        oppositeVertex = -1;
2342   PetscErrorCode  ierr;
2343 
2344   PetscFunctionBegin;
2345   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2346   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2347   for (v = 0, v2 = 0; v < coneSize; ++v) {
2348     PetscBool found = PETSC_FALSE;
2349 
2350     for (f = 0; f < faceSize; ++f) {
2351       if (face[f] == cone[v]) {
2352         found = PETSC_TRUE; break;
2353       }
2354     }
2355     if (found) {
2356       indices[v2]      = v;
2357       origVertices[v2] = cone[v];
2358       ++v2;
2359     } else {
2360       oppositeVertex = v;
2361     }
2362   }
2363   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2364   PetscFunctionReturn(0);
2365 }
2366 
2367 #undef __FUNCT__
2368 #define __FUNCT__ "DMPlexInsertFace_Internal"
2369 /*
2370   DMPlexInsertFace_Internal - Puts a face into the mesh
2371 
2372   Not collective
2373 
2374   Input Parameters:
2375   + dm              - The DMPlex
2376   . numFaceVertex   - The number of vertices in the face
2377   . faceVertices    - The vertices in the face for dm
2378   . subfaceVertices - The vertices in the face for subdm
2379   . numCorners      - The number of vertices in the cell
2380   . cell            - A cell in dm containing the face
2381   . subcell         - A cell in subdm containing the face
2382   . firstFace       - First face in the mesh
2383   - newFacePoint    - Next face in the mesh
2384 
2385   Output Parameters:
2386   . newFacePoint - Contains next face point number on input, updated on output
2387 
2388   Level: developer
2389 */
2390 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2391 {
2392   MPI_Comm        comm;
2393   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2394   const PetscInt *faces;
2395   PetscInt        numFaces, coneSize;
2396   PetscErrorCode  ierr;
2397 
2398   PetscFunctionBegin;
2399   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2400   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2401   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2402 #if 0
2403   /* Cannot use this because support() has not been constructed yet */
2404   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2405 #else
2406   {
2407     PetscInt f;
2408 
2409     numFaces = 0;
2410     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2411     for (f = firstFace; f < *newFacePoint; ++f) {
2412       PetscInt dof, off, d;
2413 
2414       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2415       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2416       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2417       for (d = 0; d < dof; ++d) {
2418         const PetscInt p = submesh->cones[off+d];
2419         PetscInt       v;
2420 
2421         for (v = 0; v < numFaceVertices; ++v) {
2422           if (subfaceVertices[v] == p) break;
2423         }
2424         if (v == numFaceVertices) break;
2425       }
2426       if (d == dof) {
2427         numFaces               = 1;
2428         ((PetscInt*) faces)[0] = f;
2429       }
2430     }
2431   }
2432 #endif
2433   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2434   else if (numFaces == 1) {
2435     /* Add the other cell neighbor for this face */
2436     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2437   } else {
2438     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2439     PetscBool posOriented;
2440 
2441     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2442     origVertices        = &orientedVertices[numFaceVertices];
2443     indices             = &orientedVertices[numFaceVertices*2];
2444     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2445     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2446     /* TODO: I know that routine should return a permutation, not the indices */
2447     for (v = 0; v < numFaceVertices; ++v) {
2448       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2449       for (ov = 0; ov < numFaceVertices; ++ov) {
2450         if (orientedVertices[ov] == vertex) {
2451           orientedSubVertices[ov] = subvertex;
2452           break;
2453         }
2454       }
2455       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2456     }
2457     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2458     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2459     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2460     ++(*newFacePoint);
2461   }
2462 #if 0
2463   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2464 #else
2465   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2466 #endif
2467   PetscFunctionReturn(0);
2468 }
2469 
2470 #undef __FUNCT__
2471 #define __FUNCT__ "DMPlexCreateSubmesh_Uninterpolated"
2472 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2473 {
2474   MPI_Comm        comm;
2475   DMLabel         subpointMap;
2476   IS              subvertexIS,  subcellIS;
2477   const PetscInt *subVertices, *subCells;
2478   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2479   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2480   PetscInt        vStart, vEnd, c, f;
2481   PetscErrorCode  ierr;
2482 
2483   PetscFunctionBegin;
2484   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2485   /* Create subpointMap which marks the submesh */
2486   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2487   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2488   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2489   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2490   /* Setup chart */
2491   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2492   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2493   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2494   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2495   /* Set cone sizes */
2496   firstSubVertex = numSubCells;
2497   firstSubFace   = numSubCells+numSubVertices;
2498   newFacePoint   = firstSubFace;
2499   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2500   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2501   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2502   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2503   for (c = 0; c < numSubCells; ++c) {
2504     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2505   }
2506   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2507     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2508   }
2509   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2510   /* Create face cones */
2511   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2512   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2513   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2514   for (c = 0; c < numSubCells; ++c) {
2515     const PetscInt cell    = subCells[c];
2516     const PetscInt subcell = c;
2517     PetscInt      *closure = NULL;
2518     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2519 
2520     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2521     for (cl = 0; cl < closureSize*2; cl += 2) {
2522       const PetscInt point = closure[cl];
2523       PetscInt       subVertex;
2524 
2525       if ((point >= vStart) && (point < vEnd)) {
2526         ++numCorners;
2527         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2528         if (subVertex >= 0) {
2529           closure[faceSize] = point;
2530           subface[faceSize] = firstSubVertex+subVertex;
2531           ++faceSize;
2532         }
2533       }
2534     }
2535     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2536     if (faceSize == nFV) {
2537       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2538     }
2539     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2540   }
2541   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2542   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2543   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2544   /* Build coordinates */
2545   {
2546     PetscSection coordSection, subCoordSection;
2547     Vec          coordinates, subCoordinates;
2548     PetscScalar *coords, *subCoords;
2549     PetscInt     numComp, coordSize, v;
2550 
2551     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2552     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2553     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2554     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2555     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2556     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2557     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2558     for (v = 0; v < numSubVertices; ++v) {
2559       const PetscInt vertex    = subVertices[v];
2560       const PetscInt subvertex = firstSubVertex+v;
2561       PetscInt       dof;
2562 
2563       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2564       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2565       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2566     }
2567     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2568     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2569     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2570     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2571     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2572     if (coordSize) {
2573       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2574       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2575       for (v = 0; v < numSubVertices; ++v) {
2576         const PetscInt vertex    = subVertices[v];
2577         const PetscInt subvertex = firstSubVertex+v;
2578         PetscInt       dof, off, sdof, soff, d;
2579 
2580         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2581         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2582         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2583         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2584         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2585         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2586       }
2587       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2588       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2589     }
2590     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2591     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2592   }
2593   /* Cleanup */
2594   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2595   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2596   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2597   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2598   PetscFunctionReturn(0);
2599 }
2600 
2601 #undef __FUNCT__
2602 #define __FUNCT__ "DMPlexCreateSubmesh_Interpolated"
2603 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2604 {
2605   MPI_Comm         comm;
2606   DMLabel          subpointMap;
2607   IS              *subpointIS;
2608   const PetscInt **subpoints;
2609   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *coneONew;
2610   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2611   PetscErrorCode   ierr;
2612 
2613   PetscFunctionBegin;
2614   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2615   /* Create subpointMap which marks the submesh */
2616   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2617   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2618   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2619   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Interpolated(dm, vertexLabel, value, subpointMap, subdm);CHKERRQ(ierr);}
2620   /* Setup chart */
2621   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2622   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
2623   for (d = 0; d <= dim; ++d) {
2624     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2625     totSubPoints += numSubPoints[d];
2626   }
2627   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2628   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2629   /* Set cone sizes */
2630   firstSubPoint[dim] = 0;
2631   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2632   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2633   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2634   for (d = 0; d <= dim; ++d) {
2635     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2636     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2637   }
2638   for (d = 0; d <= dim; ++d) {
2639     for (p = 0; p < numSubPoints[d]; ++p) {
2640       const PetscInt  point    = subpoints[d][p];
2641       const PetscInt  subpoint = firstSubPoint[d] + p;
2642       const PetscInt *cone;
2643       PetscInt        coneSize, coneSizeNew, c, val;
2644 
2645       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2646       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2647       if (d == dim) {
2648         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2649         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2650           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2651           if (val >= 0) coneSizeNew++;
2652         }
2653         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2654       }
2655     }
2656   }
2657   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2658   /* Set cones */
2659   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2660   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&coneONew);CHKERRQ(ierr);
2661   for (d = 0; d <= dim; ++d) {
2662     for (p = 0; p < numSubPoints[d]; ++p) {
2663       const PetscInt  point    = subpoints[d][p];
2664       const PetscInt  subpoint = firstSubPoint[d] + p;
2665       const PetscInt *cone, *ornt;
2666       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2667 
2668       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2669       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2670       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2671       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2672       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2673         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2674         if (subc >= 0) {
2675           coneNew[coneSizeNew]  = firstSubPoint[d-1] + subc;
2676           coneONew[coneSizeNew] = ornt[c];
2677           ++coneSizeNew;
2678         }
2679       }
2680       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2681       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2682       ierr = DMPlexSetConeOrientation(subdm, subpoint, coneONew);CHKERRQ(ierr);
2683     }
2684   }
2685   ierr = PetscFree2(coneNew,coneONew);CHKERRQ(ierr);
2686   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2687   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2688   /* Build coordinates */
2689   {
2690     PetscSection coordSection, subCoordSection;
2691     Vec          coordinates, subCoordinates;
2692     PetscScalar *coords, *subCoords;
2693     PetscInt     numComp, coordSize;
2694 
2695     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2696     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2697     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2698     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2699     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2700     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2701     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2702     for (v = 0; v < numSubPoints[0]; ++v) {
2703       const PetscInt vertex    = subpoints[0][v];
2704       const PetscInt subvertex = firstSubPoint[0]+v;
2705       PetscInt       dof;
2706 
2707       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2708       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2709       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2710     }
2711     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2712     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2713     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2714     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2715     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2716     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2717     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2718     for (v = 0; v < numSubPoints[0]; ++v) {
2719       const PetscInt vertex    = subpoints[0][v];
2720       const PetscInt subvertex = firstSubPoint[0]+v;
2721       PetscInt dof, off, sdof, soff, d;
2722 
2723       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2724       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2725       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2726       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2727       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2728       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2729     }
2730     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2731     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2732     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2733     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2734   }
2735   /* Cleanup */
2736   for (d = 0; d <= dim; ++d) {
2737     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2738     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
2739   }
2740   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
2741   PetscFunctionReturn(0);
2742 }
2743 
2744 #undef __FUNCT__
2745 #define __FUNCT__ "DMPlexCreateSubmesh"
2746 /*@
2747   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
2748 
2749   Input Parameters:
2750 + dm           - The original mesh
2751 . vertexLabel  - The DMLabel marking vertices contained in the surface
2752 - value        - The label value to use
2753 
2754   Output Parameter:
2755 . subdm - The surface mesh
2756 
2757   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
2758 
2759   Level: developer
2760 
2761 .seealso: DMPlexGetSubpointMap(), DMPlexGetLabel(), DMLabelSetValue()
2762 @*/
2763 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
2764 {
2765   PetscInt       dim, depth;
2766   PetscErrorCode ierr;
2767 
2768   PetscFunctionBegin;
2769   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2770   PetscValidPointer(subdm, 3);
2771   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2772   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2773   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
2774   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
2775   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
2776   if (depth == dim) {
2777     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2778   } else {
2779     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
2780   }
2781   PetscFunctionReturn(0);
2782 }
2783 
2784 #undef __FUNCT__
2785 #define __FUNCT__ "DMPlexFilterPoint_Internal"
2786 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2787 {
2788   PetscInt       subPoint;
2789   PetscErrorCode ierr;
2790 
2791   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2792   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2793 }
2794 
2795 #undef __FUNCT__
2796 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Uninterpolated"
2797 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
2798 {
2799   MPI_Comm        comm;
2800   DMLabel         subpointMap;
2801   IS              subvertexIS;
2802   const PetscInt *subVertices;
2803   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
2804   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
2805   PetscInt        cMax, c, f;
2806   PetscErrorCode  ierr;
2807 
2808   PetscFunctionBegin;
2809   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
2810   /* Create subpointMap which marks the submesh */
2811   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2812   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2813   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2814   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
2815   /* Setup chart */
2816   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2817   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2818   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2819   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2820   /* Set cone sizes */
2821   firstSubVertex = numSubCells;
2822   firstSubFace   = numSubCells+numSubVertices;
2823   newFacePoint   = firstSubFace;
2824   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2825   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2826   for (c = 0; c < numSubCells; ++c) {
2827     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2828   }
2829   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2830     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2831   }
2832   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2833   /* Create face cones */
2834   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2835   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2836   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2837   for (c = 0; c < numSubCells; ++c) {
2838     const PetscInt  cell    = subCells[c];
2839     const PetscInt  subcell = c;
2840     const PetscInt *cone, *cells;
2841     PetscInt        numCells, subVertex, p, v;
2842 
2843     if (cell < cMax) continue;
2844     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2845     for (v = 0; v < nFV; ++v) {
2846       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2847       subface[v] = firstSubVertex+subVertex;
2848     }
2849     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
2850     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
2851     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2852     /* Not true in parallel
2853     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2854     for (p = 0; p < numCells; ++p) {
2855       PetscInt negsubcell;
2856 
2857       if (cells[p] >= cMax) continue;
2858       /* I know this is a crap search */
2859       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
2860         if (subCells[negsubcell] == cells[p]) break;
2861       }
2862       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
2863       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
2864     }
2865     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2866     ++newFacePoint;
2867   }
2868   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2869   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2870   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2871   /* Build coordinates */
2872   {
2873     PetscSection coordSection, subCoordSection;
2874     Vec          coordinates, subCoordinates;
2875     PetscScalar *coords, *subCoords;
2876     PetscInt     numComp, coordSize, v;
2877 
2878     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2879     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2880     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2881     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2882     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2883     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2884     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2885     for (v = 0; v < numSubVertices; ++v) {
2886       const PetscInt vertex    = subVertices[v];
2887       const PetscInt subvertex = firstSubVertex+v;
2888       PetscInt       dof;
2889 
2890       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2891       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2892       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2893     }
2894     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2895     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2896     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2897     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2898     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2899     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2900     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2901     for (v = 0; v < numSubVertices; ++v) {
2902       const PetscInt vertex    = subVertices[v];
2903       const PetscInt subvertex = firstSubVertex+v;
2904       PetscInt       dof, off, sdof, soff, d;
2905 
2906       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2907       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2908       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2909       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2910       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2911       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2912     }
2913     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2914     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2915     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2916     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2917   }
2918   /* Build SF */
2919   CHKMEMQ;
2920   {
2921     PetscSF            sfPoint, sfPointSub;
2922     const PetscSFNode *remotePoints;
2923     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2924     const PetscInt    *localPoints;
2925     PetscInt          *slocalPoints;
2926     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
2927     PetscMPIInt        rank;
2928 
2929     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2930     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2931     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2932     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2933     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2934     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2935     if (numRoots >= 0) {
2936       /* Only vertices should be shared */
2937       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
2938       for (p = 0; p < pEnd-pStart; ++p) {
2939         newLocalPoints[p].rank  = -2;
2940         newLocalPoints[p].index = -2;
2941       }
2942       /* Set subleaves */
2943       for (l = 0; l < numLeaves; ++l) {
2944         const PetscInt point    = localPoints[l];
2945         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2946 
2947         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
2948         if (subPoint < 0) continue;
2949         newLocalPoints[point-pStart].rank  = rank;
2950         newLocalPoints[point-pStart].index = subPoint;
2951         ++numSubLeaves;
2952       }
2953       /* Must put in owned subpoints */
2954       for (p = pStart; p < pEnd; ++p) {
2955         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
2956 
2957         if (subPoint < 0) {
2958           newOwners[p-pStart].rank  = -3;
2959           newOwners[p-pStart].index = -3;
2960         } else {
2961           newOwners[p-pStart].rank  = rank;
2962           newOwners[p-pStart].index = subPoint;
2963         }
2964       }
2965       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2966       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
2967       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2968       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
2969       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
2970       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
2971       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
2972         const PetscInt point    = localPoints[l];
2973         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
2974 
2975         if (subPoint < 0) continue;
2976         if (newLocalPoints[point].rank == rank) {++ll; continue;}
2977         slocalPoints[sl]        = subPoint;
2978         sremotePoints[sl].rank  = newLocalPoints[point].rank;
2979         sremotePoints[sl].index = newLocalPoints[point].index;
2980         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
2981         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
2982         ++sl;
2983       }
2984       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
2985       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
2986       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
2987     }
2988   }
2989   CHKMEMQ;
2990   /* Cleanup */
2991   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2992   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2993   ierr = PetscFree(subCells);CHKERRQ(ierr);
2994   PetscFunctionReturn(0);
2995 }
2996 
2997 #undef __FUNCT__
2998 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Interpolated"
2999 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char label[], PetscInt value, DM subdm)
3000 {
3001   MPI_Comm         comm;
3002   DMLabel          subpointMap;
3003   IS              *subpointIS;
3004   const PetscInt **subpoints;
3005   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3006   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3007   PetscErrorCode   ierr;
3008 
3009   PetscFunctionBegin;
3010   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3011   /* Create subpointMap which marks the submesh */
3012   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
3013   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3014   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3015   ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);
3016   /* Setup chart */
3017   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3018   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
3019   for (d = 0; d <= dim; ++d) {
3020     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
3021     totSubPoints += numSubPoints[d];
3022   }
3023   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
3024   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
3025   /* Set cone sizes */
3026   firstSubPoint[dim] = 0;
3027   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3028   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3029   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3030   for (d = 0; d <= dim; ++d) {
3031     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
3032     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3033   }
3034   for (d = 0; d <= dim; ++d) {
3035     for (p = 0; p < numSubPoints[d]; ++p) {
3036       const PetscInt  point    = subpoints[d][p];
3037       const PetscInt  subpoint = firstSubPoint[d] + p;
3038       const PetscInt *cone;
3039       PetscInt        coneSize, coneSizeNew, c, val;
3040 
3041       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3042       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
3043       if (d == dim) {
3044         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3045         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3046           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
3047           if (val >= 0) coneSizeNew++;
3048         }
3049         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
3050       }
3051     }
3052   }
3053   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3054   /* Set cones */
3055   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3056   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
3057   for (d = 0; d <= dim; ++d) {
3058     for (p = 0; p < numSubPoints[d]; ++p) {
3059       const PetscInt  point    = subpoints[d][p];
3060       const PetscInt  subpoint = firstSubPoint[d] + p;
3061       const PetscInt *cone, *ornt;
3062       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
3063 
3064       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3065       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
3066       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3067       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3068       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3069         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
3070         if (subc >= 0) {
3071           coneNew[coneSizeNew]   = firstSubPoint[d-1] + subc;
3072           orntNew[coneSizeNew++] = ornt[c];
3073         }
3074       }
3075       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
3076       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
3077       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
3078     }
3079   }
3080   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
3081   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3082   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3083   /* Build coordinates */
3084   {
3085     PetscSection coordSection, subCoordSection;
3086     Vec          coordinates, subCoordinates;
3087     PetscScalar *coords, *subCoords;
3088     PetscInt     numComp, coordSize;
3089 
3090     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3091     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3092     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3093     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3094     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3095     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3096     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
3097     for (v = 0; v < numSubPoints[0]; ++v) {
3098       const PetscInt vertex    = subpoints[0][v];
3099       const PetscInt subvertex = firstSubPoint[0]+v;
3100       PetscInt       dof;
3101 
3102       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3103       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3104       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3105     }
3106     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3107     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3108     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
3109     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3110     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3111     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3112     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3113     for (v = 0; v < numSubPoints[0]; ++v) {
3114       const PetscInt vertex    = subpoints[0][v];
3115       const PetscInt subvertex = firstSubPoint[0]+v;
3116       PetscInt dof, off, sdof, soff, d;
3117 
3118       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3119       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3120       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3121       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3122       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3123       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3124     }
3125     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3126     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3127     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3128     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3129   }
3130   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3131   {
3132     PetscSF            sfPoint, sfPointSub;
3133     IS                 subpIS;
3134     const PetscSFNode *remotePoints;
3135     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3136     const PetscInt    *localPoints, *subpoints;
3137     PetscInt          *slocalPoints;
3138     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3139     PetscMPIInt        rank;
3140 
3141     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
3142     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3143     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3144     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3145     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
3146     ierr = DMPlexCreateSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
3147     if (subpIS) {
3148       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
3149       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
3150     }
3151     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3152     if (numRoots >= 0) {
3153       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3154       for (p = 0; p < pEnd-pStart; ++p) {
3155         newLocalPoints[p].rank  = -2;
3156         newLocalPoints[p].index = -2;
3157       }
3158       /* Set subleaves */
3159       for (l = 0; l < numLeaves; ++l) {
3160         const PetscInt point    = localPoints[l];
3161         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3162 
3163         if (subpoint < 0) continue;
3164         newLocalPoints[point-pStart].rank  = rank;
3165         newLocalPoints[point-pStart].index = subpoint;
3166         ++numSubleaves;
3167       }
3168       /* Must put in owned subpoints */
3169       for (p = pStart; p < pEnd; ++p) {
3170         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3171 
3172         if (subpoint < 0) {
3173           newOwners[p-pStart].rank  = -3;
3174           newOwners[p-pStart].index = -3;
3175         } else {
3176           newOwners[p-pStart].rank  = rank;
3177           newOwners[p-pStart].index = subpoint;
3178         }
3179       }
3180       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3181       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3182       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3183       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3184       ierr = PetscMalloc1(numSubleaves, &slocalPoints);CHKERRQ(ierr);
3185       ierr = PetscMalloc1(numSubleaves, &sremotePoints);CHKERRQ(ierr);
3186       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3187         const PetscInt point    = localPoints[l];
3188         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3189 
3190         if (subpoint < 0) continue;
3191         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3192         slocalPoints[sl]        = subpoint;
3193         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3194         sremotePoints[sl].index = newLocalPoints[point].index;
3195         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3196         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3197         ++sl;
3198       }
3199       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3200       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3201       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3202     }
3203     if (subpIS) {
3204       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3205       ierr = ISDestroy(&subpIS);CHKERRQ(ierr);
3206     }
3207   }
3208   /* Cleanup */
3209   for (d = 0; d <= dim; ++d) {
3210     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3211     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3212   }
3213   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3214   PetscFunctionReturn(0);
3215 }
3216 
3217 #undef __FUNCT__
3218 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh"
3219 /*
3220   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3221 
3222   Input Parameters:
3223 + dm          - The original mesh
3224 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3225 . label       - A label name, or NULL
3226 - value  - A label value
3227 
3228   Output Parameter:
3229 . subdm - The surface mesh
3230 
3231   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3232 
3233   Level: developer
3234 
3235 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3236 */
3237 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3238 {
3239   PetscInt       dim, depth;
3240   PetscErrorCode ierr;
3241 
3242   PetscFunctionBegin;
3243   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3244   PetscValidPointer(subdm, 5);
3245   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3246   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3247   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3248   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3249   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3250   if (depth == dim) {
3251     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3252   } else {
3253     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3254   }
3255   PetscFunctionReturn(0);
3256 }
3257 
3258 #undef __FUNCT__
3259 #define __FUNCT__ "DMPlexGetSubpointMap"
3260 /*@
3261   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3262 
3263   Input Parameter:
3264 . dm - The submesh DM
3265 
3266   Output Parameter:
3267 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3268 
3269   Level: developer
3270 
3271 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3272 @*/
3273 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3274 {
3275   DM_Plex *mesh = (DM_Plex*) dm->data;
3276 
3277   PetscFunctionBegin;
3278   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3279   PetscValidPointer(subpointMap, 2);
3280   *subpointMap = mesh->subpointMap;
3281   PetscFunctionReturn(0);
3282 }
3283 
3284 #undef __FUNCT__
3285 #define __FUNCT__ "DMPlexSetSubpointMap"
3286 /* Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh() */
3287 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3288 {
3289   DM_Plex       *mesh = (DM_Plex *) dm->data;
3290   DMLabel        tmp;
3291   PetscErrorCode ierr;
3292 
3293   PetscFunctionBegin;
3294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3295   tmp  = mesh->subpointMap;
3296   mesh->subpointMap = subpointMap;
3297   ++mesh->subpointMap->refct;
3298   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3299   PetscFunctionReturn(0);
3300 }
3301 
3302 #undef __FUNCT__
3303 #define __FUNCT__ "DMPlexCreateSubpointIS"
3304 /*@
3305   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
3306 
3307   Input Parameter:
3308 . dm - The submesh DM
3309 
3310   Output Parameter:
3311 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3312 
3313   Note: This is IS is guaranteed to be sorted by the construction of the submesh
3314 
3315   Level: developer
3316 
3317 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3318 @*/
3319 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3320 {
3321   MPI_Comm        comm;
3322   DMLabel         subpointMap;
3323   IS              is;
3324   const PetscInt *opoints;
3325   PetscInt       *points, *depths;
3326   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3327   PetscErrorCode  ierr;
3328 
3329   PetscFunctionBegin;
3330   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3331   PetscValidPointer(subpointIS, 2);
3332   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3333   *subpointIS = NULL;
3334   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3335   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3336   if (subpointMap && depth >= 0) {
3337     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3338     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3339     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3340     depths[0] = depth;
3341     depths[1] = 0;
3342     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3343     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3344     for(d = 0, off = 0; d <= depth; ++d) {
3345       const PetscInt dep = depths[d];
3346 
3347       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3348       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3349       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3350         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3351       } else {
3352         if (!n) {
3353           if (d == 0) {
3354             /* Missing cells */
3355             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3356           } else {
3357             /* Missing faces */
3358             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3359           }
3360         }
3361       }
3362       if (n) {
3363         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3364         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3365         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3366         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3367         ierr = ISDestroy(&is);CHKERRQ(ierr);
3368       }
3369     }
3370     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3371     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3372     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3373   }
3374   PetscFunctionReturn(0);
3375 }
3376