xref: /petsc/src/dm/impls/plex/plex.c (revision cc85fe4ded5189db5e5e073ce90ef04de0003fdb)
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 
6 /* Logging support */
7 PetscLogEvent DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_Stratify;
8 
9 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
10 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
11 
12 #undef __FUNCT__
13 #define __FUNCT__ "VecView_Plex_Local"
14 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
15 {
16   DM             dm;
17   PetscBool      isvtk;
18   PetscErrorCode ierr;
19 
20   PetscFunctionBegin;
21   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
22   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
23   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);CHKERRQ(ierr);
24   if (isvtk) {
25     PetscViewerVTKFieldType ft = PETSC_VTK_POINT_FIELD;
26     PetscSection            section;
27     PetscInt                dim, pStart, pEnd, cStart, fStart, vStart, cdof = 0, fdof = 0, vdof = 0;
28 
29     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
30     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
31     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
32     ierr = DMPlexGetHeightStratum(dm, 1, &fStart, NULL);CHKERRQ(ierr);
33     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, NULL);CHKERRQ(ierr);
34     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
35     /* Assumes that numer of dofs per point of each stratum is constant, natural for VTK */
36     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
37     if ((fStart >= pStart) && (fStart < pEnd)) {ierr = PetscSectionGetDof(section, fStart, &fdof);CHKERRQ(ierr);}
38     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
39     if (cdof && fdof && vdof) { /* Actually Q2 or some such, but visualize as Q1 */
40       ft = (cdof == dim) ? PETSC_VTK_POINT_VECTOR_FIELD : PETSC_VTK_POINT_FIELD;
41     } else if (cdof && vdof) {
42       SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"No support for viewing mixed space with dofs at both vertices and cells");
43     } else if (cdof) {
44       /* TODO: This assumption should be removed when there is a way of identifying whether a space is conceptually a
45        * vector or just happens to have the same number of dofs as the dimension. */
46       if (cdof == dim) {
47         ft = PETSC_VTK_CELL_VECTOR_FIELD;
48       } else {
49         ft = PETSC_VTK_CELL_FIELD;
50       }
51     } else if (vdof) {
52       if (vdof == dim) {
53         ft = PETSC_VTK_POINT_VECTOR_FIELD;
54       } else {
55         ft = PETSC_VTK_POINT_FIELD;
56       }
57     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
58 
59     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
60     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
61     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
62   } else {
63     PetscBool isseq;
64 
65     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
66     if (isseq) {
67       ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);
68     } else {
69       ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);
70     }
71   }
72   PetscFunctionReturn(0);
73 }
74 
75 #undef __FUNCT__
76 #define __FUNCT__ "VecView_Plex"
77 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
78 {
79   DM             dm;
80   PetscBool      isvtk;
81   PetscErrorCode ierr;
82 
83   PetscFunctionBegin;
84   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
85   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
86   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);CHKERRQ(ierr);
87   if (isvtk) {
88     Vec         locv;
89     const char *name;
90 
91     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
92     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
93     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
94     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
95     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
96     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
97     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
98   } else {
99     PetscBool isseq;
100 
101     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
102     if (isseq) {
103       ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);
104     } else {
105       ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);
106     }
107   }
108   PetscFunctionReturn(0);
109 }
110 
111 #undef __FUNCT__
112 #define __FUNCT__ "DMPlexView_Ascii"
113 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
114 {
115   DM_Plex          *mesh = (DM_Plex*) dm->data;
116   DM                cdm;
117   DMLabel           markers;
118   PetscSection      coordSection;
119   Vec               coordinates;
120   PetscViewerFormat format;
121   PetscErrorCode    ierr;
122 
123   PetscFunctionBegin;
124   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
125   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
126   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
127   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
128   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
129     const char *name;
130     PetscInt    maxConeSize, maxSupportSize;
131     PetscInt    pStart, pEnd, p;
132     PetscMPIInt rank, size;
133 
134     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
135     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
136     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
137     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
138     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
139     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
140     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
141     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
142     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
143     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
144     for (p = pStart; p < pEnd; ++p) {
145       PetscInt dof, off, s;
146 
147       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
148       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
149       for (s = off; s < off+dof; ++s) {
150         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
151       }
152     }
153     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
154     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
155     for (p = pStart; p < pEnd; ++p) {
156       PetscInt dof, off, c;
157 
158       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
159       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
160       for (c = off; c < off+dof; ++c) {
161         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
162       }
163     }
164     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
165     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
166     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
167     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
168     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
169     if (size > 1) {
170       PetscSF sf;
171 
172       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
173       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
174     }
175     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
176   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
177     const char  *name;
178     const char  *colors[3] = {"red", "blue", "green"};
179     const int    numColors  = 3;
180     PetscReal    scale      = 2.0;
181     PetscScalar *coords;
182     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
183     PetscMPIInt  rank, size;
184 
185     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
186     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
187     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
188     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
189     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
190     ierr = PetscViewerASCIIPrintf(viewer, "\
191 \\documentclass[crop,multi=false]{standalone}\n\n\
192 \\usepackage{tikz}\n\
193 \\usepackage{pgflibraryshapes}\n\
194 \\usetikzlibrary{backgrounds}\n\
195 \\usetikzlibrary{arrows}\n\
196 \\begin{document}\n\
197 \\section{%s}\n\
198 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
199     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
200     for (p = 0; p < size; ++p) {
201       if (p > 0 && p == size-1) {
202         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
203       } else if (p > 0) {
204         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
205       }
206       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
207     }
208     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
209 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
210     /* Plot vertices */
211     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
212     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
213     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
214     for (v = vStart; v < vEnd; ++v) {
215       PetscInt off, dof, d;
216 
217       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
218       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
219       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
220       for (d = 0; d < dof; ++d) {
221         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
222         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%G", scale*PetscRealPart(coords[off+d]));CHKERRQ(ierr);
223       }
224       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
225     }
226     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
227     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
228     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
229     /* Plot edges */
230     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
231     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
232     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
233     for (e = eStart; e < eEnd; ++e) {
234       const PetscInt *cone;
235       PetscInt        coneSize, offA, offB, dof, d;
236 
237       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
238       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
239       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
240       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
241       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
242       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
243       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
244       for (d = 0; d < dof; ++d) {
245         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
246         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%G", scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d]));CHKERRQ(ierr);
247       }
248       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
249     }
250     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
251     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
252     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
253     /* Plot cells */
254     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
255     for (c = cStart; c < cEnd; ++c) {
256       PetscInt *closure = NULL;
257       PetscInt  closureSize, firstPoint = -1;
258 
259       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
260       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
261       for (p = 0; p < closureSize*2; p += 2) {
262         const PetscInt point = closure[p];
263 
264         if ((point < vStart) || (point >= vEnd)) continue;
265         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
266         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
267         if (firstPoint < 0) firstPoint = point;
268       }
269       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
270       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
271       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
272     }
273     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
274     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
275     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
276   } else {
277     MPI_Comm    comm;
278     PetscInt   *sizes;
279     PetscInt    locDepth, depth, dim, d;
280     PetscInt    pStart, pEnd, p;
281     PetscInt    numLabels, l;
282     PetscMPIInt size;
283 
284     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
285     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
286     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
287     ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);
288     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
289     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
290     ierr = PetscMalloc(size * sizeof(PetscInt), &sizes);CHKERRQ(ierr);
291     if (depth == 1) {
292       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
293       pEnd = pEnd - pStart;
294       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
295       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
296       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
297       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
298       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
299       pEnd = pEnd - pStart;
300       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
301       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
302       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
303       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
304     } else {
305       for (d = 0; d <= dim; d++) {
306         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
307         pEnd = pEnd - pStart;
308         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
309         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
310         for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
311         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
312       }
313     }
314     ierr = PetscFree(sizes);CHKERRQ(ierr);
315     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
316     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
317     for (l = 0; l < numLabels; ++l) {
318       DMLabel         label;
319       const char     *name;
320       IS              valueIS;
321       const PetscInt *values;
322       PetscInt        numValues, v;
323 
324       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
325       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
326       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
327       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
328       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
329       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
330       for (v = 0; v < numValues; ++v) {
331         PetscInt size;
332 
333         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
334         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
335         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
336       }
337       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
338       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
339       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
340     }
341   }
342   PetscFunctionReturn(0);
343 }
344 
345 #undef __FUNCT__
346 #define __FUNCT__ "DMView_Plex"
347 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
348 {
349   PetscBool      iascii, isbinary;
350   PetscErrorCode ierr;
351 
352   PetscFunctionBegin;
353   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
354   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
355   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
356   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
357   if (iascii) {
358     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
359 #if 0
360   } else if (isbinary) {
361     ierr = DMPlexView_Binary(dm, viewer);CHKERRQ(ierr);
362 #endif
363   }
364   PetscFunctionReturn(0);
365 }
366 
367 #undef __FUNCT__
368 #define __FUNCT__ "DMDestroy_Plex"
369 PetscErrorCode DMDestroy_Plex(DM dm)
370 {
371   DM_Plex       *mesh = (DM_Plex*) dm->data;
372   DMLabel        next  = mesh->labels;
373   PetscErrorCode ierr;
374 
375   PetscFunctionBegin;
376   if (--mesh->refct > 0) PetscFunctionReturn(0);
377   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
378   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
379   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
380   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
381   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
382   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
383   while (next) {
384     DMLabel tmp = next->next;
385 
386     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
387     next = tmp;
388   }
389   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
390   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
391   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
392   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
393   ierr = PetscFree(mesh);CHKERRQ(ierr);
394   PetscFunctionReturn(0);
395 }
396 
397 #undef __FUNCT__
398 #define __FUNCT__ "DMCreateMatrix_Plex"
399 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
400 {
401   PetscSection   section, sectionGlobal;
402   PetscInt       bs = -1;
403   PetscInt       localSize;
404   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
405   PetscErrorCode ierr;
406   MatType        mtype;
407 
408   PetscFunctionBegin;
409 #if !defined(PETSC_USE_DYNAMIC_LIBRARIES)
410   ierr = MatInitializePackage();CHKERRQ(ierr);
411 #endif
412   mtype = dm->mattype;
413   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
414   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
415   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
416   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
417   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
418   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
419   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
420   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
421   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
422   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
423   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
424   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
425   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
426   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
427   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
428   if (!isShell) {
429     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
430     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
431 
432     if (bs < 0) {
433       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
434         PetscInt pStart, pEnd, p, dof, cdof;
435 
436         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
437         for (p = pStart; p < pEnd; ++p) {
438           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
439           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
440           if (dof-cdof) {
441             if (bs < 0) {
442               bs = dof-cdof;
443             } else if (bs != dof-cdof) {
444               /* Layout does not admit a pointwise block size */
445               bs = 1;
446               break;
447             }
448           }
449         }
450         /* Must have same blocksize on all procs (some might have no points) */
451         bsLocal = bs;
452         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
453         bsLocal = bs < 0 ? bsMax : bs;
454         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
455         if (bsMin != bsMax) {
456           bs = 1;
457         } else {
458           bs = bsMax;
459         }
460       } else {
461         bs = 1;
462       }
463     }
464     ierr = PetscMalloc4(localSize/bs, PetscInt, &dnz, localSize/bs, PetscInt, &onz, localSize/bs, PetscInt, &dnzu, localSize/bs, PetscInt, &onzu);CHKERRQ(ierr);
465     ierr = PetscMemzero(dnz,  localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
466     ierr = PetscMemzero(onz,  localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
467     ierr = PetscMemzero(dnzu, localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
468     ierr = PetscMemzero(onzu, localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
469     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
470     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
471   }
472   PetscFunctionReturn(0);
473 }
474 
475 #undef __FUNCT__
476 #define __FUNCT__ "DMPlexGetDimension"
477 /*@
478   DMPlexGetDimension - Return the topological mesh dimension
479 
480   Not collective
481 
482   Input Parameter:
483 . mesh - The DMPlex
484 
485   Output Parameter:
486 . dim - The topological mesh dimension
487 
488   Level: beginner
489 
490 .seealso: DMPlexCreate()
491 @*/
492 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
493 {
494   DM_Plex *mesh = (DM_Plex*) dm->data;
495 
496   PetscFunctionBegin;
497   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
498   PetscValidPointer(dim, 2);
499   *dim = mesh->dim;
500   PetscFunctionReturn(0);
501 }
502 
503 #undef __FUNCT__
504 #define __FUNCT__ "DMPlexSetDimension"
505 /*@
506   DMPlexSetDimension - Set the topological mesh dimension
507 
508   Collective on mesh
509 
510   Input Parameters:
511 + mesh - The DMPlex
512 - dim - The topological mesh dimension
513 
514   Level: beginner
515 
516 .seealso: DMPlexCreate()
517 @*/
518 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
519 {
520   DM_Plex *mesh = (DM_Plex*) dm->data;
521 
522   PetscFunctionBegin;
523   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
524   PetscValidLogicalCollectiveInt(dm, dim, 2);
525   mesh->dim               = dim;
526   mesh->preallocCenterDim = dim;
527   PetscFunctionReturn(0);
528 }
529 
530 #undef __FUNCT__
531 #define __FUNCT__ "DMPlexGetChart"
532 /*@
533   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
534 
535   Not collective
536 
537   Input Parameter:
538 . mesh - The DMPlex
539 
540   Output Parameters:
541 + pStart - The first mesh point
542 - pEnd   - The upper bound for mesh points
543 
544   Level: beginner
545 
546 .seealso: DMPlexCreate(), DMPlexSetChart()
547 @*/
548 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
549 {
550   DM_Plex       *mesh = (DM_Plex*) dm->data;
551   PetscErrorCode ierr;
552 
553   PetscFunctionBegin;
554   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
555   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
556   PetscFunctionReturn(0);
557 }
558 
559 #undef __FUNCT__
560 #define __FUNCT__ "DMPlexSetChart"
561 /*@
562   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
563 
564   Not collective
565 
566   Input Parameters:
567 + mesh - The DMPlex
568 . pStart - The first mesh point
569 - pEnd   - The upper bound for mesh points
570 
571   Output Parameters:
572 
573   Level: beginner
574 
575 .seealso: DMPlexCreate(), DMPlexGetChart()
576 @*/
577 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
578 {
579   DM_Plex       *mesh = (DM_Plex*) dm->data;
580   PetscErrorCode ierr;
581 
582   PetscFunctionBegin;
583   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
584   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
585   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
586   PetscFunctionReturn(0);
587 }
588 
589 #undef __FUNCT__
590 #define __FUNCT__ "DMPlexGetConeSize"
591 /*@
592   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
593 
594   Not collective
595 
596   Input Parameters:
597 + mesh - The DMPlex
598 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
599 
600   Output Parameter:
601 . size - The cone size for point p
602 
603   Level: beginner
604 
605 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
606 @*/
607 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
608 {
609   DM_Plex       *mesh = (DM_Plex*) dm->data;
610   PetscErrorCode ierr;
611 
612   PetscFunctionBegin;
613   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
614   PetscValidPointer(size, 3);
615   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
616   PetscFunctionReturn(0);
617 }
618 
619 #undef __FUNCT__
620 #define __FUNCT__ "DMPlexSetConeSize"
621 /*@
622   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
623 
624   Not collective
625 
626   Input Parameters:
627 + mesh - The DMPlex
628 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
629 - size - The cone size for point p
630 
631   Output Parameter:
632 
633   Note:
634   This should be called after DMPlexSetChart().
635 
636   Level: beginner
637 
638 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
639 @*/
640 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
641 {
642   DM_Plex       *mesh = (DM_Plex*) dm->data;
643   PetscErrorCode ierr;
644 
645   PetscFunctionBegin;
646   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
647   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
648 
649   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
650   PetscFunctionReturn(0);
651 }
652 
653 #undef __FUNCT__
654 #define __FUNCT__ "DMPlexGetCone"
655 /*@C
656   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
657 
658   Not collective
659 
660   Input Parameters:
661 + mesh - The DMPlex
662 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
663 
664   Output Parameter:
665 . cone - An array of points which are on the in-edges for point p
666 
667   Level: beginner
668 
669   Fortran Notes:
670   Since it returns an array, this routine is only available in Fortran 90, and you must
671   include petsc.h90 in your code.
672 
673   You must also call DMPlexRestoreCone() after you finish using the returned array.
674 
675 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
676 @*/
677 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
678 {
679   DM_Plex       *mesh = (DM_Plex*) dm->data;
680   PetscInt       off;
681   PetscErrorCode ierr;
682 
683   PetscFunctionBegin;
684   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
685   PetscValidPointer(cone, 3);
686   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
687   *cone = &mesh->cones[off];
688   PetscFunctionReturn(0);
689 }
690 
691 #undef __FUNCT__
692 #define __FUNCT__ "DMPlexSetCone"
693 /*@
694   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
695 
696   Not collective
697 
698   Input Parameters:
699 + mesh - The DMPlex
700 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
701 - cone - An array of points which are on the in-edges for point p
702 
703   Output Parameter:
704 
705   Note:
706   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
707 
708   Level: beginner
709 
710 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
711 @*/
712 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
713 {
714   DM_Plex       *mesh = (DM_Plex*) dm->data;
715   PetscInt       pStart, pEnd;
716   PetscInt       dof, off, c;
717   PetscErrorCode ierr;
718 
719   PetscFunctionBegin;
720   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
721   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
722   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
723   if (dof) PetscValidPointer(cone, 3);
724   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
725   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
726   for (c = 0; c < dof; ++c) {
727     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
728     mesh->cones[off+c] = cone[c];
729   }
730   PetscFunctionReturn(0);
731 }
732 
733 #undef __FUNCT__
734 #define __FUNCT__ "DMPlexGetConeOrientation"
735 /*@C
736   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
737 
738   Not collective
739 
740   Input Parameters:
741 + mesh - The DMPlex
742 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
743 
744   Output Parameter:
745 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
746                     integer giving the prescription for cone traversal. If it is negative, the cone is
747                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
748                     the index of the cone point on which to start.
749 
750   Level: beginner
751 
752   Fortran Notes:
753   Since it returns an array, this routine is only available in Fortran 90, and you must
754   include petsc.h90 in your code.
755 
756   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
757 
758 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
759 @*/
760 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
761 {
762   DM_Plex       *mesh = (DM_Plex*) dm->data;
763   PetscInt       off;
764   PetscErrorCode ierr;
765 
766   PetscFunctionBegin;
767   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
768 #if defined(PETSC_USE_DEBUG)
769   {
770     PetscInt dof;
771     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
772     if (dof) PetscValidPointer(coneOrientation, 3);
773   }
774 #endif
775   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
776 
777   *coneOrientation = &mesh->coneOrientations[off];
778   PetscFunctionReturn(0);
779 }
780 
781 #undef __FUNCT__
782 #define __FUNCT__ "DMPlexSetConeOrientation"
783 /*@
784   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
785 
786   Not collective
787 
788   Input Parameters:
789 + mesh - The DMPlex
790 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
791 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
792                     integer giving the prescription for cone traversal. If it is negative, the cone is
793                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
794                     the index of the cone point on which to start.
795 
796   Output Parameter:
797 
798   Note:
799   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
800 
801   Level: beginner
802 
803 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
804 @*/
805 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
806 {
807   DM_Plex       *mesh = (DM_Plex*) dm->data;
808   PetscInt       pStart, pEnd;
809   PetscInt       dof, off, c;
810   PetscErrorCode ierr;
811 
812   PetscFunctionBegin;
813   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
814   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
815   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
816   if (dof) PetscValidPointer(coneOrientation, 3);
817   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
818   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
819   for (c = 0; c < dof; ++c) {
820     PetscInt cdof, o = coneOrientation[c];
821 
822     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
823     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
824     mesh->coneOrientations[off+c] = o;
825   }
826   PetscFunctionReturn(0);
827 }
828 
829 #undef __FUNCT__
830 #define __FUNCT__ "DMPlexInsertCone"
831 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
832 {
833   DM_Plex       *mesh = (DM_Plex*) dm->data;
834   PetscInt       pStart, pEnd;
835   PetscInt       dof, off;
836   PetscErrorCode ierr;
837 
838   PetscFunctionBegin;
839   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
840   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
841   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
842   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
843   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
844   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
845   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
846   mesh->cones[off+conePos] = conePoint;
847   PetscFunctionReturn(0);
848 }
849 
850 #undef __FUNCT__
851 #define __FUNCT__ "DMPlexInsertConeOrientation"
852 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
853 {
854   DM_Plex       *mesh = (DM_Plex*) dm->data;
855   PetscInt       pStart, pEnd;
856   PetscInt       dof, off;
857   PetscErrorCode ierr;
858 
859   PetscFunctionBegin;
860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
861   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
862   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
863   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
864   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
865   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
866   mesh->coneOrientations[off+conePos] = coneOrientation;
867   PetscFunctionReturn(0);
868 }
869 
870 #undef __FUNCT__
871 #define __FUNCT__ "DMPlexGetSupportSize"
872 /*@
873   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
874 
875   Not collective
876 
877   Input Parameters:
878 + mesh - The DMPlex
879 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
880 
881   Output Parameter:
882 . size - The support size for point p
883 
884   Level: beginner
885 
886 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
887 @*/
888 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
889 {
890   DM_Plex       *mesh = (DM_Plex*) dm->data;
891   PetscErrorCode ierr;
892 
893   PetscFunctionBegin;
894   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
895   PetscValidPointer(size, 3);
896   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
897   PetscFunctionReturn(0);
898 }
899 
900 #undef __FUNCT__
901 #define __FUNCT__ "DMPlexSetSupportSize"
902 /*@
903   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
904 
905   Not collective
906 
907   Input Parameters:
908 + mesh - The DMPlex
909 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
910 - size - The support size for point p
911 
912   Output Parameter:
913 
914   Note:
915   This should be called after DMPlexSetChart().
916 
917   Level: beginner
918 
919 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
920 @*/
921 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
922 {
923   DM_Plex       *mesh = (DM_Plex*) dm->data;
924   PetscErrorCode ierr;
925 
926   PetscFunctionBegin;
927   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
928   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
929 
930   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
931   PetscFunctionReturn(0);
932 }
933 
934 #undef __FUNCT__
935 #define __FUNCT__ "DMPlexGetSupport"
936 /*@C
937   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
938 
939   Not collective
940 
941   Input Parameters:
942 + mesh - The DMPlex
943 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
944 
945   Output Parameter:
946 . support - An array of points which are on the out-edges for point p
947 
948   Level: beginner
949 
950   Fortran Notes:
951   Since it returns an array, this routine is only available in Fortran 90, and you must
952   include petsc.h90 in your code.
953 
954   You must also call DMPlexRestoreSupport() after you finish using the returned array.
955 
956 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
957 @*/
958 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
959 {
960   DM_Plex       *mesh = (DM_Plex*) dm->data;
961   PetscInt       off;
962   PetscErrorCode ierr;
963 
964   PetscFunctionBegin;
965   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
966   PetscValidPointer(support, 3);
967   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
968   *support = &mesh->supports[off];
969   PetscFunctionReturn(0);
970 }
971 
972 #undef __FUNCT__
973 #define __FUNCT__ "DMPlexSetSupport"
974 /*@
975   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
976 
977   Not collective
978 
979   Input Parameters:
980 + mesh - The DMPlex
981 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
982 - support - An array of points which are on the in-edges for point p
983 
984   Output Parameter:
985 
986   Note:
987   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
988 
989   Level: beginner
990 
991 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
992 @*/
993 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
994 {
995   DM_Plex       *mesh = (DM_Plex*) dm->data;
996   PetscInt       pStart, pEnd;
997   PetscInt       dof, off, c;
998   PetscErrorCode ierr;
999 
1000   PetscFunctionBegin;
1001   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1002   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1003   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1004   if (dof) PetscValidPointer(support, 3);
1005   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1006   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1007   for (c = 0; c < dof; ++c) {
1008     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1009     mesh->supports[off+c] = support[c];
1010   }
1011   PetscFunctionReturn(0);
1012 }
1013 
1014 #undef __FUNCT__
1015 #define __FUNCT__ "DMPlexInsertSupport"
1016 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1017 {
1018   DM_Plex       *mesh = (DM_Plex*) dm->data;
1019   PetscInt       pStart, pEnd;
1020   PetscInt       dof, off;
1021   PetscErrorCode ierr;
1022 
1023   PetscFunctionBegin;
1024   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1025   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1026   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1027   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1028   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1029   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1030   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1031   mesh->supports[off+supportPos] = supportPoint;
1032   PetscFunctionReturn(0);
1033 }
1034 
1035 #undef __FUNCT__
1036 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1037 /*@C
1038   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1039 
1040   Not collective
1041 
1042   Input Parameters:
1043 + mesh - The DMPlex
1044 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1045 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1046 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1047 
1048   Output Parameters:
1049 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1050 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1051 
1052   Note:
1053   If using internal storage (points is NULL on input), each call overwrites the last output.
1054 
1055   Fortran Notes:
1056   Since it returns an array, this routine is only available in Fortran 90, and you must
1057   include petsc.h90 in your code.
1058 
1059   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1060 
1061   Level: beginner
1062 
1063 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1064 @*/
1065 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1066 {
1067   DM_Plex        *mesh = (DM_Plex*) dm->data;
1068   PetscInt       *closure, *fifo;
1069   const PetscInt *tmp = NULL, *tmpO = NULL;
1070   PetscInt        tmpSize, t;
1071   PetscInt        depth       = 0, maxSize;
1072   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1073   PetscErrorCode  ierr;
1074 
1075   PetscFunctionBegin;
1076   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1077   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1078   /* This is only 1-level */
1079   if (useCone) {
1080     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1081     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1082     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1083   } else {
1084     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1085     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1086   }
1087   if (depth == 1) {
1088     if (*points) {
1089       closure = *points;
1090     } else {
1091       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1092       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1093     }
1094     closure[0] = p; closure[1] = 0;
1095     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1096       closure[closureSize]   = tmp[t];
1097       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1098     }
1099     if (numPoints) *numPoints = closureSize/2;
1100     if (points)    *points    = closure;
1101     PetscFunctionReturn(0);
1102   }
1103   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1104   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1105   if (*points) {
1106     closure = *points;
1107   } else {
1108     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1109   }
1110   closure[0] = p; closure[1] = 0;
1111   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1112     const PetscInt cp = tmp[t];
1113     const PetscInt co = tmpO ? tmpO[t] : 0;
1114 
1115     closure[closureSize]   = cp;
1116     closure[closureSize+1] = co;
1117     fifo[fifoSize]         = cp;
1118     fifo[fifoSize+1]       = co;
1119   }
1120   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1121   while (fifoSize - fifoStart) {
1122     const PetscInt q   = fifo[fifoStart];
1123     const PetscInt o   = fifo[fifoStart+1];
1124     const PetscInt rev = o >= 0 ? 0 : 1;
1125     const PetscInt off = rev ? -(o+1) : o;
1126 
1127     if (useCone) {
1128       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1129       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1130       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1131     } else {
1132       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1133       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1134       tmpO = NULL;
1135     }
1136     for (t = 0; t < tmpSize; ++t) {
1137       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1138       const PetscInt cp = tmp[i];
1139       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1140       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1141        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1142       PetscInt       co = tmpO ? tmpO[i] : 0;
1143       PetscInt       c;
1144 
1145       if (rev) {
1146         PetscInt childSize, coff;
1147         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1148         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1149         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1150       }
1151       /* Check for duplicate */
1152       for (c = 0; c < closureSize; c += 2) {
1153         if (closure[c] == cp) break;
1154       }
1155       if (c == closureSize) {
1156         closure[closureSize]   = cp;
1157         closure[closureSize+1] = co;
1158         fifo[fifoSize]         = cp;
1159         fifo[fifoSize+1]       = co;
1160         closureSize           += 2;
1161         fifoSize              += 2;
1162       }
1163     }
1164     fifoStart += 2;
1165   }
1166   if (numPoints) *numPoints = closureSize/2;
1167   if (points)    *points    = closure;
1168   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1169   PetscFunctionReturn(0);
1170 }
1171 
1172 #undef __FUNCT__
1173 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1174 /*@C
1175   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1176 
1177   Not collective
1178 
1179   Input Parameters:
1180 + mesh - The DMPlex
1181 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1182 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1183 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1184 
1185   Output Parameters:
1186 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1187 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1188 
1189   Note:
1190   If not using internal storage (points is not NULL on input), this call is unnecessary
1191 
1192   Fortran Notes:
1193   Since it returns an array, this routine is only available in Fortran 90, and you must
1194   include petsc.h90 in your code.
1195 
1196   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1197 
1198   Level: beginner
1199 
1200 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1201 @*/
1202 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1203 {
1204   PetscErrorCode ierr;
1205 
1206   PetscFunctionBegin;
1207   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1208   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1209   PetscFunctionReturn(0);
1210 }
1211 
1212 #undef __FUNCT__
1213 #define __FUNCT__ "DMPlexGetMaxSizes"
1214 /*@
1215   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1216 
1217   Not collective
1218 
1219   Input Parameter:
1220 . mesh - The DMPlex
1221 
1222   Output Parameters:
1223 + maxConeSize - The maximum number of in-edges
1224 - maxSupportSize - The maximum number of out-edges
1225 
1226   Level: beginner
1227 
1228 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1229 @*/
1230 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
1231 {
1232   DM_Plex *mesh = (DM_Plex*) dm->data;
1233 
1234   PetscFunctionBegin;
1235   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1236   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
1237   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
1238   PetscFunctionReturn(0);
1239 }
1240 
1241 #undef __FUNCT__
1242 #define __FUNCT__ "DMSetUp_Plex"
1243 PetscErrorCode DMSetUp_Plex(DM dm)
1244 {
1245   DM_Plex       *mesh = (DM_Plex*) dm->data;
1246   PetscInt       size;
1247   PetscErrorCode ierr;
1248 
1249   PetscFunctionBegin;
1250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1251   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
1252   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
1253   ierr = PetscMalloc(size * sizeof(PetscInt), &mesh->cones);CHKERRQ(ierr);
1254   ierr = PetscMalloc(size * sizeof(PetscInt), &mesh->coneOrientations);CHKERRQ(ierr);
1255   ierr = PetscMemzero(mesh->coneOrientations, size * sizeof(PetscInt));CHKERRQ(ierr);
1256   if (mesh->maxSupportSize) {
1257     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1258     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
1259     ierr = PetscMalloc(size * sizeof(PetscInt), &mesh->supports);CHKERRQ(ierr);
1260   }
1261   PetscFunctionReturn(0);
1262 }
1263 
1264 #undef __FUNCT__
1265 #define __FUNCT__ "DMCreateSubDM_Plex"
1266 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
1267 {
1268   PetscErrorCode ierr;
1269 
1270   PetscFunctionBegin;
1271   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
1272   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
1273   PetscFunctionReturn(0);
1274 }
1275 
1276 #undef __FUNCT__
1277 #define __FUNCT__ "DMPlexSymmetrize"
1278 /*@
1279   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
1280 
1281   Not collective
1282 
1283   Input Parameter:
1284 . mesh - The DMPlex
1285 
1286   Output Parameter:
1287 
1288   Note:
1289   This should be called after all calls to DMPlexSetCone()
1290 
1291   Level: beginner
1292 
1293 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
1294 @*/
1295 PetscErrorCode DMPlexSymmetrize(DM dm)
1296 {
1297   DM_Plex       *mesh = (DM_Plex*) dm->data;
1298   PetscInt      *offsets;
1299   PetscInt       supportSize;
1300   PetscInt       pStart, pEnd, p;
1301   PetscErrorCode ierr;
1302 
1303   PetscFunctionBegin;
1304   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1305   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
1306   /* Calculate support sizes */
1307   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1308   for (p = pStart; p < pEnd; ++p) {
1309     PetscInt dof, off, c;
1310 
1311     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1312     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1313     for (c = off; c < off+dof; ++c) {
1314       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
1315     }
1316   }
1317   for (p = pStart; p < pEnd; ++p) {
1318     PetscInt dof;
1319 
1320     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1321 
1322     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
1323   }
1324   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1325   /* Calculate supports */
1326   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
1327   ierr = PetscMalloc(supportSize * sizeof(PetscInt), &mesh->supports);CHKERRQ(ierr);
1328   ierr = PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &offsets);CHKERRQ(ierr);
1329   ierr = PetscMemzero(offsets, (pEnd - pStart) * sizeof(PetscInt));CHKERRQ(ierr);
1330   for (p = pStart; p < pEnd; ++p) {
1331     PetscInt dof, off, c;
1332 
1333     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1334     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1335     for (c = off; c < off+dof; ++c) {
1336       const PetscInt q = mesh->cones[c];
1337       PetscInt       offS;
1338 
1339       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
1340 
1341       mesh->supports[offS+offsets[q]] = p;
1342       ++offsets[q];
1343     }
1344   }
1345   ierr = PetscFree(offsets);CHKERRQ(ierr);
1346   PetscFunctionReturn(0);
1347 }
1348 
1349 #undef __FUNCT__
1350 #define __FUNCT__ "DMPlexStratify"
1351 /*@
1352   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
1353   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
1354   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
1355   the DAG.
1356 
1357   Not collective
1358 
1359   Input Parameter:
1360 . mesh - The DMPlex
1361 
1362   Output Parameter:
1363 
1364   Notes:
1365   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
1366   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
1367 
1368   This should be called after all calls to DMPlexSymmetrize()
1369 
1370   Level: beginner
1371 
1372 .seealso: DMPlexCreate(), DMPlexSymmetrize()
1373 @*/
1374 PetscErrorCode DMPlexStratify(DM dm)
1375 {
1376   DMLabel        label;
1377   PetscInt       pStart, pEnd, p;
1378   PetscInt       numRoots = 0, numLeaves = 0;
1379   PetscErrorCode ierr;
1380 
1381   PetscFunctionBegin;
1382   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1383   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1384   /* Calculate depth */
1385   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1386   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
1387   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
1388   /* Initialize roots and count leaves */
1389   for (p = pStart; p < pEnd; ++p) {
1390     PetscInt coneSize, supportSize;
1391 
1392     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1393     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1394     if (!coneSize && supportSize) {
1395       ++numRoots;
1396       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1397     } else if (!supportSize && coneSize) {
1398       ++numLeaves;
1399     } else if (!supportSize && !coneSize) {
1400       /* Isolated points */
1401       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1402     }
1403   }
1404   if (numRoots + numLeaves == (pEnd - pStart)) {
1405     for (p = pStart; p < pEnd; ++p) {
1406       PetscInt coneSize, supportSize;
1407 
1408       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1409       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1410       if (!supportSize && coneSize) {
1411         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
1412       }
1413     }
1414   } else {
1415     IS       pointIS;
1416     PetscInt numPoints = 0, level = 0;
1417 
1418     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1419     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1420     while (numPoints) {
1421       const PetscInt *points;
1422       const PetscInt  newLevel = level+1;
1423 
1424       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
1425       for (p = 0; p < numPoints; ++p) {
1426         const PetscInt  point = points[p];
1427         const PetscInt *support;
1428         PetscInt        supportSize, s;
1429 
1430         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1431         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1432         for (s = 0; s < supportSize; ++s) {
1433           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
1434         }
1435       }
1436       ++level;
1437       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1438       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1439       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1440       else         {numPoints = 0;}
1441     }
1442     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1443   }
1444   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1445   PetscFunctionReturn(0);
1446 }
1447 
1448 #undef __FUNCT__
1449 #define __FUNCT__ "DMPlexGetJoin"
1450 /*@C
1451   DMPlexGetJoin - Get an array for the join of the set of points
1452 
1453   Not Collective
1454 
1455   Input Parameters:
1456 + dm - The DMPlex object
1457 . numPoints - The number of input points for the join
1458 - points - The input points
1459 
1460   Output Parameters:
1461 + numCoveredPoints - The number of points in the join
1462 - coveredPoints - The points in the join
1463 
1464   Level: intermediate
1465 
1466   Note: Currently, this is restricted to a single level join
1467 
1468   Fortran Notes:
1469   Since it returns an array, this routine is only available in Fortran 90, and you must
1470   include petsc.h90 in your code.
1471 
1472   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1473 
1474 .keywords: mesh
1475 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
1476 @*/
1477 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1478 {
1479   DM_Plex       *mesh = (DM_Plex*) dm->data;
1480   PetscInt      *join[2];
1481   PetscInt       joinSize, i = 0;
1482   PetscInt       dof, off, p, c, m;
1483   PetscErrorCode ierr;
1484 
1485   PetscFunctionBegin;
1486   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1487   PetscValidPointer(points, 2);
1488   PetscValidPointer(numCoveredPoints, 3);
1489   PetscValidPointer(coveredPoints, 4);
1490   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1491   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1492   /* Copy in support of first point */
1493   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
1494   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
1495   for (joinSize = 0; joinSize < dof; ++joinSize) {
1496     join[i][joinSize] = mesh->supports[off+joinSize];
1497   }
1498   /* Check each successive support */
1499   for (p = 1; p < numPoints; ++p) {
1500     PetscInt newJoinSize = 0;
1501 
1502     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
1503     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
1504     for (c = 0; c < dof; ++c) {
1505       const PetscInt point = mesh->supports[off+c];
1506 
1507       for (m = 0; m < joinSize; ++m) {
1508         if (point == join[i][m]) {
1509           join[1-i][newJoinSize++] = point;
1510           break;
1511         }
1512       }
1513     }
1514     joinSize = newJoinSize;
1515     i        = 1-i;
1516   }
1517   *numCoveredPoints = joinSize;
1518   *coveredPoints    = join[i];
1519   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1520   PetscFunctionReturn(0);
1521 }
1522 
1523 #undef __FUNCT__
1524 #define __FUNCT__ "DMPlexRestoreJoin"
1525 /*@C
1526   DMPlexRestoreJoin - Restore an array for the join of the set of points
1527 
1528   Not Collective
1529 
1530   Input Parameters:
1531 + dm - The DMPlex object
1532 . numPoints - The number of input points for the join
1533 - points - The input points
1534 
1535   Output Parameters:
1536 + numCoveredPoints - The number of points in the join
1537 - coveredPoints - The points in the join
1538 
1539   Fortran Notes:
1540   Since it returns an array, this routine is only available in Fortran 90, and you must
1541   include petsc.h90 in your code.
1542 
1543   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1544 
1545   Level: intermediate
1546 
1547 .keywords: mesh
1548 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
1549 @*/
1550 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1551 {
1552   PetscErrorCode ierr;
1553 
1554   PetscFunctionBegin;
1555   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1556   PetscValidPointer(coveredPoints, 4);
1557   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
1558   PetscFunctionReturn(0);
1559 }
1560 
1561 #undef __FUNCT__
1562 #define __FUNCT__ "DMPlexGetFullJoin"
1563 /*@C
1564   DMPlexGetFullJoin - Get an array for the join of the set of points
1565 
1566   Not Collective
1567 
1568   Input Parameters:
1569 + dm - The DMPlex object
1570 . numPoints - The number of input points for the join
1571 - points - The input points
1572 
1573   Output Parameters:
1574 + numCoveredPoints - The number of points in the join
1575 - coveredPoints - The points in the join
1576 
1577   Fortran Notes:
1578   Since it returns an array, this routine is only available in Fortran 90, and you must
1579   include petsc.h90 in your code.
1580 
1581   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1582 
1583   Level: intermediate
1584 
1585 .keywords: mesh
1586 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
1587 @*/
1588 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1589 {
1590   DM_Plex       *mesh = (DM_Plex*) dm->data;
1591   PetscInt      *offsets, **closures;
1592   PetscInt      *join[2];
1593   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
1594   PetscInt       p, d, c, m;
1595   PetscErrorCode ierr;
1596 
1597   PetscFunctionBegin;
1598   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1599   PetscValidPointer(points, 2);
1600   PetscValidPointer(numCoveredPoints, 3);
1601   PetscValidPointer(coveredPoints, 4);
1602 
1603   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1604   ierr    = PetscMalloc(numPoints * sizeof(PetscInt*), &closures);CHKERRQ(ierr);
1605   ierr    = PetscMemzero(closures,numPoints*sizeof(PetscInt*));CHKERRQ(ierr);
1606   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1607   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
1608   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1609   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1610 
1611   for (p = 0; p < numPoints; ++p) {
1612     PetscInt closureSize;
1613 
1614     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
1615 
1616     offsets[p*(depth+2)+0] = 0;
1617     for (d = 0; d < depth+1; ++d) {
1618       PetscInt pStart, pEnd, i;
1619 
1620       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1621       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
1622         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
1623           offsets[p*(depth+2)+d+1] = i;
1624           break;
1625         }
1626       }
1627       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
1628     }
1629     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
1630   }
1631   for (d = 0; d < depth+1; ++d) {
1632     PetscInt dof;
1633 
1634     /* Copy in support of first point */
1635     dof = offsets[d+1] - offsets[d];
1636     for (joinSize = 0; joinSize < dof; ++joinSize) {
1637       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
1638     }
1639     /* Check each successive cone */
1640     for (p = 1; p < numPoints && joinSize; ++p) {
1641       PetscInt newJoinSize = 0;
1642 
1643       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
1644       for (c = 0; c < dof; ++c) {
1645         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
1646 
1647         for (m = 0; m < joinSize; ++m) {
1648           if (point == join[i][m]) {
1649             join[1-i][newJoinSize++] = point;
1650             break;
1651           }
1652         }
1653       }
1654       joinSize = newJoinSize;
1655       i        = 1-i;
1656     }
1657     if (joinSize) break;
1658   }
1659   *numCoveredPoints = joinSize;
1660   *coveredPoints    = join[i];
1661   for (p = 0; p < numPoints; ++p) {
1662     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
1663   }
1664   ierr = PetscFree(closures);CHKERRQ(ierr);
1665   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1666   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1667   PetscFunctionReturn(0);
1668 }
1669 
1670 #undef __FUNCT__
1671 #define __FUNCT__ "DMPlexGetMeet"
1672 /*@C
1673   DMPlexGetMeet - Get an array for the meet of the set of points
1674 
1675   Not Collective
1676 
1677   Input Parameters:
1678 + dm - The DMPlex object
1679 . numPoints - The number of input points for the meet
1680 - points - The input points
1681 
1682   Output Parameters:
1683 + numCoveredPoints - The number of points in the meet
1684 - coveredPoints - The points in the meet
1685 
1686   Level: intermediate
1687 
1688   Note: Currently, this is restricted to a single level meet
1689 
1690   Fortran Notes:
1691   Since it returns an array, this routine is only available in Fortran 90, and you must
1692   include petsc.h90 in your code.
1693 
1694   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1695 
1696 .keywords: mesh
1697 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
1698 @*/
1699 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
1700 {
1701   DM_Plex       *mesh = (DM_Plex*) dm->data;
1702   PetscInt      *meet[2];
1703   PetscInt       meetSize, i = 0;
1704   PetscInt       dof, off, p, c, m;
1705   PetscErrorCode ierr;
1706 
1707   PetscFunctionBegin;
1708   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1709   PetscValidPointer(points, 2);
1710   PetscValidPointer(numCoveringPoints, 3);
1711   PetscValidPointer(coveringPoints, 4);
1712   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
1713   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
1714   /* Copy in cone of first point */
1715   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
1716   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
1717   for (meetSize = 0; meetSize < dof; ++meetSize) {
1718     meet[i][meetSize] = mesh->cones[off+meetSize];
1719   }
1720   /* Check each successive cone */
1721   for (p = 1; p < numPoints; ++p) {
1722     PetscInt newMeetSize = 0;
1723 
1724     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
1725     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
1726     for (c = 0; c < dof; ++c) {
1727       const PetscInt point = mesh->cones[off+c];
1728 
1729       for (m = 0; m < meetSize; ++m) {
1730         if (point == meet[i][m]) {
1731           meet[1-i][newMeetSize++] = point;
1732           break;
1733         }
1734       }
1735     }
1736     meetSize = newMeetSize;
1737     i        = 1-i;
1738   }
1739   *numCoveringPoints = meetSize;
1740   *coveringPoints    = meet[i];
1741   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
1742   PetscFunctionReturn(0);
1743 }
1744 
1745 #undef __FUNCT__
1746 #define __FUNCT__ "DMPlexRestoreMeet"
1747 /*@C
1748   DMPlexRestoreMeet - Restore an array for the meet of the set of points
1749 
1750   Not Collective
1751 
1752   Input Parameters:
1753 + dm - The DMPlex object
1754 . numPoints - The number of input points for the meet
1755 - points - The input points
1756 
1757   Output Parameters:
1758 + numCoveredPoints - The number of points in the meet
1759 - coveredPoints - The points in the meet
1760 
1761   Level: intermediate
1762 
1763   Fortran Notes:
1764   Since it returns an array, this routine is only available in Fortran 90, and you must
1765   include petsc.h90 in your code.
1766 
1767   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1768 
1769 .keywords: mesh
1770 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
1771 @*/
1772 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1773 {
1774   PetscErrorCode ierr;
1775 
1776   PetscFunctionBegin;
1777   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1778   PetscValidPointer(coveredPoints, 4);
1779   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
1780   PetscFunctionReturn(0);
1781 }
1782 
1783 #undef __FUNCT__
1784 #define __FUNCT__ "DMPlexGetFullMeet"
1785 /*@C
1786   DMPlexGetFullMeet - Get an array for the meet of the set of points
1787 
1788   Not Collective
1789 
1790   Input Parameters:
1791 + dm - The DMPlex object
1792 . numPoints - The number of input points for the meet
1793 - points - The input points
1794 
1795   Output Parameters:
1796 + numCoveredPoints - The number of points in the meet
1797 - coveredPoints - The points in the meet
1798 
1799   Level: intermediate
1800 
1801   Fortran Notes:
1802   Since it returns an array, this routine is only available in Fortran 90, and you must
1803   include petsc.h90 in your code.
1804 
1805   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1806 
1807 .keywords: mesh
1808 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
1809 @*/
1810 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1811 {
1812   DM_Plex       *mesh = (DM_Plex*) dm->data;
1813   PetscInt      *offsets, **closures;
1814   PetscInt      *meet[2];
1815   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
1816   PetscInt       p, h, c, m;
1817   PetscErrorCode ierr;
1818 
1819   PetscFunctionBegin;
1820   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1821   PetscValidPointer(points, 2);
1822   PetscValidPointer(numCoveredPoints, 3);
1823   PetscValidPointer(coveredPoints, 4);
1824 
1825   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
1826   ierr    = PetscMalloc(numPoints * sizeof(PetscInt*), &closures);CHKERRQ(ierr);
1827   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1828   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
1829   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
1830   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
1831 
1832   for (p = 0; p < numPoints; ++p) {
1833     PetscInt closureSize;
1834 
1835     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
1836 
1837     offsets[p*(height+2)+0] = 0;
1838     for (h = 0; h < height+1; ++h) {
1839       PetscInt pStart, pEnd, i;
1840 
1841       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
1842       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
1843         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
1844           offsets[p*(height+2)+h+1] = i;
1845           break;
1846         }
1847       }
1848       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
1849     }
1850     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
1851   }
1852   for (h = 0; h < height+1; ++h) {
1853     PetscInt dof;
1854 
1855     /* Copy in cone of first point */
1856     dof = offsets[h+1] - offsets[h];
1857     for (meetSize = 0; meetSize < dof; ++meetSize) {
1858       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
1859     }
1860     /* Check each successive cone */
1861     for (p = 1; p < numPoints && meetSize; ++p) {
1862       PetscInt newMeetSize = 0;
1863 
1864       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
1865       for (c = 0; c < dof; ++c) {
1866         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
1867 
1868         for (m = 0; m < meetSize; ++m) {
1869           if (point == meet[i][m]) {
1870             meet[1-i][newMeetSize++] = point;
1871             break;
1872           }
1873         }
1874       }
1875       meetSize = newMeetSize;
1876       i        = 1-i;
1877     }
1878     if (meetSize) break;
1879   }
1880   *numCoveredPoints = meetSize;
1881   *coveredPoints    = meet[i];
1882   for (p = 0; p < numPoints; ++p) {
1883     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
1884   }
1885   ierr = PetscFree(closures);CHKERRQ(ierr);
1886   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1887   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
1888   PetscFunctionReturn(0);
1889 }
1890 
1891 #undef __FUNCT__
1892 #define __FUNCT__ "DMPlexGetNumFaceVertices"
1893 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
1894 {
1895   MPI_Comm       comm;
1896   PetscErrorCode ierr;
1897 
1898   PetscFunctionBegin;
1899   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1900   PetscValidPointer(numFaceVertices,3);
1901   switch (cellDim) {
1902   case 0:
1903     *numFaceVertices = 0;
1904     break;
1905   case 1:
1906     *numFaceVertices = 1;
1907     break;
1908   case 2:
1909     switch (numCorners) {
1910     case 3: /* triangle */
1911       *numFaceVertices = 2; /* Edge has 2 vertices */
1912       break;
1913     case 4: /* quadrilateral */
1914       *numFaceVertices = 2; /* Edge has 2 vertices */
1915       break;
1916     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
1917       *numFaceVertices = 3; /* Edge has 3 vertices */
1918       break;
1919     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
1920       *numFaceVertices = 3; /* Edge has 3 vertices */
1921       break;
1922     default:
1923       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
1924     }
1925     break;
1926   case 3:
1927     switch (numCorners) {
1928     case 4: /* tetradehdron */
1929       *numFaceVertices = 3; /* Face has 3 vertices */
1930       break;
1931     case 6: /* tet cohesive cells */
1932       *numFaceVertices = 4; /* Face has 4 vertices */
1933       break;
1934     case 8: /* hexahedron */
1935       *numFaceVertices = 4; /* Face has 4 vertices */
1936       break;
1937     case 9: /* tet cohesive Lagrange cells */
1938       *numFaceVertices = 6; /* Face has 6 vertices */
1939       break;
1940     case 10: /* quadratic tetrahedron */
1941       *numFaceVertices = 6; /* Face has 6 vertices */
1942       break;
1943     case 12: /* hex cohesive Lagrange cells */
1944       *numFaceVertices = 6; /* Face has 6 vertices */
1945       break;
1946     case 18: /* quadratic tet cohesive Lagrange cells */
1947       *numFaceVertices = 6; /* Face has 6 vertices */
1948       break;
1949     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
1950       *numFaceVertices = 9; /* Face has 9 vertices */
1951       break;
1952     default:
1953       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
1954     }
1955     break;
1956   default:
1957     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
1958   }
1959   PetscFunctionReturn(0);
1960 }
1961 
1962 #undef __FUNCT__
1963 #define __FUNCT__ "DMPlexOrient"
1964 /* Trys to give the mesh a consistent orientation */
1965 PetscErrorCode DMPlexOrient(DM dm)
1966 {
1967   PetscBT        seenCells, flippedCells, seenFaces;
1968   PetscInt      *faceFIFO, fTop, fBottom;
1969   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
1970   PetscErrorCode ierr;
1971 
1972   PetscFunctionBegin;
1973   /* Truth Table
1974      mismatch    flips   do action   mismatch   flipA ^ flipB   action
1975          F       0 flips     no         F             F           F
1976          F       1 flip      yes        F             T           T
1977          F       2 flips     no         T             F           T
1978          T       0 flips     yes        T             T           F
1979          T       1 flip      no
1980          T       2 flips     yes
1981   */
1982   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1983   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
1984   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
1985   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
1986   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
1987   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
1988   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
1989   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
1990   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
1991   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
1992   ierr = PetscMalloc((fEnd - fStart) * sizeof(PetscInt), &faceFIFO);CHKERRQ(ierr);
1993   fTop = fBottom = 0;
1994   /* Initialize FIFO with first cell */
1995   if (cEnd > cStart) {
1996     const PetscInt *cone;
1997     PetscInt        coneSize;
1998 
1999     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
2000     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2001     for (c = 0; c < coneSize; ++c) {
2002       faceFIFO[fBottom++] = cone[c];
2003       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2004     }
2005   }
2006   /* Consider each face in FIFO */
2007   while (fTop < fBottom) {
2008     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2009     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2010     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2011 
2012     face = faceFIFO[fTop++];
2013     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2014     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2015     if (supportSize < 2) continue;
2016     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2017     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2018     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2019     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2020     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2021 
2022     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2023     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2024     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2025     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2026     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2027     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2028     for (c = 0; c < coneSizeA; ++c) {
2029       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2030         faceFIFO[fBottom++] = coneA[c];
2031         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2032       }
2033       if (coneA[c] == face) posA = c;
2034       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2035     }
2036     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2037     for (c = 0; c < coneSizeB; ++c) {
2038       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2039         faceFIFO[fBottom++] = coneB[c];
2040         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2041       }
2042       if (coneB[c] == face) posB = c;
2043       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2044     }
2045     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2046 
2047     if (dim == 1) {
2048       mismatch = posA == posB;
2049     } else {
2050       mismatch = coneOA[posA] == coneOB[posB];
2051     }
2052 
2053     if (mismatch ^ (flippedA ^ flippedB)) {
2054       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2055       if (!seenA && !flippedA) {
2056         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2057       } else if (!seenB && !flippedB) {
2058         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2059       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2060     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2061     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2062     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2063   }
2064 
2065   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2066   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2067   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2068   for (c = cStart; c < cEnd; ++c) {
2069     const PetscInt *cone, *coneO, *support;
2070     PetscInt        coneSize, supportSize, faceSize, cp, sp;
2071 
2072     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
2073     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2074     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2075     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
2076     for (cp = 0; cp < coneSize; ++cp) {
2077       const PetscInt rcp = coneSize-cp-1;
2078 
2079       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
2080       revcone[cp]  = cone[rcp];
2081       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
2082     }
2083     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
2084     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
2085     /* Reverse orientations of support */
2086     faceSize = coneSize;
2087     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
2088     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
2089     for (sp = 0; sp < supportSize; ++sp) {
2090       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
2091       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
2092       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
2093       for (cp = 0; cp < coneSize; ++cp) {
2094         if (cone[cp] != c) continue;
2095         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
2096       }
2097     }
2098   }
2099   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2100   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2101   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
2102   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
2103   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
2104   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
2105   PetscFunctionReturn(0);
2106 }
2107 
2108 #undef __FUNCT__
2109 #define __FUNCT__ "DMPlexGetAdjacencySingleLevel_Internal"
2110 static PetscErrorCode DMPlexGetAdjacencySingleLevel_Internal(DM dm, PetscInt p, PetscBool useClosure, const PetscInt *tmpClosure, PetscInt *adjSize, PetscInt adj[])
2111 {
2112   const PetscInt *support = NULL;
2113   PetscInt        numAdj   = 0, maxAdjSize = *adjSize, supportSize, s;
2114   PetscErrorCode  ierr;
2115 
2116   PetscFunctionBegin;
2117   if (useClosure) {
2118     ierr = DMPlexGetConeSize(dm, p, &supportSize);CHKERRQ(ierr);
2119     ierr = DMPlexGetCone(dm, p, &support);CHKERRQ(ierr);
2120     for (s = 0; s < supportSize; ++s) {
2121       const PetscInt *cone = NULL;
2122       PetscInt        coneSize, c, q;
2123 
2124       ierr = DMPlexGetSupportSize(dm, support[s], &coneSize);CHKERRQ(ierr);
2125       ierr = DMPlexGetSupport(dm, support[s], &cone);CHKERRQ(ierr);
2126       for (c = 0; c < coneSize; ++c) {
2127         for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
2128           if (cone[c] == adj[q]) break;
2129         }
2130         if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
2131       }
2132     }
2133   } else {
2134     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2135     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
2136     for (s = 0; s < supportSize; ++s) {
2137       const PetscInt *cone = NULL;
2138       PetscInt        coneSize, c, q;
2139 
2140       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
2141       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
2142       for (c = 0; c < coneSize; ++c) {
2143         for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
2144           if (cone[c] == adj[q]) break;
2145         }
2146         if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
2147       }
2148     }
2149   }
2150   *adjSize = numAdj;
2151   PetscFunctionReturn(0);
2152 }
2153 
2154 #undef __FUNCT__
2155 #define __FUNCT__ "DMPlexCreateNeighborCSR"
2156 PetscErrorCode DMPlexCreateNeighborCSR(DM dm, PetscInt cellHeight, PetscInt *numVertices, PetscInt **offsets, PetscInt **adjacency)
2157 {
2158   const PetscInt maxFaceCases = 30;
2159   PetscInt       numFaceCases = 0;
2160   PetscInt       numFaceVertices[30]; /* maxFaceCases, C89 sucks sucks sucks */
2161   PetscInt      *off, *adj;
2162   PetscInt      *neighborCells, *tmpClosure;
2163   PetscInt       maxConeSize, maxSupportSize, maxClosure, maxNeighbors;
2164   PetscInt       dim, cellDim, depth = 0, faceDepth, cStart, cEnd, c, numCells, cell;
2165   PetscErrorCode ierr;
2166 
2167   PetscFunctionBegin;
2168   /* For parallel partitioning, I think you have to communicate supports */
2169   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2170   cellDim = dim - cellHeight;
2171   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2172   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
2173   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
2174   if (cEnd - cStart == 0) {
2175     if (numVertices) *numVertices = 0;
2176     if (offsets)   *offsets   = NULL;
2177     if (adjacency) *adjacency = NULL;
2178     PetscFunctionReturn(0);
2179   }
2180   numCells  = cEnd - cStart;
2181   faceDepth = depth - cellHeight;
2182   /* Setup face recognition */
2183   if (faceDepth == 1) {
2184     PetscInt cornersSeen[30] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Could use PetscBT */
2185 
2186     for (c = cStart; c < cEnd; ++c) {
2187       PetscInt corners;
2188 
2189       ierr = DMPlexGetConeSize(dm, c, &corners);CHKERRQ(ierr);
2190       if (!cornersSeen[corners]) {
2191         PetscInt nFV;
2192 
2193         if (numFaceCases >= maxFaceCases) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Exceeded maximum number of face recognition cases");
2194         cornersSeen[corners] = 1;
2195 
2196         ierr = DMPlexGetNumFaceVertices(dm, cellDim, corners, &nFV);CHKERRQ(ierr);
2197 
2198         numFaceVertices[numFaceCases++] = nFV;
2199       }
2200     }
2201   }
2202   maxClosure   = 2*PetscMax(PetscPowInt(maxConeSize,depth+1),PetscPowInt(maxSupportSize,depth+1));
2203   maxNeighbors = PetscPowInt(maxConeSize,depth+1)*PetscPowInt(maxSupportSize,depth+1);
2204   ierr         = PetscMalloc2(maxNeighbors,PetscInt,&neighborCells,maxClosure,PetscInt,&tmpClosure);CHKERRQ(ierr);
2205   ierr         = PetscMalloc((numCells+1) * sizeof(PetscInt), &off);CHKERRQ(ierr);
2206   ierr         = PetscMemzero(off, (numCells+1) * sizeof(PetscInt));CHKERRQ(ierr);
2207   /* Count neighboring cells */
2208   for (cell = cStart; cell < cEnd; ++cell) {
2209     PetscInt numNeighbors = maxNeighbors, n;
2210 
2211     ierr = DMPlexGetAdjacencySingleLevel_Internal(dm, cell, PETSC_TRUE, tmpClosure, &numNeighbors, neighborCells);CHKERRQ(ierr);
2212     /* Get meet with each cell, and check with recognizer (could optimize to check each pair only once) */
2213     for (n = 0; n < numNeighbors; ++n) {
2214       PetscInt        cellPair[2];
2215       PetscBool       found    = faceDepth > 1 ? PETSC_TRUE : PETSC_FALSE;
2216       PetscInt        meetSize = 0;
2217       const PetscInt *meet    = NULL;
2218 
2219       cellPair[0] = cell; cellPair[1] = neighborCells[n];
2220       if (cellPair[0] == cellPair[1]) continue;
2221       if (!found) {
2222         ierr = DMPlexGetMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2223         if (meetSize) {
2224           PetscInt f;
2225 
2226           for (f = 0; f < numFaceCases; ++f) {
2227             if (numFaceVertices[f] == meetSize) {
2228               found = PETSC_TRUE;
2229               break;
2230             }
2231           }
2232         }
2233         ierr = DMPlexRestoreMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2234       }
2235       if (found) ++off[cell-cStart+1];
2236     }
2237   }
2238   /* Prefix sum */
2239   for (cell = 1; cell <= numCells; ++cell) off[cell] += off[cell-1];
2240 
2241   if (adjacency) {
2242     ierr = PetscMalloc(off[numCells] * sizeof(PetscInt), &adj);CHKERRQ(ierr);
2243     /* Get neighboring cells */
2244     for (cell = cStart; cell < cEnd; ++cell) {
2245       PetscInt numNeighbors = maxNeighbors, n;
2246       PetscInt cellOffset   = 0;
2247 
2248       ierr = DMPlexGetAdjacencySingleLevel_Internal(dm, cell, PETSC_TRUE, tmpClosure, &numNeighbors, neighborCells);CHKERRQ(ierr);
2249       /* Get meet with each cell, and check with recognizer (could optimize to check each pair only once) */
2250       for (n = 0; n < numNeighbors; ++n) {
2251         PetscInt        cellPair[2];
2252         PetscBool       found    = faceDepth > 1 ? PETSC_TRUE : PETSC_FALSE;
2253         PetscInt        meetSize = 0;
2254         const PetscInt *meet    = NULL;
2255 
2256         cellPair[0] = cell; cellPair[1] = neighborCells[n];
2257         if (cellPair[0] == cellPair[1]) continue;
2258         if (!found) {
2259           ierr = DMPlexGetMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2260           if (meetSize) {
2261             PetscInt f;
2262 
2263             for (f = 0; f < numFaceCases; ++f) {
2264               if (numFaceVertices[f] == meetSize) {
2265                 found = PETSC_TRUE;
2266                 break;
2267               }
2268             }
2269           }
2270           ierr = DMPlexRestoreMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2271         }
2272         if (found) {
2273           adj[off[cell-cStart]+cellOffset] = neighborCells[n];
2274           ++cellOffset;
2275         }
2276       }
2277     }
2278   }
2279   ierr = PetscFree2(neighborCells,tmpClosure);CHKERRQ(ierr);
2280   if (numVertices) *numVertices = numCells;
2281   if (offsets)   *offsets   = off;
2282   if (adjacency) *adjacency = adj;
2283   PetscFunctionReturn(0);
2284 }
2285 
2286 #if defined(PETSC_HAVE_CHACO)
2287 #if defined(PETSC_HAVE_UNISTD_H)
2288 #include <unistd.h>
2289 #endif
2290 /* Chaco does not have an include file */
2291 PETSC_EXTERN int interface(int nvtxs, int *start, int *adjacency, int *vwgts,
2292                        float *ewgts, float *x, float *y, float *z, char *outassignname,
2293                        char *outfilename, short *assignment, int architecture, int ndims_tot,
2294                        int mesh_dims[3], double *goal, int global_method, int local_method,
2295                        int rqi_flag, int vmax, int ndims, double eigtol, long seed);
2296 
2297 extern int FREE_GRAPH;
2298 
2299 #undef __FUNCT__
2300 #define __FUNCT__ "DMPlexPartition_Chaco"
2301 PetscErrorCode DMPlexPartition_Chaco(DM dm, PetscInt numVertices, PetscInt start[], PetscInt adjacency[], PetscSection *partSection, IS *partition)
2302 {
2303   enum {DEFAULT_METHOD = 1, INERTIAL_METHOD = 3};
2304   MPI_Comm       comm;
2305   int            nvtxs          = numVertices; /* number of vertices in full graph */
2306   int           *vwgts          = NULL;   /* weights for all vertices */
2307   float         *ewgts          = NULL;   /* weights for all edges */
2308   float         *x              = NULL, *y = NULL, *z = NULL; /* coordinates for inertial method */
2309   char          *outassignname  = NULL;   /*  name of assignment output file */
2310   char          *outfilename    = NULL;   /* output file name */
2311   int            architecture   = 1;      /* 0 => hypercube, d => d-dimensional mesh */
2312   int            ndims_tot      = 0;      /* total number of cube dimensions to divide */
2313   int            mesh_dims[3];            /* dimensions of mesh of processors */
2314   double        *goal          = NULL;    /* desired set sizes for each set */
2315   int            global_method = 1;       /* global partitioning algorithm */
2316   int            local_method  = 1;       /* local partitioning algorithm */
2317   int            rqi_flag      = 0;       /* should I use RQI/Symmlq eigensolver? */
2318   int            vmax          = 200;     /* how many vertices to coarsen down to? */
2319   int            ndims         = 1;       /* number of eigenvectors (2^d sets) */
2320   double         eigtol        = 0.001;   /* tolerance on eigenvectors */
2321   long           seed          = 123636512; /* for random graph mutations */
2322   short int     *assignment;              /* Output partition */
2323   int            fd_stdout, fd_pipe[2];
2324   PetscInt      *points;
2325   PetscMPIInt    commSize;
2326   int            i, v, p;
2327   PetscErrorCode ierr;
2328 
2329   PetscFunctionBegin;
2330   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2331   ierr = MPI_Comm_size(comm, &commSize);CHKERRQ(ierr);
2332   if (!numVertices) {
2333     ierr = PetscSectionCreate(comm, partSection);CHKERRQ(ierr);
2334     ierr = PetscSectionSetChart(*partSection, 0, commSize);CHKERRQ(ierr);
2335     ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2336     ierr = ISCreateGeneral(comm, 0, NULL, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2337     PetscFunctionReturn(0);
2338   }
2339   FREE_GRAPH = 0;                         /* Do not let Chaco free my memory */
2340   for (i = 0; i < start[numVertices]; ++i) ++adjacency[i];
2341 
2342   if (global_method == INERTIAL_METHOD) {
2343     /* manager.createCellCoordinates(nvtxs, &x, &y, &z); */
2344     SETERRQ(comm, PETSC_ERR_SUP, "Inertial partitioning not yet supported");
2345   }
2346   mesh_dims[0] = commSize;
2347   mesh_dims[1] = 1;
2348   mesh_dims[2] = 1;
2349   ierr = PetscMalloc(nvtxs * sizeof(short int), &assignment);CHKERRQ(ierr);
2350   /* Chaco outputs to stdout. We redirect this to a buffer. */
2351   /* TODO: check error codes for UNIX calls */
2352 #if defined(PETSC_HAVE_UNISTD_H)
2353   {
2354     int piperet;
2355     piperet = pipe(fd_pipe);
2356     if (piperet) SETERRQ(comm,PETSC_ERR_SYS,"Could not create pipe");
2357     fd_stdout = dup(1);
2358     close(1);
2359     dup2(fd_pipe[1], 1);
2360   }
2361 #endif
2362   ierr = interface(nvtxs, (int*) start, (int*) adjacency, vwgts, ewgts, x, y, z, outassignname, outfilename,
2363                    assignment, architecture, ndims_tot, mesh_dims, goal, global_method, local_method, rqi_flag,
2364                    vmax, ndims, eigtol, seed);
2365 #if defined(PETSC_HAVE_UNISTD_H)
2366   {
2367     char msgLog[10000];
2368     int  count;
2369 
2370     fflush(stdout);
2371     count = read(fd_pipe[0], msgLog, (10000-1)*sizeof(char));
2372     if (count < 0) count = 0;
2373     msgLog[count] = 0;
2374     close(1);
2375     dup2(fd_stdout, 1);
2376     close(fd_stdout);
2377     close(fd_pipe[0]);
2378     close(fd_pipe[1]);
2379     if (ierr) SETERRQ1(comm, PETSC_ERR_LIB, "Error in Chaco library: %s", msgLog);
2380   }
2381 #endif
2382   /* Convert to PetscSection+IS */
2383   ierr = PetscSectionCreate(comm, partSection);CHKERRQ(ierr);
2384   ierr = PetscSectionSetChart(*partSection, 0, commSize);CHKERRQ(ierr);
2385   for (v = 0; v < nvtxs; ++v) {
2386     ierr = PetscSectionAddDof(*partSection, assignment[v], 1);CHKERRQ(ierr);
2387   }
2388   ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2389   ierr = PetscMalloc(nvtxs * sizeof(PetscInt), &points);CHKERRQ(ierr);
2390   for (p = 0, i = 0; p < commSize; ++p) {
2391     for (v = 0; v < nvtxs; ++v) {
2392       if (assignment[v] == p) points[i++] = v;
2393     }
2394   }
2395   if (i != nvtxs) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of points %D should be %D", i, nvtxs);
2396   ierr = ISCreateGeneral(comm, nvtxs, points, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2397   if (global_method == INERTIAL_METHOD) {
2398     /* manager.destroyCellCoordinates(nvtxs, &x, &y, &z); */
2399   }
2400   ierr = PetscFree(assignment);CHKERRQ(ierr);
2401   for (i = 0; i < start[numVertices]; ++i) --adjacency[i];
2402   PetscFunctionReturn(0);
2403 }
2404 #endif
2405 
2406 #if defined(PETSC_HAVE_PARMETIS)
2407 #undef __FUNCT__
2408 #define __FUNCT__ "DMPlexPartition_ParMetis"
2409 PetscErrorCode DMPlexPartition_ParMetis(DM dm, PetscInt numVertices, PetscInt start[], PetscInt adjacency[], PetscSection *partSection, IS *partition)
2410 {
2411   PetscFunctionBegin;
2412   SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "ParMetis not yet supported");
2413   PetscFunctionReturn(0);
2414 }
2415 #endif
2416 
2417 #undef __FUNCT__
2418 #define __FUNCT__ "DMPlexEnlargePartition"
2419 /* Expand the partition by BFS on the adjacency graph */
2420 PetscErrorCode DMPlexEnlargePartition(DM dm, const PetscInt start[], const PetscInt adjacency[], PetscSection origPartSection, IS origPartition, PetscSection *partSection, IS *partition)
2421 {
2422   PetscHashI      h;
2423   const PetscInt *points;
2424   PetscInt      **tmpPoints, *newPoints, totPoints = 0;
2425   PetscInt        pStart, pEnd, part, q;
2426   PetscErrorCode  ierr;
2427 
2428   PetscFunctionBegin;
2429   PetscHashICreate(h);
2430   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), partSection);CHKERRQ(ierr);
2431   ierr = PetscSectionGetChart(origPartSection, &pStart, &pEnd);CHKERRQ(ierr);
2432   ierr = PetscSectionSetChart(*partSection, pStart, pEnd);CHKERRQ(ierr);
2433   ierr = ISGetIndices(origPartition, &points);CHKERRQ(ierr);
2434   ierr = PetscMalloc((pEnd - pStart) * sizeof(PetscInt*), &tmpPoints);CHKERRQ(ierr);
2435   for (part = pStart; part < pEnd; ++part) {
2436     PetscInt numPoints, nP, numNewPoints, off, p, n = 0;
2437 
2438     PetscHashIClear(h);
2439     ierr = PetscSectionGetDof(origPartSection, part, &numPoints);CHKERRQ(ierr);
2440     ierr = PetscSectionGetOffset(origPartSection, part, &off);CHKERRQ(ierr);
2441     /* Add all existing points to h */
2442     for (p = 0; p < numPoints; ++p) {
2443       const PetscInt point = points[off+p];
2444       PetscHashIAdd(h, point, 1);
2445     }
2446     PetscHashISize(h, nP);
2447     if (nP != numPoints) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid partition has %d points, but only %d were unique", numPoints, nP);
2448     /* Add all points in next BFS level */
2449     /*   TODO We are brute forcing here, but could check the adjacency size to find the boundary */
2450     for (p = 0; p < numPoints; ++p) {
2451       const PetscInt point = points[off+p];
2452       PetscInt       s     = start[point], e = start[point+1], a;
2453 
2454       for (a = s; a < e; ++a) PetscHashIAdd(h, adjacency[a], 1);
2455     }
2456     PetscHashISize(h, numNewPoints);
2457     ierr = PetscSectionSetDof(*partSection, part, numNewPoints);CHKERRQ(ierr);
2458     ierr = PetscMalloc(numNewPoints * sizeof(PetscInt), &tmpPoints[part]);CHKERRQ(ierr);
2459     if (numNewPoints) PetscHashIGetKeys(h, n, tmpPoints[part]); /* Should not need this conditional */
2460     totPoints += numNewPoints;
2461   }
2462   ierr = ISRestoreIndices(origPartition, &points);CHKERRQ(ierr);
2463   PetscHashIDestroy(h);
2464   ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2465   ierr = PetscMalloc(totPoints * sizeof(PetscInt), &newPoints);CHKERRQ(ierr);
2466   for (part = pStart, q = 0; part < pEnd; ++part) {
2467     PetscInt numPoints, p;
2468 
2469     ierr = PetscSectionGetDof(*partSection, part, &numPoints);CHKERRQ(ierr);
2470     for (p = 0; p < numPoints; ++p, ++q) newPoints[q] = tmpPoints[part][p];
2471     ierr = PetscFree(tmpPoints[part]);CHKERRQ(ierr);
2472   }
2473   ierr = PetscFree(tmpPoints);CHKERRQ(ierr);
2474   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), totPoints, newPoints, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2475   PetscFunctionReturn(0);
2476 }
2477 
2478 #undef __FUNCT__
2479 #define __FUNCT__ "DMPlexCreatePartition"
2480 /*
2481   DMPlexCreatePartition - Create a non-overlapping partition of the points at the given height
2482 
2483   Collective on DM
2484 
2485   Input Parameters:
2486   + dm - The DM
2487   . height - The height for points in the partition
2488   - enlarge - Expand each partition with neighbors
2489 
2490   Output Parameters:
2491   + partSection - The PetscSection giving the division of points by partition
2492   . partition - The list of points by partition
2493   . origPartSection - If enlarge is true, the PetscSection giving the division of points before enlarging by partition, otherwise NULL
2494   - origPartition - If enlarge is true, the list of points before enlarging by partition, otherwise NULL
2495 
2496   Level: developer
2497 
2498 .seealso DMPlexDistribute()
2499 */
2500 PetscErrorCode DMPlexCreatePartition(DM dm, PetscInt height, PetscBool enlarge, PetscSection *partSection, IS *partition, PetscSection *origPartSection, IS *origPartition)
2501 {
2502   PetscMPIInt    size;
2503   PetscErrorCode ierr;
2504 
2505   PetscFunctionBegin;
2506   ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
2507 
2508   *origPartSection = NULL;
2509   *origPartition   = NULL;
2510   if (size == 1) {
2511     PetscInt *points;
2512     PetscInt  cStart, cEnd, c;
2513 
2514     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
2515     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), partSection);CHKERRQ(ierr);
2516     ierr = PetscSectionSetChart(*partSection, 0, size);CHKERRQ(ierr);
2517     ierr = PetscSectionSetDof(*partSection, 0, cEnd-cStart);CHKERRQ(ierr);
2518     ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2519     ierr = PetscMalloc((cEnd - cStart) * sizeof(PetscInt), &points);CHKERRQ(ierr);
2520     for (c = cStart; c < cEnd; ++c) points[c] = c;
2521     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), cEnd-cStart, points, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2522     PetscFunctionReturn(0);
2523   }
2524   if (height == 0) {
2525     PetscInt  numVertices;
2526     PetscInt *start     = NULL;
2527     PetscInt *adjacency = NULL;
2528 
2529     ierr = DMPlexCreateNeighborCSR(dm, 0, &numVertices, &start, &adjacency);CHKERRQ(ierr);
2530     if (1) {
2531 #if defined(PETSC_HAVE_CHACO)
2532       ierr = DMPlexPartition_Chaco(dm, numVertices, start, adjacency, partSection, partition);CHKERRQ(ierr);
2533 #endif
2534     } else {
2535 #if defined(PETSC_HAVE_PARMETIS)
2536       ierr = DMPlexPartition_ParMetis(dm, numVertices, start, adjacency, partSection, partition);CHKERRQ(ierr);
2537 #endif
2538     }
2539     if (enlarge) {
2540       *origPartSection = *partSection;
2541       *origPartition   = *partition;
2542 
2543       ierr = DMPlexEnlargePartition(dm, start, adjacency, *origPartSection, *origPartition, partSection, partition);CHKERRQ(ierr);
2544     }
2545     ierr = PetscFree(start);CHKERRQ(ierr);
2546     ierr = PetscFree(adjacency);CHKERRQ(ierr);
2547 # if 0
2548   } else if (height == 1) {
2549     /* Build the dual graph for faces and partition the hypergraph */
2550     PetscInt numEdges;
2551 
2552     buildFaceCSRV(mesh, mesh->getFactory()->getNumbering(mesh, mesh->depth()-1), &numEdges, &start, &adjacency, GraphPartitioner::zeroBase());
2553     GraphPartitioner().partition(numEdges, start, adjacency, partition, manager);
2554     destroyCSR(numEdges, start, adjacency);
2555 #endif
2556   } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid partition height %D", height);
2557   PetscFunctionReturn(0);
2558 }
2559 
2560 #undef __FUNCT__
2561 #define __FUNCT__ "DMPlexCreatePartitionClosure"
2562 PetscErrorCode DMPlexCreatePartitionClosure(DM dm, PetscSection pointSection, IS pointPartition, PetscSection *section, IS *partition)
2563 {
2564   /* const PetscInt  height = 0; */
2565   const PetscInt *partArray;
2566   PetscInt       *allPoints, *packPoints;
2567   PetscInt        rStart, rEnd, rank, pStart, pEnd, newSize;
2568   PetscErrorCode  ierr;
2569   PetscBT         bt;
2570   PetscSegBuffer  segpack,segpart;
2571 
2572   PetscFunctionBegin;
2573   ierr = PetscSectionGetChart(pointSection, &rStart, &rEnd);CHKERRQ(ierr);
2574   ierr = ISGetIndices(pointPartition, &partArray);CHKERRQ(ierr);
2575   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
2576   ierr = PetscSectionSetChart(*section, rStart, rEnd);CHKERRQ(ierr);
2577   ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
2578   ierr = PetscBTCreate(pEnd-pStart,&bt);CHKERRQ(ierr);
2579   ierr = PetscSegBufferCreate(sizeof(PetscInt),1000,&segpack);CHKERRQ(ierr);
2580   ierr = PetscSegBufferCreate(sizeof(PetscInt),1000,&segpart);CHKERRQ(ierr);
2581   for (rank = rStart; rank < rEnd; ++rank) {
2582     PetscInt partSize = 0, numPoints, offset, p, *PETSC_RESTRICT placePoints;
2583 
2584     ierr = PetscSectionGetDof(pointSection, rank, &numPoints);CHKERRQ(ierr);
2585     ierr = PetscSectionGetOffset(pointSection, rank, &offset);CHKERRQ(ierr);
2586     for (p = 0; p < numPoints; ++p) {
2587       PetscInt  point   = partArray[offset+p], closureSize, c;
2588       PetscInt *closure = NULL;
2589 
2590       /* TODO Include support for height > 0 case */
2591       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2592       for (c=0; c<closureSize; c++) {
2593         PetscInt cpoint = closure[c*2];
2594         if (!PetscBTLookupSet(bt,cpoint-pStart)) {
2595           PetscInt *PETSC_RESTRICT pt;
2596           partSize++;
2597           ierr = PetscSegBufferGetInts(segpart,1,&pt);CHKERRQ(ierr);
2598           *pt = cpoint;
2599         }
2600       }
2601       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2602     }
2603     ierr = PetscSectionSetDof(*section, rank, partSize);CHKERRQ(ierr);
2604     ierr = PetscSegBufferGetInts(segpack,partSize,&placePoints);CHKERRQ(ierr);
2605     ierr = PetscSegBufferExtractTo(segpart,placePoints);CHKERRQ(ierr);
2606     ierr = PetscSortInt(partSize,placePoints);CHKERRQ(ierr);
2607     for (p=0; p<partSize; p++) {ierr = PetscBTClear(bt,placePoints[p]-pStart);CHKERRQ(ierr);}
2608   }
2609   ierr = PetscBTDestroy(&bt);CHKERRQ(ierr);
2610   ierr = PetscSegBufferDestroy(&segpart);CHKERRQ(ierr);
2611 
2612   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
2613   ierr = PetscSectionGetStorageSize(*section, &newSize);CHKERRQ(ierr);
2614   ierr = PetscMalloc(newSize * sizeof(PetscInt), &allPoints);CHKERRQ(ierr);
2615 
2616   ierr = PetscSegBufferExtractInPlace(segpack,&packPoints);CHKERRQ(ierr);
2617   for (rank = rStart; rank < rEnd; ++rank) {
2618     PetscInt numPoints, offset;
2619 
2620     ierr = PetscSectionGetDof(*section, rank, &numPoints);CHKERRQ(ierr);
2621     ierr = PetscSectionGetOffset(*section, rank, &offset);CHKERRQ(ierr);
2622     ierr = PetscMemcpy(&allPoints[offset], packPoints, numPoints * sizeof(PetscInt));CHKERRQ(ierr);
2623     packPoints += numPoints;
2624   }
2625 
2626   ierr = PetscSegBufferDestroy(&segpack);CHKERRQ(ierr);
2627   ierr = ISRestoreIndices(pointPartition, &partArray);CHKERRQ(ierr);
2628   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), newSize, allPoints, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2629   PetscFunctionReturn(0);
2630 }
2631 
2632 #undef __FUNCT__
2633 #define __FUNCT__ "DMPlexDistributeField"
2634 /*
2635   Input Parameters:
2636 . originalSection
2637 , originalVec
2638 
2639   Output Parameters:
2640 . newSection
2641 . newVec
2642 */
2643 PetscErrorCode DMPlexDistributeField(DM dm, PetscSF pointSF, PetscSection originalSection, Vec originalVec, PetscSection newSection, Vec newVec)
2644 {
2645   PetscSF        fieldSF;
2646   PetscInt      *remoteOffsets, fieldSize;
2647   PetscScalar   *originalValues, *newValues;
2648   PetscErrorCode ierr;
2649 
2650   PetscFunctionBegin;
2651   ierr = PetscSFDistributeSection(pointSF, originalSection, &remoteOffsets, newSection);CHKERRQ(ierr);
2652 
2653   ierr = PetscSectionGetStorageSize(newSection, &fieldSize);CHKERRQ(ierr);
2654   ierr = VecSetSizes(newVec, fieldSize, PETSC_DETERMINE);CHKERRQ(ierr);
2655   ierr = VecSetType(newVec,dm->vectype);CHKERRQ(ierr);
2656 
2657   ierr = VecGetArray(originalVec, &originalValues);CHKERRQ(ierr);
2658   ierr = VecGetArray(newVec, &newValues);CHKERRQ(ierr);
2659   ierr = PetscSFCreateSectionSF(pointSF, originalSection, remoteOffsets, newSection, &fieldSF);CHKERRQ(ierr);
2660   ierr = PetscSFBcastBegin(fieldSF, MPIU_SCALAR, originalValues, newValues);CHKERRQ(ierr);
2661   ierr = PetscSFBcastEnd(fieldSF, MPIU_SCALAR, originalValues, newValues);CHKERRQ(ierr);
2662   ierr = PetscSFDestroy(&fieldSF);CHKERRQ(ierr);
2663   ierr = VecRestoreArray(newVec, &newValues);CHKERRQ(ierr);
2664   ierr = VecRestoreArray(originalVec, &originalValues);CHKERRQ(ierr);
2665   PetscFunctionReturn(0);
2666 }
2667 
2668 #undef __FUNCT__
2669 #define __FUNCT__ "DMPlexDistribute"
2670 /*@C
2671   DMPlexDistribute - Distributes the mesh and any associated sections.
2672 
2673   Not Collective
2674 
2675   Input Parameter:
2676 + dm  - The original DMPlex object
2677 . partitioner - The partitioning package, or NULL for the default
2678 - overlap - The overlap of partitions, 0 is the default
2679 
2680   Output Parameter:
2681 . parallelMesh - The distributed DMPlex object, or NULL
2682 
2683   Note: If the mesh was not distributed, the return value is NULL
2684 
2685   Level: intermediate
2686 
2687 .keywords: mesh, elements
2688 .seealso: DMPlexCreate(), DMPlexDistributeByFace()
2689 @*/
2690 PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, DM *dmParallel)
2691 {
2692   DM_Plex               *mesh   = (DM_Plex*) dm->data, *pmesh;
2693   MPI_Comm               comm;
2694   const PetscInt         height = 0;
2695   PetscInt               dim, numRemoteRanks;
2696   IS                     origCellPart,        cellPart,        part;
2697   PetscSection           origCellPartSection, cellPartSection, partSection;
2698   PetscSFNode           *remoteRanks;
2699   PetscSF                partSF, pointSF, coneSF;
2700   ISLocalToGlobalMapping renumbering;
2701   PetscSection           originalConeSection, newConeSection;
2702   PetscInt              *remoteOffsets;
2703   PetscInt              *cones, *newCones, newConesSize;
2704   PetscBool              flg;
2705   PetscMPIInt            rank, numProcs, p;
2706   PetscErrorCode         ierr;
2707 
2708   PetscFunctionBegin;
2709   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2710   PetscValidPointer(dmParallel,4);
2711 
2712   ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
2713   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2714   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2715   ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
2716 
2717   *dmParallel = NULL;
2718   if (numProcs == 1) PetscFunctionReturn(0);
2719 
2720   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2721   /* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
2722   ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
2723   if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
2724   ierr = DMPlexCreatePartition(dm, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
2725   /* Create SF assuming a serial partition for all processes: Could check for IS length here */
2726   if (!rank) numRemoteRanks = numProcs;
2727   else       numRemoteRanks = 0;
2728   ierr = PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);CHKERRQ(ierr);
2729   for (p = 0; p < numRemoteRanks; ++p) {
2730     remoteRanks[p].rank  = p;
2731     remoteRanks[p].index = 0;
2732   }
2733   ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
2734   ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
2735   ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
2736   if (flg) {
2737     ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
2738     ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2739     ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
2740     if (origCellPart) {
2741       ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
2742       ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2743       ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
2744     }
2745     ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
2746   }
2747   /* Close the partition over the mesh */
2748   ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
2749   ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
2750   ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
2751   /* Create new mesh */
2752   ierr  = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
2753   ierr  = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
2754   ierr  = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
2755   pmesh = (DM_Plex*) (*dmParallel)->data;
2756   /* Distribute sieve points and the global point numbering (replaces creating remote bases) */
2757   ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
2758   if (flg) {
2759     ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
2760     ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2761     ierr = ISView(part, NULL);CHKERRQ(ierr);
2762     ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
2763     ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
2764     ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
2765   }
2766   ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
2767   /* Distribute cone section */
2768   ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
2769   ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
2770   ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
2771   ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
2772   {
2773     PetscInt pStart, pEnd, p;
2774 
2775     ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
2776     for (p = pStart; p < pEnd; ++p) {
2777       PetscInt coneSize;
2778       ierr               = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
2779       pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
2780     }
2781   }
2782   /* Communicate and renumber cones */
2783   ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
2784   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2785   ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
2786   ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2787   ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2788   ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
2789   ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
2790   ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
2791   if (flg) {
2792     ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
2793     ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2794     ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
2795     ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2796     ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
2797   }
2798   ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
2799   ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
2800   ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2801   ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2802   ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
2803   /* Create supports and stratify sieve */
2804   {
2805     PetscInt pStart, pEnd;
2806 
2807     ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2808     ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2809   }
2810   ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
2811   ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
2812   /* Distribute Coordinates */
2813   {
2814     PetscSection originalCoordSection, newCoordSection;
2815     Vec          originalCoordinates, newCoordinates;
2816     const char  *name;
2817 
2818     ierr = DMPlexGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
2819     ierr = DMPlexGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
2820     ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
2821     ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
2822     ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
2823     ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
2824 
2825     ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
2826     ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
2827     ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
2828   }
2829   /* Distribute labels */
2830   ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
2831   {
2832     DMLabel  next      = mesh->labels, newNext = pmesh->labels;
2833     PetscInt numLabels = 0, l;
2834 
2835     /* Bcast number of labels */
2836     while (next) {
2837       ++numLabels; next = next->next;
2838     }
2839     ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
2840     next = mesh->labels;
2841     for (l = 0; l < numLabels; ++l) {
2842       DMLabel         newLabel;
2843       const PetscInt *partArray;
2844       char           *name;
2845       PetscInt       *stratumSizes = NULL, *points = NULL;
2846       PetscMPIInt    *sendcnts     = NULL, *offsets = NULL, *displs = NULL;
2847       PetscInt        nameSize, s, p;
2848       PetscBool       isdepth;
2849       size_t          len = 0;
2850 
2851       /* Bcast name (could filter for no points) */
2852       if (!rank) {ierr = PetscStrlen(next->name, &len);CHKERRQ(ierr);}
2853       nameSize = len;
2854       ierr     = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
2855       ierr     = PetscMalloc(nameSize+1, &name);CHKERRQ(ierr);
2856       if (!rank) {ierr = PetscMemcpy(name, next->name, nameSize+1);CHKERRQ(ierr);}
2857       ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
2858       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
2859       if (isdepth) {            /* skip because "depth" is not distributed */
2860         ierr = PetscFree(name);CHKERRQ(ierr);
2861         if (!rank) next = next->next;
2862         continue;
2863       }
2864       ierr           = PetscNew(struct _n_DMLabel, &newLabel);CHKERRQ(ierr);
2865       newLabel->name = name;
2866       /* Bcast numStrata (could filter for no points in stratum) */
2867       if (!rank) newLabel->numStrata = next->numStrata;
2868       ierr = MPI_Bcast(&newLabel->numStrata, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
2869       ierr = PetscMalloc3(newLabel->numStrata,PetscInt,&newLabel->stratumValues,
2870                           newLabel->numStrata,PetscInt,&newLabel->stratumSizes,
2871                           newLabel->numStrata+1,PetscInt,&newLabel->stratumOffsets);CHKERRQ(ierr);
2872       /* Bcast stratumValues (could filter for no points in stratum) */
2873       if (!rank) {ierr = PetscMemcpy(newLabel->stratumValues, next->stratumValues, next->numStrata * sizeof(PetscInt));CHKERRQ(ierr);}
2874       ierr = MPI_Bcast(newLabel->stratumValues, newLabel->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
2875       /* Find size on each process and Scatter */
2876       if (!rank) {
2877         ierr = ISGetIndices(part, &partArray);CHKERRQ(ierr);
2878         ierr = PetscMalloc(numProcs*next->numStrata * sizeof(PetscInt), &stratumSizes);CHKERRQ(ierr);
2879         ierr = PetscMemzero(stratumSizes, numProcs*next->numStrata * sizeof(PetscInt));CHKERRQ(ierr);
2880         for (s = 0; s < next->numStrata; ++s) {
2881           for (p = next->stratumOffsets[s]; p < next->stratumOffsets[s]+next->stratumSizes[s]; ++p) {
2882             const PetscInt point = next->points[p];
2883             PetscInt       proc;
2884 
2885             for (proc = 0; proc < numProcs; ++proc) {
2886               PetscInt dof, off, pPart;
2887 
2888               ierr = PetscSectionGetDof(partSection, proc, &dof);CHKERRQ(ierr);
2889               ierr = PetscSectionGetOffset(partSection, proc, &off);CHKERRQ(ierr);
2890               for (pPart = off; pPart < off+dof; ++pPart) {
2891                 if (partArray[pPart] == point) {
2892                   ++stratumSizes[proc*next->numStrata+s];
2893                   break;
2894                 }
2895               }
2896             }
2897           }
2898         }
2899         ierr = ISRestoreIndices(part, &partArray);CHKERRQ(ierr);
2900       }
2901       ierr = MPI_Scatter(stratumSizes, newLabel->numStrata, MPIU_INT, newLabel->stratumSizes, newLabel->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
2902       /* Calculate stratumOffsets */
2903       newLabel->stratumOffsets[0] = 0;
2904       for (s = 0; s < newLabel->numStrata; ++s) {
2905         newLabel->stratumOffsets[s+1] = newLabel->stratumSizes[s] + newLabel->stratumOffsets[s];
2906       }
2907       /* Pack points and Scatter */
2908       if (!rank) {
2909         ierr = PetscMalloc3(numProcs,PetscMPIInt,&sendcnts,numProcs,PetscMPIInt,&offsets,numProcs+1,PetscMPIInt,&displs);CHKERRQ(ierr);
2910         displs[0] = 0;
2911         for (p = 0; p < numProcs; ++p) {
2912           sendcnts[p] = 0;
2913           for (s = 0; s < next->numStrata; ++s) {
2914             sendcnts[p] += stratumSizes[p*next->numStrata+s];
2915           }
2916           offsets[p]  = displs[p];
2917           displs[p+1] = displs[p] + sendcnts[p];
2918         }
2919         ierr = PetscMalloc(displs[numProcs] * sizeof(PetscInt), &points);CHKERRQ(ierr);
2920         for (s = 0; s < next->numStrata; ++s) {
2921           for (p = next->stratumOffsets[s]; p < next->stratumOffsets[s]+next->stratumSizes[s]; ++p) {
2922             const PetscInt point = next->points[p];
2923             PetscInt       proc;
2924 
2925             for (proc = 0; proc < numProcs; ++proc) {
2926               PetscInt dof, off, pPart;
2927 
2928               ierr = PetscSectionGetDof(partSection, proc, &dof);CHKERRQ(ierr);
2929               ierr = PetscSectionGetOffset(partSection, proc, &off);CHKERRQ(ierr);
2930               for (pPart = off; pPart < off+dof; ++pPart) {
2931                 if (partArray[pPart] == point) {
2932                   points[offsets[proc]++] = point;
2933                   break;
2934                 }
2935               }
2936             }
2937           }
2938         }
2939       }
2940       ierr = PetscMalloc(newLabel->stratumOffsets[newLabel->numStrata] * sizeof(PetscInt), &newLabel->points);CHKERRQ(ierr);
2941       ierr = MPI_Scatterv(points, sendcnts, displs, MPIU_INT, newLabel->points, newLabel->stratumOffsets[newLabel->numStrata], MPIU_INT, 0, comm);CHKERRQ(ierr);
2942       ierr = PetscFree(points);CHKERRQ(ierr);
2943       ierr = PetscFree3(sendcnts,offsets,displs);CHKERRQ(ierr);
2944       ierr = PetscFree(stratumSizes);CHKERRQ(ierr);
2945       /* Renumber points */
2946       ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newLabel->stratumOffsets[newLabel->numStrata], newLabel->points, NULL, newLabel->points);CHKERRQ(ierr);
2947       /* Sort points */
2948       for (s = 0; s < newLabel->numStrata; ++s) {
2949         ierr = PetscSortInt(newLabel->stratumSizes[s], &newLabel->points[newLabel->stratumOffsets[s]]);CHKERRQ(ierr);
2950       }
2951       /* Insert into list */
2952       if (newNext) newNext->next = newLabel;
2953       else pmesh->labels = newLabel;
2954       newNext = newLabel;
2955       if (!rank) next = next->next;
2956     }
2957   }
2958   ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
2959   /* Cleanup Partition */
2960   ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
2961   ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
2962   ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
2963   ierr = ISDestroy(&part);CHKERRQ(ierr);
2964   /* Create point SF for parallel mesh */
2965   ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
2966   {
2967     const PetscInt *leaves;
2968     PetscSFNode    *remotePoints, *rowners, *lowners;
2969     PetscInt        numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
2970     PetscInt        pStart, pEnd;
2971 
2972     ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
2973     ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
2974     ierr = PetscMalloc2(numRoots,PetscSFNode,&rowners,numLeaves,PetscSFNode,&lowners);CHKERRQ(ierr);
2975     for (p=0; p<numRoots; p++) {
2976       rowners[p].rank  = -1;
2977       rowners[p].index = -1;
2978     }
2979     if (origCellPart) {
2980       /* Make sure cells in the original partition are not assigned to other procs */
2981       const PetscInt *origCells;
2982 
2983       ierr = ISGetIndices(origCellPart, &origCells);CHKERRQ(ierr);
2984       for (p = 0; p < numProcs; ++p) {
2985         PetscInt dof, off, d;
2986 
2987         ierr = PetscSectionGetDof(origCellPartSection, p, &dof);CHKERRQ(ierr);
2988         ierr = PetscSectionGetOffset(origCellPartSection, p, &off);CHKERRQ(ierr);
2989         for (d = off; d < off+dof; ++d) {
2990           rowners[origCells[d]].rank = p;
2991         }
2992       }
2993       ierr = ISRestoreIndices(origCellPart, &origCells);CHKERRQ(ierr);
2994     }
2995     ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
2996     ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
2997 
2998     ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
2999     ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
3000     for (p = 0; p < numLeaves; ++p) {
3001       if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
3002         lowners[p].rank  = rank;
3003         lowners[p].index = leaves ? leaves[p] : p;
3004       } else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
3005         lowners[p].rank  = -2;
3006         lowners[p].index = -2;
3007       }
3008     }
3009     for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
3010       rowners[p].rank  = -3;
3011       rowners[p].index = -3;
3012     }
3013     ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
3014     ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
3015     ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
3016     ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
3017     for (p = 0; p < numLeaves; ++p) {
3018       if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
3019       if (lowners[p].rank != rank) ++numGhostPoints;
3020     }
3021     ierr = PetscMalloc(numGhostPoints * sizeof(PetscInt),    &ghostPoints);CHKERRQ(ierr);
3022     ierr = PetscMalloc(numGhostPoints * sizeof(PetscSFNode), &remotePoints);CHKERRQ(ierr);
3023     for (p = 0, gp = 0; p < numLeaves; ++p) {
3024       if (lowners[p].rank != rank) {
3025         ghostPoints[gp]        = leaves ? leaves[p] : p;
3026         remotePoints[gp].rank  = lowners[p].rank;
3027         remotePoints[gp].index = lowners[p].index;
3028         ++gp;
3029       }
3030     }
3031     ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
3032     ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3033     ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
3034   }
3035   ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
3036   /* Cleanup */
3037   ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);
3038   ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
3039   ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
3040   PetscFunctionReturn(0);
3041 }
3042 
3043 #undef __FUNCT__
3044 #define __FUNCT__ "DMPlexInvertCell"
3045 /*@C
3046   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3047 
3048   Input Parameters:
3049 + numCorners - The number of vertices in a cell
3050 - cone - The incoming cone
3051 
3052   Output Parameter:
3053 . cone - The inverted cone (in-place)
3054 
3055   Level: developer
3056 
3057 .seealso: DMPlexGenerate()
3058 @*/
3059 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3060 {
3061   int tmpc;
3062 
3063   PetscFunctionBegin;
3064   if (dim != 3) PetscFunctionReturn(0);
3065   switch (numCorners) {
3066   case 4:
3067     tmpc    = cone[0];
3068     cone[0] = cone[1];
3069     cone[1] = tmpc;
3070     break;
3071   case 8:
3072     tmpc    = cone[1];
3073     cone[1] = cone[3];
3074     cone[3] = tmpc;
3075     break;
3076   default: break;
3077   }
3078   PetscFunctionReturn(0);
3079 }
3080 
3081 #undef __FUNCT__
3082 #define __FUNCT__ "DMPlexInvertCells_Internal"
3083 /* This is to fix the tetrahedron orientation from TetGen */
3084 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3085 {
3086   PetscInt       bound = numCells*numCorners, coff;
3087   PetscErrorCode ierr;
3088 
3089   PetscFunctionBegin;
3090   for (coff = 0; coff < bound; coff += numCorners) {
3091     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3092   }
3093   PetscFunctionReturn(0);
3094 }
3095 
3096 #if defined(PETSC_HAVE_TRIANGLE)
3097 #include <triangle.h>
3098 
3099 #undef __FUNCT__
3100 #define __FUNCT__ "InitInput_Triangle"
3101 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3102 {
3103   PetscFunctionBegin;
3104   inputCtx->numberofpoints             = 0;
3105   inputCtx->numberofpointattributes    = 0;
3106   inputCtx->pointlist                  = NULL;
3107   inputCtx->pointattributelist         = NULL;
3108   inputCtx->pointmarkerlist            = NULL;
3109   inputCtx->numberofsegments           = 0;
3110   inputCtx->segmentlist                = NULL;
3111   inputCtx->segmentmarkerlist          = NULL;
3112   inputCtx->numberoftriangleattributes = 0;
3113   inputCtx->trianglelist               = NULL;
3114   inputCtx->numberofholes              = 0;
3115   inputCtx->holelist                   = NULL;
3116   inputCtx->numberofregions            = 0;
3117   inputCtx->regionlist                 = NULL;
3118   PetscFunctionReturn(0);
3119 }
3120 
3121 #undef __FUNCT__
3122 #define __FUNCT__ "InitOutput_Triangle"
3123 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3124 {
3125   PetscFunctionBegin;
3126   outputCtx->numberofpoints        = 0;
3127   outputCtx->pointlist             = NULL;
3128   outputCtx->pointattributelist    = NULL;
3129   outputCtx->pointmarkerlist       = NULL;
3130   outputCtx->numberoftriangles     = 0;
3131   outputCtx->trianglelist          = NULL;
3132   outputCtx->triangleattributelist = NULL;
3133   outputCtx->neighborlist          = NULL;
3134   outputCtx->segmentlist           = NULL;
3135   outputCtx->segmentmarkerlist     = NULL;
3136   outputCtx->numberofedges         = 0;
3137   outputCtx->edgelist              = NULL;
3138   outputCtx->edgemarkerlist        = NULL;
3139   PetscFunctionReturn(0);
3140 }
3141 
3142 #undef __FUNCT__
3143 #define __FUNCT__ "FiniOutput_Triangle"
3144 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3145 {
3146   PetscFunctionBegin;
3147   free(outputCtx->pointmarkerlist);
3148   free(outputCtx->edgelist);
3149   free(outputCtx->edgemarkerlist);
3150   free(outputCtx->trianglelist);
3151   free(outputCtx->neighborlist);
3152   PetscFunctionReturn(0);
3153 }
3154 
3155 #undef __FUNCT__
3156 #define __FUNCT__ "DMPlexGenerate_Triangle"
3157 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3158 {
3159   MPI_Comm             comm;
3160   PetscInt             dim              = 2;
3161   const PetscBool      createConvexHull = PETSC_FALSE;
3162   const PetscBool      constrained      = PETSC_FALSE;
3163   struct triangulateio in;
3164   struct triangulateio out;
3165   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3166   PetscMPIInt          rank;
3167   PetscErrorCode       ierr;
3168 
3169   PetscFunctionBegin;
3170   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3171   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3172   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3173   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3174   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3175 
3176   in.numberofpoints = vEnd - vStart;
3177   if (in.numberofpoints > 0) {
3178     PetscSection coordSection;
3179     Vec          coordinates;
3180     PetscScalar *array;
3181 
3182     ierr = PetscMalloc(in.numberofpoints*dim * sizeof(double), &in.pointlist);CHKERRQ(ierr);
3183     ierr = PetscMalloc(in.numberofpoints * sizeof(int), &in.pointmarkerlist);CHKERRQ(ierr);
3184     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3185     ierr = DMPlexGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3186     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3187     for (v = vStart; v < vEnd; ++v) {
3188       const PetscInt idx = v - vStart;
3189       PetscInt       off, d;
3190 
3191       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3192       for (d = 0; d < dim; ++d) {
3193         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3194       }
3195       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3196     }
3197     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3198   }
3199   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3200   in.numberofsegments = eEnd - eStart;
3201   if (in.numberofsegments > 0) {
3202     ierr = PetscMalloc(in.numberofsegments*2 * sizeof(int), &in.segmentlist);CHKERRQ(ierr);
3203     ierr = PetscMalloc(in.numberofsegments   * sizeof(int), &in.segmentmarkerlist);CHKERRQ(ierr);
3204     for (e = eStart; e < eEnd; ++e) {
3205       const PetscInt  idx = e - eStart;
3206       const PetscInt *cone;
3207 
3208       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3209 
3210       in.segmentlist[idx*2+0] = cone[0] - vStart;
3211       in.segmentlist[idx*2+1] = cone[1] - vStart;
3212 
3213       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3214     }
3215   }
3216 #if 0 /* Do not currently support holes */
3217   PetscReal *holeCoords;
3218   PetscInt   h, d;
3219 
3220   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3221   if (in.numberofholes > 0) {
3222     ierr = PetscMalloc(in.numberofholes*dim * sizeof(double), &in.holelist);CHKERRQ(ierr);
3223     for (h = 0; h < in.numberofholes; ++h) {
3224       for (d = 0; d < dim; ++d) {
3225         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3226       }
3227     }
3228   }
3229 #endif
3230   if (!rank) {
3231     char args[32];
3232 
3233     /* Take away 'Q' for verbose output */
3234     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3235     if (createConvexHull) {
3236       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3237     }
3238     if (constrained) {
3239       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3240     }
3241     triangulate(args, &in, &out, NULL);
3242   }
3243   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3244   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3245   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3246   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3247   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3248 
3249   {
3250     const PetscInt numCorners  = 3;
3251     const PetscInt numCells    = out.numberoftriangles;
3252     const PetscInt numVertices = out.numberofpoints;
3253     const int     *cells      = out.trianglelist;
3254     const double  *meshCoords = out.pointlist;
3255 
3256     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3257     /* Set labels */
3258     for (v = 0; v < numVertices; ++v) {
3259       if (out.pointmarkerlist[v]) {
3260         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3261       }
3262     }
3263     if (interpolate) {
3264       for (e = 0; e < out.numberofedges; e++) {
3265         if (out.edgemarkerlist[e]) {
3266           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3267           const PetscInt *edges;
3268           PetscInt        numEdges;
3269 
3270           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3271           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3272           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3273           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3274         }
3275       }
3276     }
3277     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3278   }
3279 #if 0 /* Do not currently support holes */
3280   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3281 #endif
3282   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3283   PetscFunctionReturn(0);
3284 }
3285 
3286 #undef __FUNCT__
3287 #define __FUNCT__ "DMPlexRefine_Triangle"
3288 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3289 {
3290   MPI_Comm             comm;
3291   PetscInt             dim  = 2;
3292   struct triangulateio in;
3293   struct triangulateio out;
3294   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3295   PetscMPIInt          rank;
3296   PetscErrorCode       ierr;
3297 
3298   PetscFunctionBegin;
3299   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3300   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3301   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3302   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3303   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3304   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3305   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3306 
3307   in.numberofpoints = vEnd - vStart;
3308   if (in.numberofpoints > 0) {
3309     PetscSection coordSection;
3310     Vec          coordinates;
3311     PetscScalar *array;
3312 
3313     ierr = PetscMalloc(in.numberofpoints*dim * sizeof(double), &in.pointlist);CHKERRQ(ierr);
3314     ierr = PetscMalloc(in.numberofpoints * sizeof(int), &in.pointmarkerlist);CHKERRQ(ierr);
3315     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3316     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3317     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3318     for (v = vStart; v < vEnd; ++v) {
3319       const PetscInt idx = v - vStart;
3320       PetscInt       off, d;
3321 
3322       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3323       for (d = 0; d < dim; ++d) {
3324         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3325       }
3326       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3327     }
3328     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3329   }
3330   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3331 
3332   in.numberofcorners   = 3;
3333   in.numberoftriangles = cEnd - cStart;
3334 
3335   in.trianglearealist  = (double*) maxVolumes;
3336   if (in.numberoftriangles > 0) {
3337     ierr = PetscMalloc(in.numberoftriangles*in.numberofcorners * sizeof(int), &in.trianglelist);CHKERRQ(ierr);
3338     for (c = cStart; c < cEnd; ++c) {
3339       const PetscInt idx      = c - cStart;
3340       PetscInt      *closure = NULL;
3341       PetscInt       closureSize;
3342 
3343       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3344       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3345       for (v = 0; v < 3; ++v) {
3346         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3347       }
3348       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3349     }
3350   }
3351   /* TODO: Segment markers are missing on input */
3352 #if 0 /* Do not currently support holes */
3353   PetscReal *holeCoords;
3354   PetscInt   h, d;
3355 
3356   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3357   if (in.numberofholes > 0) {
3358     ierr = PetscMalloc(in.numberofholes*dim * sizeof(double), &in.holelist);CHKERRQ(ierr);
3359     for (h = 0; h < in.numberofholes; ++h) {
3360       for (d = 0; d < dim; ++d) {
3361         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3362       }
3363     }
3364   }
3365 #endif
3366   if (!rank) {
3367     char args[32];
3368 
3369     /* Take away 'Q' for verbose output */
3370     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3371     triangulate(args, &in, &out, NULL);
3372   }
3373   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3374   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3375   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3376   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3377   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3378 
3379   {
3380     const PetscInt numCorners  = 3;
3381     const PetscInt numCells    = out.numberoftriangles;
3382     const PetscInt numVertices = out.numberofpoints;
3383     const int     *cells      = out.trianglelist;
3384     const double  *meshCoords = out.pointlist;
3385     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3386 
3387     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3388     /* Set labels */
3389     for (v = 0; v < numVertices; ++v) {
3390       if (out.pointmarkerlist[v]) {
3391         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3392       }
3393     }
3394     if (interpolate) {
3395       PetscInt e;
3396 
3397       for (e = 0; e < out.numberofedges; e++) {
3398         if (out.edgemarkerlist[e]) {
3399           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3400           const PetscInt *edges;
3401           PetscInt        numEdges;
3402 
3403           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3404           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3405           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3406           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3407         }
3408       }
3409     }
3410     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3411   }
3412 #if 0 /* Do not currently support holes */
3413   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3414 #endif
3415   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3416   PetscFunctionReturn(0);
3417 }
3418 #endif
3419 
3420 #if defined(PETSC_HAVE_TETGEN)
3421 #include <tetgen.h>
3422 #undef __FUNCT__
3423 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3424 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3425 {
3426   MPI_Comm       comm;
3427   const PetscInt dim  = 3;
3428   ::tetgenio     in;
3429   ::tetgenio     out;
3430   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3431   PetscMPIInt    rank;
3432   PetscErrorCode ierr;
3433 
3434   PetscFunctionBegin;
3435   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3436   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3437   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3438   in.numberofpoints = vEnd - vStart;
3439   if (in.numberofpoints > 0) {
3440     PetscSection coordSection;
3441     Vec          coordinates;
3442     PetscScalar *array;
3443 
3444     in.pointlist       = new double[in.numberofpoints*dim];
3445     in.pointmarkerlist = new int[in.numberofpoints];
3446 
3447     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3448     ierr = DMPlexGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3449     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3450     for (v = vStart; v < vEnd; ++v) {
3451       const PetscInt idx = v - vStart;
3452       PetscInt       off, d;
3453 
3454       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3455       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3456       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3457     }
3458     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3459   }
3460   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3461 
3462   in.numberoffacets = fEnd - fStart;
3463   if (in.numberoffacets > 0) {
3464     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3465     in.facetmarkerlist = new int[in.numberoffacets];
3466     for (f = fStart; f < fEnd; ++f) {
3467       const PetscInt idx     = f - fStart;
3468       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3469 
3470       in.facetlist[idx].numberofpolygons = 1;
3471       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3472       in.facetlist[idx].numberofholes    = 0;
3473       in.facetlist[idx].holelist         = NULL;
3474 
3475       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3476       for (p = 0; p < numPoints*2; p += 2) {
3477         const PetscInt point = points[p];
3478         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3479       }
3480 
3481       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3482       poly->numberofvertices = numVertices;
3483       poly->vertexlist       = new int[poly->numberofvertices];
3484       for (v = 0; v < numVertices; ++v) {
3485         const PetscInt vIdx = points[v] - vStart;
3486         poly->vertexlist[v] = vIdx;
3487       }
3488       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3489       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3490     }
3491   }
3492   if (!rank) {
3493     char args[32];
3494 
3495     /* Take away 'Q' for verbose output */
3496     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3497     ::tetrahedralize(args, &in, &out);
3498   }
3499   {
3500     const PetscInt numCorners  = 4;
3501     const PetscInt numCells    = out.numberoftetrahedra;
3502     const PetscInt numVertices = out.numberofpoints;
3503     const double   *meshCoords = out.pointlist;
3504     int            *cells      = out.tetrahedronlist;
3505 
3506     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3507     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3508     /* Set labels */
3509     for (v = 0; v < numVertices; ++v) {
3510       if (out.pointmarkerlist[v]) {
3511         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3512       }
3513     }
3514     if (interpolate) {
3515       PetscInt e;
3516 
3517       for (e = 0; e < out.numberofedges; e++) {
3518         if (out.edgemarkerlist[e]) {
3519           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3520           const PetscInt *edges;
3521           PetscInt        numEdges;
3522 
3523           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3524           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3525           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3526           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3527         }
3528       }
3529       for (f = 0; f < out.numberoftrifaces; f++) {
3530         if (out.trifacemarkerlist[f]) {
3531           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3532           const PetscInt *faces;
3533           PetscInt        numFaces;
3534 
3535           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3536           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3537           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3538           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3539         }
3540       }
3541     }
3542     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3543   }
3544   PetscFunctionReturn(0);
3545 }
3546 
3547 #undef __FUNCT__
3548 #define __FUNCT__ "DMPlexRefine_Tetgen"
3549 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3550 {
3551   MPI_Comm       comm;
3552   const PetscInt dim  = 3;
3553   ::tetgenio     in;
3554   ::tetgenio     out;
3555   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3556   PetscMPIInt    rank;
3557   PetscErrorCode ierr;
3558 
3559   PetscFunctionBegin;
3560   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3561   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3562   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3563   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3564   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3565 
3566   in.numberofpoints = vEnd - vStart;
3567   if (in.numberofpoints > 0) {
3568     PetscSection coordSection;
3569     Vec          coordinates;
3570     PetscScalar *array;
3571 
3572     in.pointlist       = new double[in.numberofpoints*dim];
3573     in.pointmarkerlist = new int[in.numberofpoints];
3574 
3575     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3576     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3577     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3578     for (v = vStart; v < vEnd; ++v) {
3579       const PetscInt idx = v - vStart;
3580       PetscInt       off, d;
3581 
3582       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3583       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3584       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3585     }
3586     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3587   }
3588   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3589 
3590   in.numberofcorners       = 4;
3591   in.numberoftetrahedra    = cEnd - cStart;
3592   in.tetrahedronvolumelist = (double*) maxVolumes;
3593   if (in.numberoftetrahedra > 0) {
3594     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3595     for (c = cStart; c < cEnd; ++c) {
3596       const PetscInt idx      = c - cStart;
3597       PetscInt      *closure = NULL;
3598       PetscInt       closureSize;
3599 
3600       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3601       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3602       for (v = 0; v < 4; ++v) {
3603         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3604       }
3605       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3606     }
3607   }
3608   /* TODO: Put in boundary faces with markers */
3609   if (!rank) {
3610     char args[32];
3611 
3612     /* Take away 'Q' for verbose output */
3613     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3614     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3615     ::tetrahedralize(args, &in, &out);
3616   }
3617   in.tetrahedronvolumelist = NULL;
3618 
3619   {
3620     const PetscInt numCorners  = 4;
3621     const PetscInt numCells    = out.numberoftetrahedra;
3622     const PetscInt numVertices = out.numberofpoints;
3623     const double   *meshCoords = out.pointlist;
3624     int            *cells      = out.tetrahedronlist;
3625 
3626     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3627 
3628     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3629     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3630     /* Set labels */
3631     for (v = 0; v < numVertices; ++v) {
3632       if (out.pointmarkerlist[v]) {
3633         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3634       }
3635     }
3636     if (interpolate) {
3637       PetscInt e, f;
3638 
3639       for (e = 0; e < out.numberofedges; e++) {
3640         if (out.edgemarkerlist[e]) {
3641           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3642           const PetscInt *edges;
3643           PetscInt        numEdges;
3644 
3645           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3646           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3647           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3648           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3649         }
3650       }
3651       for (f = 0; f < out.numberoftrifaces; f++) {
3652         if (out.trifacemarkerlist[f]) {
3653           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3654           const PetscInt *faces;
3655           PetscInt        numFaces;
3656 
3657           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3658           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3659           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3660           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3661         }
3662       }
3663     }
3664     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3665   }
3666   PetscFunctionReturn(0);
3667 }
3668 #endif
3669 
3670 #if defined(PETSC_HAVE_CTETGEN)
3671 #include "ctetgen.h"
3672 
3673 #undef __FUNCT__
3674 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3675 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3676 {
3677   MPI_Comm       comm;
3678   const PetscInt dim  = 3;
3679   PLC           *in, *out;
3680   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3681   PetscMPIInt    rank;
3682   PetscErrorCode ierr;
3683 
3684   PetscFunctionBegin;
3685   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3686   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3687   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3688   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3689   ierr = PLCCreate(&in);CHKERRQ(ierr);
3690   ierr = PLCCreate(&out);CHKERRQ(ierr);
3691 
3692   in->numberofpoints = vEnd - vStart;
3693   if (in->numberofpoints > 0) {
3694     PetscSection coordSection;
3695     Vec          coordinates;
3696     PetscScalar *array;
3697 
3698     ierr = PetscMalloc(in->numberofpoints*dim * sizeof(PetscReal), &in->pointlist);CHKERRQ(ierr);
3699     ierr = PetscMalloc(in->numberofpoints     * sizeof(int),       &in->pointmarkerlist);CHKERRQ(ierr);
3700     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3701     ierr = DMPlexGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3702     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3703     for (v = vStart; v < vEnd; ++v) {
3704       const PetscInt idx = v - vStart;
3705       PetscInt       off, d, m;
3706 
3707       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3708       for (d = 0; d < dim; ++d) {
3709         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3710       }
3711       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3712 
3713       in->pointmarkerlist[idx] = (int) m;
3714     }
3715     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3716   }
3717   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3718 
3719   in->numberoffacets = fEnd - fStart;
3720   if (in->numberoffacets > 0) {
3721     ierr = PetscMalloc(in->numberoffacets * sizeof(facet), &in->facetlist);CHKERRQ(ierr);
3722     ierr = PetscMalloc(in->numberoffacets * sizeof(int),   &in->facetmarkerlist);CHKERRQ(ierr);
3723     for (f = fStart; f < fEnd; ++f) {
3724       const PetscInt idx     = f - fStart;
3725       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3726       polygon       *poly;
3727 
3728       in->facetlist[idx].numberofpolygons = 1;
3729 
3730       ierr = PetscMalloc(in->facetlist[idx].numberofpolygons * sizeof(polygon), &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3731 
3732       in->facetlist[idx].numberofholes    = 0;
3733       in->facetlist[idx].holelist         = NULL;
3734 
3735       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3736       for (p = 0; p < numPoints*2; p += 2) {
3737         const PetscInt point = points[p];
3738         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3739       }
3740 
3741       poly                   = in->facetlist[idx].polygonlist;
3742       poly->numberofvertices = numVertices;
3743       ierr                   = PetscMalloc(poly->numberofvertices * sizeof(int), &poly->vertexlist);CHKERRQ(ierr);
3744       for (v = 0; v < numVertices; ++v) {
3745         const PetscInt vIdx = points[v] - vStart;
3746         poly->vertexlist[v] = vIdx;
3747       }
3748       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3749       in->facetmarkerlist[idx] = (int) m;
3750       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3751     }
3752   }
3753   if (!rank) {
3754     TetGenOpts t;
3755 
3756     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3757     t.in        = boundary; /* Should go away */
3758     t.plc       = 1;
3759     t.quality   = 1;
3760     t.edgesout  = 1;
3761     t.zeroindex = 1;
3762     t.quiet     = 1;
3763     t.verbose   = verbose;
3764     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3765     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3766   }
3767   {
3768     const PetscInt numCorners  = 4;
3769     const PetscInt numCells    = out->numberoftetrahedra;
3770     const PetscInt numVertices = out->numberofpoints;
3771     const double   *meshCoords = out->pointlist;
3772     int            *cells      = out->tetrahedronlist;
3773 
3774     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3775     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3776     /* Set labels */
3777     for (v = 0; v < numVertices; ++v) {
3778       if (out->pointmarkerlist[v]) {
3779         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3780       }
3781     }
3782     if (interpolate) {
3783       PetscInt e;
3784 
3785       for (e = 0; e < out->numberofedges; e++) {
3786         if (out->edgemarkerlist[e]) {
3787           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3788           const PetscInt *edges;
3789           PetscInt        numEdges;
3790 
3791           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3792           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3793           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3794           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3795         }
3796       }
3797       for (f = 0; f < out->numberoftrifaces; f++) {
3798         if (out->trifacemarkerlist[f]) {
3799           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3800           const PetscInt *faces;
3801           PetscInt        numFaces;
3802 
3803           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3804           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3805           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3806           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3807         }
3808       }
3809     }
3810     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3811   }
3812 
3813   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3814   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3815   PetscFunctionReturn(0);
3816 }
3817 
3818 #undef __FUNCT__
3819 #define __FUNCT__ "DMPlexRefine_CTetgen"
3820 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3821 {
3822   MPI_Comm       comm;
3823   const PetscInt dim  = 3;
3824   PLC           *in, *out;
3825   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3826   PetscMPIInt    rank;
3827   PetscErrorCode ierr;
3828 
3829   PetscFunctionBegin;
3830   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3831   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3832   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3833   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3834   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3835   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3836   ierr = PLCCreate(&in);CHKERRQ(ierr);
3837   ierr = PLCCreate(&out);CHKERRQ(ierr);
3838 
3839   in->numberofpoints = vEnd - vStart;
3840   if (in->numberofpoints > 0) {
3841     PetscSection coordSection;
3842     Vec          coordinates;
3843     PetscScalar *array;
3844 
3845     ierr = PetscMalloc(in->numberofpoints*dim * sizeof(PetscReal), &in->pointlist);CHKERRQ(ierr);
3846     ierr = PetscMalloc(in->numberofpoints     * sizeof(int),       &in->pointmarkerlist);CHKERRQ(ierr);
3847     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3848     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3849     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3850     for (v = vStart; v < vEnd; ++v) {
3851       const PetscInt idx = v - vStart;
3852       PetscInt       off, d, m;
3853 
3854       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3855       for (d = 0; d < dim; ++d) {
3856         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3857       }
3858       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3859 
3860       in->pointmarkerlist[idx] = (int) m;
3861     }
3862     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3863   }
3864   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3865 
3866   in->numberofcorners       = 4;
3867   in->numberoftetrahedra    = cEnd - cStart;
3868   in->tetrahedronvolumelist = maxVolumes;
3869   if (in->numberoftetrahedra > 0) {
3870     ierr = PetscMalloc(in->numberoftetrahedra*in->numberofcorners * sizeof(int), &in->tetrahedronlist);CHKERRQ(ierr);
3871     for (c = cStart; c < cEnd; ++c) {
3872       const PetscInt idx      = c - cStart;
3873       PetscInt      *closure = NULL;
3874       PetscInt       closureSize;
3875 
3876       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3877       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3878       for (v = 0; v < 4; ++v) {
3879         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3880       }
3881       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3882     }
3883   }
3884   if (!rank) {
3885     TetGenOpts t;
3886 
3887     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3888 
3889     t.in        = dm; /* Should go away */
3890     t.refine    = 1;
3891     t.varvolume = 1;
3892     t.quality   = 1;
3893     t.edgesout  = 1;
3894     t.zeroindex = 1;
3895     t.quiet     = 1;
3896     t.verbose   = verbose; /* Change this */
3897 
3898     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3899     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3900   }
3901   {
3902     const PetscInt numCorners  = 4;
3903     const PetscInt numCells    = out->numberoftetrahedra;
3904     const PetscInt numVertices = out->numberofpoints;
3905     const double   *meshCoords = out->pointlist;
3906     int            *cells      = out->tetrahedronlist;
3907     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3908 
3909     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3910     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3911     /* Set labels */
3912     for (v = 0; v < numVertices; ++v) {
3913       if (out->pointmarkerlist[v]) {
3914         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3915       }
3916     }
3917     if (interpolate) {
3918       PetscInt e, f;
3919 
3920       for (e = 0; e < out->numberofedges; e++) {
3921         if (out->edgemarkerlist[e]) {
3922           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3923           const PetscInt *edges;
3924           PetscInt        numEdges;
3925 
3926           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3927           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3928           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3929           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3930         }
3931       }
3932       for (f = 0; f < out->numberoftrifaces; f++) {
3933         if (out->trifacemarkerlist[f]) {
3934           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3935           const PetscInt *faces;
3936           PetscInt        numFaces;
3937 
3938           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3939           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3940           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3941           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3942         }
3943       }
3944     }
3945     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3946   }
3947   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3948   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3949   PetscFunctionReturn(0);
3950 }
3951 #endif
3952 
3953 #undef __FUNCT__
3954 #define __FUNCT__ "DMPlexGenerate"
3955 /*@C
3956   DMPlexGenerate - Generates a mesh.
3957 
3958   Not Collective
3959 
3960   Input Parameters:
3961 + boundary - The DMPlex boundary object
3962 . name - The mesh generation package name
3963 - interpolate - Flag to create intermediate mesh elements
3964 
3965   Output Parameter:
3966 . mesh - The DMPlex object
3967 
3968   Level: intermediate
3969 
3970 .keywords: mesh, elements
3971 .seealso: DMPlexCreate(), DMRefine()
3972 @*/
3973 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
3974 {
3975   PetscInt       dim;
3976   char           genname[1024];
3977   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
3978   PetscErrorCode ierr;
3979 
3980   PetscFunctionBegin;
3981   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
3982   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
3983   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
3984   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
3985   if (flg) name = genname;
3986   if (name) {
3987     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
3988     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
3989     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
3990   }
3991   switch (dim) {
3992   case 1:
3993     if (!name || isTriangle) {
3994 #if defined(PETSC_HAVE_TRIANGLE)
3995       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
3996 #else
3997       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
3998 #endif
3999     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
4000     break;
4001   case 2:
4002     if (!name || isCTetgen) {
4003 #if defined(PETSC_HAVE_CTETGEN)
4004       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4005 #else
4006       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4007 #endif
4008     } else if (isTetgen) {
4009 #if defined(PETSC_HAVE_TETGEN)
4010       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4011 #else
4012       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4013 #endif
4014     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4015     break;
4016   default:
4017     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4018   }
4019   PetscFunctionReturn(0);
4020 }
4021 
4022 typedef PetscInt CellRefiner;
4023 
4024 #undef __FUNCT__
4025 #define __FUNCT__ "GetDepthStart_Private"
4026 PETSC_STATIC_INLINE PetscErrorCode GetDepthStart_Private(PetscInt depth, PetscInt depthSize[], PetscInt *cStart, PetscInt *fStart, PetscInt *eStart, PetscInt *vStart)
4027 {
4028   PetscFunctionBegin;
4029   if (cStart) *cStart = 0;
4030   if (vStart) *vStart = depthSize[depth];
4031   if (fStart) *fStart = depthSize[depth] + depthSize[0];
4032   if (eStart) *eStart = depthSize[depth] + depthSize[0] + depthSize[depth-1];
4033   PetscFunctionReturn(0);
4034 }
4035 
4036 #undef __FUNCT__
4037 #define __FUNCT__ "GetDepthEnd_Private"
4038 PETSC_STATIC_INLINE PetscErrorCode GetDepthEnd_Private(PetscInt depth, PetscInt depthSize[], PetscInt *cEnd, PetscInt *fEnd, PetscInt *eEnd, PetscInt *vEnd)
4039 {
4040   PetscFunctionBegin;
4041   if (cEnd) *cEnd = depthSize[depth];
4042   if (vEnd) *vEnd = depthSize[depth] + depthSize[0];
4043   if (fEnd) *fEnd = depthSize[depth] + depthSize[0] + depthSize[depth-1];
4044   if (eEnd) *eEnd = depthSize[depth] + depthSize[0] + depthSize[depth-1] + depthSize[1];
4045   PetscFunctionReturn(0);
4046 }
4047 
4048 #undef __FUNCT__
4049 #define __FUNCT__ "CellRefinerGetSizes"
4050 PetscErrorCode CellRefinerGetSizes(CellRefiner refiner, DM dm, PetscInt depthSize[])
4051 {
4052   PetscInt       cStart, cEnd, cMax, vStart, vEnd, vMax, fStart, fEnd, fMax, eStart, eEnd, eMax;
4053   PetscErrorCode ierr;
4054 
4055   PetscFunctionBegin;
4056   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4057   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
4058   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4059   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
4060   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
4061   switch (refiner) {
4062   case 1:
4063     /* Simplicial 2D */
4064     depthSize[0] = vEnd - vStart + fEnd - fStart;         /* Add a vertex on every face */
4065     depthSize[1] = 2*(fEnd - fStart) + 3*(cEnd - cStart); /* Every face is split into 2 faces and 3 faces are added for each cell */
4066     depthSize[2] = 4*(cEnd - cStart);                     /* Every cell split into 4 cells */
4067     break;
4068   case 3:
4069     /* Hybrid 2D */
4070     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
4071     cMax = PetscMin(cEnd, cMax);
4072     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
4073     fMax         = PetscMin(fEnd, fMax);
4074     depthSize[0] = vEnd - vStart + fMax - fStart;                                         /* Add a vertex on every face, but not hybrid faces */
4075     depthSize[1] = 2*(fMax - fStart) + 3*(cMax - cStart) + (fEnd - fMax) + (cEnd - cMax); /* Every interior face is split into 2 faces, 3 faces are added for each interior cell, and one in each hybrid cell */
4076     depthSize[2] = 4*(cMax - cStart) + 2*(cEnd - cMax);                                   /* Interior cells split into 4 cells, Hybrid cells split into 2 cells */
4077     break;
4078   case 2:
4079     /* Hex 2D */
4080     depthSize[0] = vEnd - vStart + cEnd - cStart + fEnd - fStart; /* Add a vertex on every face and cell */
4081     depthSize[1] = 2*(fEnd - fStart) + 4*(cEnd - cStart);         /* Every face is split into 2 faces and 4 faces are added for each cell */
4082     depthSize[2] = 4*(cEnd - cStart);                             /* Every cell split into 4 cells */
4083     break;
4084   default:
4085     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
4086   }
4087   PetscFunctionReturn(0);
4088 }
4089 
4090 #undef __FUNCT__
4091 #define __FUNCT__ "CellRefinerSetConeSizes"
4092 PetscErrorCode CellRefinerSetConeSizes(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
4093 {
4094   PetscInt       depth, cStart, cStartNew, cEnd, cMax, c, vStart, vStartNew, vEnd, vMax, v, fStart, fStartNew, fEnd, fMax, f, eStart, eStartNew, eEnd, eMax, r;
4095   PetscErrorCode ierr;
4096 
4097   PetscFunctionBegin;
4098   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4099   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4100   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
4101   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4102   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
4103   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
4104   ierr = GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);CHKERRQ(ierr);
4105   switch (refiner) {
4106   case 1:
4107     /* Simplicial 2D */
4108     /* All cells have 3 faces */
4109     for (c = cStart; c < cEnd; ++c) {
4110       for (r = 0; r < 4; ++r) {
4111         const PetscInt newp = (c - cStart)*4 + r;
4112 
4113         ierr = DMPlexSetConeSize(rdm, newp, 3);CHKERRQ(ierr);
4114       }
4115     }
4116     /* Split faces have 2 vertices and the same cells as the parent */
4117     for (f = fStart; f < fEnd; ++f) {
4118       for (r = 0; r < 2; ++r) {
4119         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
4120         PetscInt       size;
4121 
4122         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4123         ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4124         ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4125       }
4126     }
4127     /* Interior faces have 2 vertices and 2 cells */
4128     for (c = cStart; c < cEnd; ++c) {
4129       for (r = 0; r < 3; ++r) {
4130         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*3 + r;
4131 
4132         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4133         ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4134       }
4135     }
4136     /* Old vertices have identical supports */
4137     for (v = vStart; v < vEnd; ++v) {
4138       const PetscInt newp = vStartNew + (v - vStart);
4139       PetscInt       size;
4140 
4141       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4142       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4143     }
4144     /* Face vertices have 2 + cells*2 supports */
4145     for (f = fStart; f < fEnd; ++f) {
4146       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
4147       PetscInt       size;
4148 
4149       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4150       ierr = DMPlexSetSupportSize(rdm, newp, 2 + size*2);CHKERRQ(ierr);
4151     }
4152     break;
4153   case 2:
4154     /* Hex 2D */
4155     /* All cells have 4 faces */
4156     for (c = cStart; c < cEnd; ++c) {
4157       for (r = 0; r < 4; ++r) {
4158         const PetscInt newp = (c - cStart)*4 + r;
4159 
4160         ierr = DMPlexSetConeSize(rdm, newp, 4);CHKERRQ(ierr);
4161       }
4162     }
4163     /* Split faces have 2 vertices and the same cells as the parent */
4164     for (f = fStart; f < fEnd; ++f) {
4165       for (r = 0; r < 2; ++r) {
4166         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
4167         PetscInt       size;
4168 
4169         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4170         ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4171         ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4172       }
4173     }
4174     /* Interior faces have 2 vertices and 2 cells */
4175     for (c = cStart; c < cEnd; ++c) {
4176       for (r = 0; r < 4; ++r) {
4177         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
4178 
4179         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4180         ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4181       }
4182     }
4183     /* Old vertices have identical supports */
4184     for (v = vStart; v < vEnd; ++v) {
4185       const PetscInt newp = vStartNew + (v - vStart);
4186       PetscInt       size;
4187 
4188       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4189       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4190     }
4191     /* Face vertices have 2 + cells supports */
4192     for (f = fStart; f < fEnd; ++f) {
4193       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
4194       PetscInt       size;
4195 
4196       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4197       ierr = DMPlexSetSupportSize(rdm, newp, 2 + size);CHKERRQ(ierr);
4198     }
4199     /* Cell vertices have 4 supports */
4200     for (c = cStart; c < cEnd; ++c) {
4201       const PetscInt newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
4202 
4203       ierr = DMPlexSetSupportSize(rdm, newp, 4);CHKERRQ(ierr);
4204     }
4205     break;
4206   case 3:
4207     /* Hybrid 2D */
4208     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
4209     cMax = PetscMin(cEnd, cMax);
4210     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
4211     fMax = PetscMin(fEnd, fMax);
4212     ierr = DMPlexSetHybridBounds(rdm, cStartNew + (cMax - cStart)*4, fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
4213     /* Interior cells have 3 faces */
4214     for (c = cStart; c < cMax; ++c) {
4215       for (r = 0; r < 4; ++r) {
4216         const PetscInt newp = cStartNew + (c - cStart)*4 + r;
4217 
4218         ierr = DMPlexSetConeSize(rdm, newp, 3);CHKERRQ(ierr);
4219       }
4220     }
4221     /* Hybrid cells have 4 faces */
4222     for (c = cMax; c < cEnd; ++c) {
4223       for (r = 0; r < 2; ++r) {
4224         const PetscInt newp = cStartNew + (cMax - cStart)*4 + (c - cMax)*2 + r;
4225 
4226         ierr = DMPlexSetConeSize(rdm, newp, 4);CHKERRQ(ierr);
4227       }
4228     }
4229     /* Interior split faces have 2 vertices and the same cells as the parent */
4230     for (f = fStart; f < fMax; ++f) {
4231       for (r = 0; r < 2; ++r) {
4232         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
4233         PetscInt       size;
4234 
4235         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4236         ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4237         ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4238       }
4239     }
4240     /* Interior cell faces have 2 vertices and 2 cells */
4241     for (c = cStart; c < cMax; ++c) {
4242       for (r = 0; r < 3; ++r) {
4243         const PetscInt newp = fStartNew + (fMax - fStart)*2 + (c - cStart)*3 + r;
4244 
4245         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4246         ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4247       }
4248     }
4249     /* Hybrid faces have 2 vertices and the same cells */
4250     for (f = fMax; f < fEnd; ++f) {
4251       const PetscInt newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (f - fMax);
4252       PetscInt       size;
4253 
4254       ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4255       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4256       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4257     }
4258     /* Hybrid cell faces have 2 vertices and 2 cells */
4259     for (c = cMax; c < cEnd; ++c) {
4260       const PetscInt newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (c - cMax);
4261 
4262       ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4263       ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4264     }
4265     /* Old vertices have identical supports */
4266     for (v = vStart; v < vEnd; ++v) {
4267       const PetscInt newp = vStartNew + (v - vStart);
4268       PetscInt       size;
4269 
4270       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4271       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4272     }
4273     /* Face vertices have 2 + (2 interior, 1 hybrid) supports */
4274     for (f = fStart; f < fMax; ++f) {
4275       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
4276       const PetscInt *support;
4277       PetscInt       size, newSize = 2, s;
4278 
4279       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4280       ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4281       for (s = 0; s < size; ++s) {
4282         if (support[s] >= cMax) newSize += 1;
4283         else newSize += 2;
4284       }
4285       ierr = DMPlexSetSupportSize(rdm, newp, newSize);CHKERRQ(ierr);
4286     }
4287     break;
4288   default:
4289     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
4290   }
4291   PetscFunctionReturn(0);
4292 }
4293 
4294 #undef __FUNCT__
4295 #define __FUNCT__ "CellRefinerSetCones"
4296 PetscErrorCode CellRefinerSetCones(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
4297 {
4298   PetscInt       depth, cStart, cEnd, cMax, cStartNew, cEndNew, c, vStart, vEnd, vMax, vStartNew, vEndNew, v, fStart, fEnd, fMax, fStartNew, fEndNew, f, eStart, eEnd, eMax, eStartNew, eEndNew, r, p;
4299   PetscInt       maxSupportSize, *supportRef;
4300   PetscErrorCode ierr;
4301 
4302   PetscFunctionBegin;
4303   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4304   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4305   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
4306   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4307   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
4308   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
4309   ierr = GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);CHKERRQ(ierr);
4310   ierr = GetDepthEnd_Private(depth, depthSize, &cEndNew, &fEndNew, &eEndNew, &vEndNew);CHKERRQ(ierr);
4311   switch (refiner) {
4312   case 1:
4313     /* Simplicial 2D */
4314     /*
4315      2
4316      |\
4317      | \
4318      |  \
4319      |   \
4320      | C  \
4321      |     \
4322      |      \
4323      2---1---1
4324      |\  D  / \
4325      | 2   0   \
4326      |A \ /  B  \
4327      0---0-------1
4328      */
4329     /* All cells have 3 faces */
4330     for (c = cStart; c < cEnd; ++c) {
4331       const PetscInt  newp = cStartNew + (c - cStart)*4;
4332       const PetscInt *cone, *ornt;
4333       PetscInt        coneNew[3], orntNew[3];
4334 
4335       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4336       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4337       /* A triangle */
4338       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4339       orntNew[0] = ornt[0];
4340       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 2;
4341       orntNew[1] = -2;
4342       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
4343       orntNew[2] = ornt[2];
4344       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4345       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4346 #if 1
4347       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4348       for (p = 0; p < 3; ++p) {
4349         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4350       }
4351 #endif
4352       /* B triangle */
4353       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4354       orntNew[0] = ornt[0];
4355       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4356       orntNew[1] = ornt[1];
4357       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 0;
4358       orntNew[2] = -2;
4359       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4360       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4361 #if 1
4362       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4363       for (p = 0; p < 3; ++p) {
4364         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4365       }
4366 #endif
4367       /* C triangle */
4368       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 1;
4369       orntNew[0] = -2;
4370       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4371       orntNew[1] = ornt[1];
4372       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
4373       orntNew[2] = ornt[2];
4374       ierr       = DMPlexSetCone(rdm, newp+2, coneNew);CHKERRQ(ierr);
4375       ierr       = DMPlexSetConeOrientation(rdm, newp+2, orntNew);CHKERRQ(ierr);
4376 #if 1
4377       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
4378       for (p = 0; p < 3; ++p) {
4379         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4380       }
4381 #endif
4382       /* D triangle */
4383       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 0;
4384       orntNew[0] = 0;
4385       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 1;
4386       orntNew[1] = 0;
4387       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 2;
4388       orntNew[2] = 0;
4389       ierr       = DMPlexSetCone(rdm, newp+3, coneNew);CHKERRQ(ierr);
4390       ierr       = DMPlexSetConeOrientation(rdm, newp+3, orntNew);CHKERRQ(ierr);
4391 #if 1
4392       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
4393       for (p = 0; p < 3; ++p) {
4394         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4395       }
4396 #endif
4397     }
4398     /* Split faces have 2 vertices and the same cells as the parent */
4399     ierr = DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);CHKERRQ(ierr);
4400     ierr = PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);CHKERRQ(ierr);
4401     for (f = fStart; f < fEnd; ++f) {
4402       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);
4403 
4404       for (r = 0; r < 2; ++r) {
4405         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
4406         const PetscInt *cone, *support;
4407         PetscInt        coneNew[2], coneSize, c, supportSize, s;
4408 
4409         ierr             = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4410         coneNew[0]       = vStartNew + (cone[0] - vStart);
4411         coneNew[1]       = vStartNew + (cone[1] - vStart);
4412         coneNew[(r+1)%2] = newv;
4413         ierr             = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4414 #if 1
4415         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4416         for (p = 0; p < 2; ++p) {
4417           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4418         }
4419 #endif
4420         ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
4421         ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4422         for (s = 0; s < supportSize; ++s) {
4423           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
4424           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4425           for (c = 0; c < coneSize; ++c) {
4426             if (cone[c] == f) break;
4427           }
4428           supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%3;
4429         }
4430         ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4431 #if 1
4432         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4433         for (p = 0; p < supportSize; ++p) {
4434           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
4435         }
4436 #endif
4437       }
4438     }
4439     /* Interior faces have 2 vertices and 2 cells */
4440     for (c = cStart; c < cEnd; ++c) {
4441       const PetscInt *cone;
4442 
4443       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4444       for (r = 0; r < 3; ++r) {
4445         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*3 + r;
4446         PetscInt       coneNew[2];
4447         PetscInt       supportNew[2];
4448 
4449         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r]       - fStart);
4450         coneNew[1] = vStartNew + (vEnd - vStart) + (cone[(r+1)%3] - fStart);
4451         ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4452 #if 1
4453         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4454         for (p = 0; p < 2; ++p) {
4455           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4456         }
4457 #endif
4458         supportNew[0] = (c - cStart)*4 + (r+1)%3;
4459         supportNew[1] = (c - cStart)*4 + 3;
4460         ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4461 #if 1
4462         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4463         for (p = 0; p < 2; ++p) {
4464           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4465         }
4466 #endif
4467       }
4468     }
4469     /* Old vertices have identical supports */
4470     for (v = vStart; v < vEnd; ++v) {
4471       const PetscInt  newp = vStartNew + (v - vStart);
4472       const PetscInt *support, *cone;
4473       PetscInt        size, s;
4474 
4475       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4476       ierr = DMPlexGetSupport(dm, v, &support);CHKERRQ(ierr);
4477       for (s = 0; s < size; ++s) {
4478         PetscInt r = 0;
4479 
4480         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4481         if (cone[1] == v) r = 1;
4482         supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
4483       }
4484       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4485 #if 1
4486       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4487       for (p = 0; p < size; ++p) {
4488         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4489       }
4490 #endif
4491     }
4492     /* Face vertices have 2 + cells*2 supports */
4493     for (f = fStart; f < fEnd; ++f) {
4494       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
4495       const PetscInt *cone, *support;
4496       PetscInt        size, s;
4497 
4498       ierr          = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4499       ierr          = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4500       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
4501       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
4502       for (s = 0; s < size; ++s) {
4503         PetscInt r = 0;
4504 
4505         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4506         if      (cone[1] == f) r = 1;
4507         else if (cone[2] == f) r = 2;
4508         supportRef[2+s*2+0] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*3 + (r+2)%3;
4509         supportRef[2+s*2+1] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*3 + r;
4510       }
4511       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4512 #if 1
4513       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4514       for (p = 0; p < 2+size*2; ++p) {
4515         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4516       }
4517 #endif
4518     }
4519     ierr = PetscFree(supportRef);CHKERRQ(ierr);
4520     break;
4521   case 2:
4522     /* Hex 2D */
4523     /*
4524      3---------2---------2
4525      |         |         |
4526      |    D    2    C    |
4527      |         |         |
4528      3----3----0----1----1
4529      |         |         |
4530      |    A    0    B    |
4531      |         |         |
4532      0---------0---------1
4533      */
4534     /* All cells have 4 faces */
4535     for (c = cStart; c < cEnd; ++c) {
4536       const PetscInt  newp = (c - cStart)*4;
4537       const PetscInt *cone, *ornt;
4538       PetscInt        coneNew[4], orntNew[4];
4539 
4540       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4541       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4542       /* A quad */
4543       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4544       orntNew[0] = ornt[0];
4545       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 0;
4546       orntNew[1] = 0;
4547       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 3;
4548       orntNew[2] = -2;
4549       coneNew[3] = fStartNew + (cone[3] - fStart)*2 + (ornt[3] < 0 ? 0 : 1);
4550       orntNew[3] = ornt[3];
4551       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4552       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4553 #if 1
4554       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4555       for (p = 0; p < 4; ++p) {
4556         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4557       }
4558 #endif
4559       /* B quad */
4560       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4561       orntNew[0] = ornt[0];
4562       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4563       orntNew[1] = ornt[1];
4564       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 1;
4565       orntNew[2] = 0;
4566       coneNew[3] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 0;
4567       orntNew[3] = -2;
4568       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4569       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4570 #if 1
4571       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4572       for (p = 0; p < 4; ++p) {
4573         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4574       }
4575 #endif
4576       /* C quad */
4577       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 1;
4578       orntNew[0] = -2;
4579       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4580       orntNew[1] = ornt[1];
4581       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
4582       orntNew[2] = ornt[2];
4583       coneNew[3] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 2;
4584       orntNew[3] = 0;
4585       ierr       = DMPlexSetCone(rdm, newp+2, coneNew);CHKERRQ(ierr);
4586       ierr       = DMPlexSetConeOrientation(rdm, newp+2, orntNew);CHKERRQ(ierr);
4587 #if 1
4588       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
4589       for (p = 0; p < 4; ++p) {
4590         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4591       }
4592 #endif
4593       /* D quad */
4594       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 3;
4595       orntNew[0] = 0;
4596       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 2;
4597       orntNew[1] = -2;
4598       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
4599       orntNew[2] = ornt[2];
4600       coneNew[3] = fStartNew + (cone[3] - fStart)*2 + (ornt[3] < 0 ? 1 : 0);
4601       orntNew[3] = ornt[3];
4602       ierr       = DMPlexSetCone(rdm, newp+3, coneNew);CHKERRQ(ierr);
4603       ierr       = DMPlexSetConeOrientation(rdm, newp+3, orntNew);CHKERRQ(ierr);
4604 #if 1
4605       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
4606       for (p = 0; p < 4; ++p) {
4607         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4608       }
4609 #endif
4610     }
4611     /* Split faces have 2 vertices and the same cells as the parent */
4612     ierr = DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);CHKERRQ(ierr);
4613     ierr = PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);CHKERRQ(ierr);
4614     for (f = fStart; f < fEnd; ++f) {
4615       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);
4616 
4617       for (r = 0; r < 2; ++r) {
4618         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
4619         const PetscInt *cone, *support;
4620         PetscInt        coneNew[2], coneSize, c, supportSize, s;
4621 
4622         ierr             = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4623         coneNew[0]       = vStartNew + (cone[0] - vStart);
4624         coneNew[1]       = vStartNew + (cone[1] - vStart);
4625         coneNew[(r+1)%2] = newv;
4626         ierr             = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4627 #if 1
4628         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4629         for (p = 0; p < 2; ++p) {
4630           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4631         }
4632 #endif
4633         ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
4634         ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4635         for (s = 0; s < supportSize; ++s) {
4636           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
4637           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4638           for (c = 0; c < coneSize; ++c) {
4639             if (cone[c] == f) break;
4640           }
4641           supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%4;
4642         }
4643         ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4644 #if 1
4645         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4646         for (p = 0; p < supportSize; ++p) {
4647           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
4648         }
4649 #endif
4650       }
4651     }
4652     /* Interior faces have 2 vertices and 2 cells */
4653     for (c = cStart; c < cEnd; ++c) {
4654       const PetscInt *cone;
4655       PetscInt        coneNew[2], supportNew[2];
4656 
4657       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4658       for (r = 0; r < 4; ++r) {
4659         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
4660 
4661         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r] - fStart);
4662         coneNew[1] = vStartNew + (vEnd - vStart) + (fEnd    - fStart) + (c - cStart);
4663         ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4664 #if 1
4665         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4666         for (p = 0; p < 2; ++p) {
4667           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4668         }
4669 #endif
4670         supportNew[0] = (c - cStart)*4 + r;
4671         supportNew[1] = (c - cStart)*4 + (r+1)%4;
4672         ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4673 #if 1
4674         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4675         for (p = 0; p < 2; ++p) {
4676           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4677         }
4678 #endif
4679       }
4680     }
4681     /* Old vertices have identical supports */
4682     for (v = vStart; v < vEnd; ++v) {
4683       const PetscInt  newp = vStartNew + (v - vStart);
4684       const PetscInt *support, *cone;
4685       PetscInt        size, s;
4686 
4687       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4688       ierr = DMPlexGetSupport(dm, v, &support);CHKERRQ(ierr);
4689       for (s = 0; s < size; ++s) {
4690         PetscInt r = 0;
4691 
4692         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4693         if (cone[1] == v) r = 1;
4694         supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
4695       }
4696       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4697 #if 1
4698       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4699       for (p = 0; p < size; ++p) {
4700         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4701       }
4702 #endif
4703     }
4704     /* Face vertices have 2 + cells supports */
4705     for (f = fStart; f < fEnd; ++f) {
4706       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
4707       const PetscInt *cone, *support;
4708       PetscInt        size, s;
4709 
4710       ierr          = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4711       ierr          = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4712       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
4713       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
4714       for (s = 0; s < size; ++s) {
4715         PetscInt r = 0;
4716 
4717         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4718         if      (cone[1] == f) r = 1;
4719         else if (cone[2] == f) r = 2;
4720         else if (cone[3] == f) r = 3;
4721         supportRef[2+s] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*4 + r;
4722       }
4723       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4724 #if 1
4725       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4726       for (p = 0; p < 2+size; ++p) {
4727         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4728       }
4729 #endif
4730     }
4731     /* Cell vertices have 4 supports */
4732     for (c = cStart; c < cEnd; ++c) {
4733       const PetscInt newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
4734       PetscInt       supportNew[4];
4735 
4736       for (r = 0; r < 4; ++r) {
4737         supportNew[r] = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
4738       }
4739       ierr = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4740     }
4741     break;
4742   case 3:
4743     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
4744     cMax = PetscMin(cEnd, cMax);
4745     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
4746     fMax = PetscMin(fEnd, fMax);
4747     /* Interior cells have 3 faces */
4748     for (c = cStart; c < cMax; ++c) {
4749       const PetscInt  newp = cStartNew + (c - cStart)*4;
4750       const PetscInt *cone, *ornt;
4751       PetscInt        coneNew[3], orntNew[3];
4752 
4753       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4754       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4755       /* A triangle */
4756       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4757       orntNew[0] = ornt[0];
4758       coneNew[1] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 2;
4759       orntNew[1] = -2;
4760       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
4761       orntNew[2] = ornt[2];
4762       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4763       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4764 #if 1
4765       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4766       for (p = 0; p < 3; ++p) {
4767         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4768       }
4769 #endif
4770       /* B triangle */
4771       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4772       orntNew[0] = ornt[0];
4773       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4774       orntNew[1] = ornt[1];
4775       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 0;
4776       orntNew[2] = -2;
4777       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4778       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4779 #if 1
4780       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4781       for (p = 0; p < 3; ++p) {
4782         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4783       }
4784 #endif
4785       /* C triangle */
4786       coneNew[0] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 1;
4787       orntNew[0] = -2;
4788       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4789       orntNew[1] = ornt[1];
4790       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
4791       orntNew[2] = ornt[2];
4792       ierr       = DMPlexSetCone(rdm, newp+2, coneNew);CHKERRQ(ierr);
4793       ierr       = DMPlexSetConeOrientation(rdm, newp+2, orntNew);CHKERRQ(ierr);
4794 #if 1
4795       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
4796       for (p = 0; p < 3; ++p) {
4797         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4798       }
4799 #endif
4800       /* D triangle */
4801       coneNew[0] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 0;
4802       orntNew[0] = 0;
4803       coneNew[1] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 1;
4804       orntNew[1] = 0;
4805       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 2;
4806       orntNew[2] = 0;
4807       ierr       = DMPlexSetCone(rdm, newp+3, coneNew);CHKERRQ(ierr);
4808       ierr       = DMPlexSetConeOrientation(rdm, newp+3, orntNew);CHKERRQ(ierr);
4809 #if 1
4810       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
4811       for (p = 0; p < 3; ++p) {
4812         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4813       }
4814 #endif
4815     }
4816     /*
4817      2----3----3
4818      |         |
4819      |    B    |
4820      |         |
4821      0----4--- 1
4822      |         |
4823      |    A    |
4824      |         |
4825      0----2----1
4826      */
4827     /* Hybrid cells have 4 faces */
4828     for (c = cMax; c < cEnd; ++c) {
4829       const PetscInt  newp = cStartNew + (cMax - cStart)*4 + (c - cMax)*2;
4830       const PetscInt *cone, *ornt;
4831       PetscInt        coneNew[4], orntNew[4];
4832 
4833       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4834       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4835       /* A quad */
4836       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4837       orntNew[0] = ornt[0];
4838       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4839       orntNew[1] = ornt[1];
4840       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (cone[2] - fMax);
4841       orntNew[2] = 0;
4842       coneNew[3] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (fEnd    - fMax) + (c - cMax);
4843       orntNew[3] = 0;
4844       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4845       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4846 #if 1
4847       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4848       for (p = 0; p < 4; ++p) {
4849         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4850       }
4851 #endif
4852       /* B quad */
4853       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4854       orntNew[0] = ornt[0];
4855       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4856       orntNew[1] = ornt[1];
4857       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (fEnd    - fMax) + (c - cMax);
4858       orntNew[2] = 0;
4859       coneNew[3] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (cone[3] - fMax);
4860       orntNew[3] = 0;
4861       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4862       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4863 #if 1
4864       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4865       for (p = 0; p < 4; ++p) {
4866         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4867       }
4868 #endif
4869     }
4870     /* Interior split faces have 2 vertices and the same cells as the parent */
4871     ierr = DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);CHKERRQ(ierr);
4872     ierr = PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);CHKERRQ(ierr);
4873     for (f = fStart; f < fMax; ++f) {
4874       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);
4875 
4876       for (r = 0; r < 2; ++r) {
4877         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
4878         const PetscInt *cone, *support;
4879         PetscInt        coneNew[2], coneSize, c, supportSize, s;
4880 
4881         ierr             = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4882         coneNew[0]       = vStartNew + (cone[0] - vStart);
4883         coneNew[1]       = vStartNew + (cone[1] - vStart);
4884         coneNew[(r+1)%2] = newv;
4885         ierr             = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4886 #if 1
4887         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4888         for (p = 0; p < 2; ++p) {
4889           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4890         }
4891 #endif
4892         ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
4893         ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4894         for (s = 0; s < supportSize; ++s) {
4895           if (support[s] >= cMax) {
4896             supportRef[s] = cStartNew + (cMax - cStart)*4 + (support[s] - cMax)*2 + r;
4897           } else {
4898             ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
4899             ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4900             for (c = 0; c < coneSize; ++c) {
4901               if (cone[c] == f) break;
4902             }
4903             supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%3;
4904           }
4905         }
4906         ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4907 #if 1
4908         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4909         for (p = 0; p < supportSize; ++p) {
4910           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
4911         }
4912 #endif
4913       }
4914     }
4915     /* Interior cell faces have 2 vertices and 2 cells */
4916     for (c = cStart; c < cMax; ++c) {
4917       const PetscInt *cone;
4918 
4919       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4920       for (r = 0; r < 3; ++r) {
4921         const PetscInt newp = fStartNew + (fMax - fStart)*2 + (c - cStart)*3 + r;
4922         PetscInt       coneNew[2];
4923         PetscInt       supportNew[2];
4924 
4925         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r]       - fStart);
4926         coneNew[1] = vStartNew + (vEnd - vStart) + (cone[(r+1)%3] - fStart);
4927         ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4928 #if 1
4929         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4930         for (p = 0; p < 2; ++p) {
4931           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4932         }
4933 #endif
4934         supportNew[0] = (c - cStart)*4 + (r+1)%3;
4935         supportNew[1] = (c - cStart)*4 + 3;
4936         ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4937 #if 1
4938         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4939         for (p = 0; p < 2; ++p) {
4940           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4941         }
4942 #endif
4943       }
4944     }
4945     /* Interior hybrid faces have 2 vertices and the same cells */
4946     for (f = fMax; f < fEnd; ++f) {
4947       const PetscInt  newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (f - fMax);
4948       const PetscInt *cone;
4949       const PetscInt *support;
4950       PetscInt        coneNew[2];
4951       PetscInt        supportNew[2];
4952       PetscInt        size, s, r;
4953 
4954       ierr       = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4955       coneNew[0] = vStartNew + (cone[0] - vStart);
4956       coneNew[1] = vStartNew + (cone[1] - vStart);
4957       ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4958 #if 1
4959       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4960       for (p = 0; p < 2; ++p) {
4961         if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4962       }
4963 #endif
4964       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4965       ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4966       for (s = 0; s < size; ++s) {
4967         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4968         for (r = 0; r < 2; ++r) {
4969           if (cone[r+2] == f) break;
4970         }
4971         supportNew[s] = (cMax - cStart)*4 + (support[s] - cMax)*2 + r;
4972       }
4973       ierr = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4974 #if 1
4975       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4976       for (p = 0; p < size; ++p) {
4977         if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4978       }
4979 #endif
4980     }
4981     /* Cell hybrid faces have 2 vertices and 2 cells */
4982     for (c = cMax; c < cEnd; ++c) {
4983       const PetscInt  newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (c - cMax);
4984       const PetscInt *cone;
4985       PetscInt        coneNew[2];
4986       PetscInt        supportNew[2];
4987 
4988       ierr       = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4989       coneNew[0] = vStartNew + (vEnd - vStart) + (cone[0] - fStart);
4990       coneNew[1] = vStartNew + (vEnd - vStart) + (cone[1] - fStart);
4991       ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4992 #if 1
4993       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4994       for (p = 0; p < 2; ++p) {
4995         if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4996       }
4997 #endif
4998       supportNew[0] = (cMax - cStart)*4 + (c - cMax)*2 + 0;
4999       supportNew[1] = (cMax - cStart)*4 + (c - cMax)*2 + 1;
5000       ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
5001 #if 1
5002       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
5003       for (p = 0; p < 2; ++p) {
5004         if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
5005       }
5006 #endif
5007     }
5008     /* Old vertices have identical supports */
5009     for (v = vStart; v < vEnd; ++v) {
5010       const PetscInt  newp = vStartNew + (v - vStart);
5011       const PetscInt *support, *cone;
5012       PetscInt        size, s;
5013 
5014       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
5015       ierr = DMPlexGetSupport(dm, v, &support);CHKERRQ(ierr);
5016       for (s = 0; s < size; ++s) {
5017         if (support[s] >= fMax) {
5018           supportRef[s] = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (support[s] - fMax);
5019         } else {
5020           PetscInt r = 0;
5021 
5022           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
5023           if (cone[1] == v) r = 1;
5024           supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
5025         }
5026       }
5027       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
5028 #if 1
5029       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
5030       for (p = 0; p < size; ++p) {
5031         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
5032       }
5033 #endif
5034     }
5035     /* Face vertices have 2 + (2 interior, 1 hybrid) supports */
5036     for (f = fStart; f < fMax; ++f) {
5037       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
5038       const PetscInt *cone, *support;
5039       PetscInt        size, newSize = 2, s;
5040 
5041       ierr          = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
5042       ierr          = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
5043       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
5044       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
5045       for (s = 0; s < size; ++s) {
5046         PetscInt r = 0;
5047 
5048         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
5049         if (support[s] >= cMax) {
5050           supportRef[newSize+0] = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (support[s] - cMax);
5051 
5052           newSize += 1;
5053         } else {
5054           if      (cone[1] == f) r = 1;
5055           else if (cone[2] == f) r = 2;
5056           supportRef[newSize+0] = fStartNew + (fMax - fStart)*2 + (support[s] - cStart)*3 + (r+2)%3;
5057           supportRef[newSize+1] = fStartNew + (fMax - fStart)*2 + (support[s] - cStart)*3 + r;
5058 
5059           newSize += 2;
5060         }
5061       }
5062       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
5063 #if 1
5064       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
5065       for (p = 0; p < newSize; ++p) {
5066         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
5067       }
5068 #endif
5069     }
5070     ierr = PetscFree(supportRef);CHKERRQ(ierr);
5071     break;
5072   default:
5073     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5074   }
5075   PetscFunctionReturn(0);
5076 }
5077 
5078 #undef __FUNCT__
5079 #define __FUNCT__ "CellRefinerSetCoordinates"
5080 PetscErrorCode CellRefinerSetCoordinates(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
5081 {
5082   PetscSection   coordSection, coordSectionNew;
5083   Vec            coordinates, coordinatesNew;
5084   PetscScalar   *coords, *coordsNew;
5085   PetscInt       dim, depth, coordSizeNew, cStart, cEnd, c, vStart, vStartNew, vEnd, v, fStart, fEnd, fMax, f;
5086   PetscErrorCode ierr;
5087 
5088   PetscFunctionBegin;
5089   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5090   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5091   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5092   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5093   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
5094   ierr = DMPlexGetHybridBounds(dm, NULL, &fMax, NULL, NULL);CHKERRQ(ierr);
5095   ierr = GetDepthStart_Private(depth, depthSize, NULL, NULL, NULL, &vStartNew);CHKERRQ(ierr);
5096   ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
5097   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &coordSectionNew);CHKERRQ(ierr);
5098   ierr = PetscSectionSetNumFields(coordSectionNew, 1);CHKERRQ(ierr);
5099   ierr = PetscSectionSetFieldComponents(coordSectionNew, 0, dim);CHKERRQ(ierr);
5100   ierr = PetscSectionSetChart(coordSectionNew, vStartNew, vStartNew+depthSize[0]);CHKERRQ(ierr);
5101   if (fMax < 0) fMax = fEnd;
5102   switch (refiner) {
5103   case 1:
5104   case 2:
5105   case 3:
5106     /* Simplicial and Hex 2D */
5107     /* All vertices have the dim coordinates */
5108     for (v = vStartNew; v < vStartNew+depthSize[0]; ++v) {
5109       ierr = PetscSectionSetDof(coordSectionNew, v, dim);CHKERRQ(ierr);
5110       ierr = PetscSectionSetFieldDof(coordSectionNew, v, 0, dim);CHKERRQ(ierr);
5111     }
5112     ierr = PetscSectionSetUp(coordSectionNew);CHKERRQ(ierr);
5113     ierr = DMPlexSetCoordinateSection(rdm, coordSectionNew);CHKERRQ(ierr);
5114     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
5115     ierr = PetscSectionGetStorageSize(coordSectionNew, &coordSizeNew);CHKERRQ(ierr);
5116     ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinatesNew);CHKERRQ(ierr);
5117     ierr = PetscObjectSetName((PetscObject) coordinatesNew, "coordinates");CHKERRQ(ierr);
5118     ierr = VecSetSizes(coordinatesNew, coordSizeNew, PETSC_DETERMINE);CHKERRQ(ierr);
5119     ierr = VecSetType(coordinatesNew,dm->vectype);CHKERRQ(ierr);
5120     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
5121     ierr = VecGetArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
5122     /* Old vertices have the same coordinates */
5123     for (v = vStart; v < vEnd; ++v) {
5124       const PetscInt newv = vStartNew + (v - vStart);
5125       PetscInt       off, offnew, d;
5126 
5127       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
5128       ierr = PetscSectionGetOffset(coordSectionNew, newv, &offnew);CHKERRQ(ierr);
5129       for (d = 0; d < dim; ++d) {
5130         coordsNew[offnew+d] = coords[off+d];
5131       }
5132     }
5133     /* Face vertices have the average of endpoint coordinates */
5134     for (f = fStart; f < fMax; ++f) {
5135       const PetscInt  newv = vStartNew + (vEnd - vStart) + (f - fStart);
5136       const PetscInt *cone;
5137       PetscInt        coneSize, offA, offB, offnew, d;
5138 
5139       ierr = DMPlexGetConeSize(dm, f, &coneSize);CHKERRQ(ierr);
5140       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Face %d cone should have two vertices, not %d", f, coneSize);
5141       ierr = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
5142       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
5143       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
5144       ierr = PetscSectionGetOffset(coordSectionNew, newv, &offnew);CHKERRQ(ierr);
5145       for (d = 0; d < dim; ++d) {
5146         coordsNew[offnew+d] = 0.5*(coords[offA+d] + coords[offB+d]);
5147       }
5148     }
5149     /* Just Hex 2D */
5150     if (refiner == 2) {
5151       /* Cell vertices have the average of corner coordinates */
5152       for (c = cStart; c < cEnd; ++c) {
5153         const PetscInt newv = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
5154         PetscInt      *cone = NULL;
5155         PetscInt       closureSize, coneSize = 0, offA, offB, offC, offD, offnew, p, d;
5156 
5157         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &cone);CHKERRQ(ierr);
5158         for (p = 0; p < closureSize*2; p += 2) {
5159           const PetscInt point = cone[p];
5160           if ((point >= vStart) && (point < vEnd)) cone[coneSize++] = point;
5161         }
5162         if (coneSize != 4) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Quad %d cone should have four vertices, not %d", c, coneSize);
5163         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
5164         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
5165         ierr = PetscSectionGetOffset(coordSection, cone[2], &offC);CHKERRQ(ierr);
5166         ierr = PetscSectionGetOffset(coordSection, cone[3], &offD);CHKERRQ(ierr);
5167         ierr = PetscSectionGetOffset(coordSectionNew, newv, &offnew);CHKERRQ(ierr);
5168         for (d = 0; d < dim; ++d) {
5169           coordsNew[offnew+d] = 0.25*(coords[offA+d] + coords[offB+d] + coords[offC+d] + coords[offD+d]);
5170         }
5171         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &cone);CHKERRQ(ierr);
5172       }
5173     }
5174     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
5175     ierr = VecRestoreArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
5176     ierr = DMSetCoordinatesLocal(rdm, coordinatesNew);CHKERRQ(ierr);
5177     ierr = VecDestroy(&coordinatesNew);CHKERRQ(ierr);
5178     ierr = PetscSectionDestroy(&coordSectionNew);CHKERRQ(ierr);
5179     break;
5180   default:
5181     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5182   }
5183   PetscFunctionReturn(0);
5184 }
5185 
5186 #undef __FUNCT__
5187 #define __FUNCT__ "DMPlexCreateProcessSF"
5188 PetscErrorCode DMPlexCreateProcessSF(DM dm, PetscSF sfPoint, IS *processRanks, PetscSF *sfProcess)
5189 {
5190   PetscInt           numRoots, numLeaves, l;
5191   const PetscInt    *localPoints;
5192   const PetscSFNode *remotePoints;
5193   PetscInt          *localPointsNew;
5194   PetscSFNode       *remotePointsNew;
5195   PetscInt          *ranks, *ranksNew;
5196   PetscErrorCode     ierr;
5197 
5198   PetscFunctionBegin;
5199   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
5200   ierr = PetscMalloc(numLeaves * sizeof(PetscInt), &ranks);CHKERRQ(ierr);
5201   for (l = 0; l < numLeaves; ++l) {
5202     ranks[l] = remotePoints[l].rank;
5203   }
5204   ierr = PetscSortRemoveDupsInt(&numLeaves, ranks);CHKERRQ(ierr);
5205   ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &ranksNew);CHKERRQ(ierr);
5206   ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &localPointsNew);CHKERRQ(ierr);
5207   ierr = PetscMalloc(numLeaves * sizeof(PetscSFNode), &remotePointsNew);CHKERRQ(ierr);
5208   for (l = 0; l < numLeaves; ++l) {
5209     ranksNew[l]              = ranks[l];
5210     localPointsNew[l]        = l;
5211     remotePointsNew[l].index = 0;
5212     remotePointsNew[l].rank  = ranksNew[l];
5213   }
5214   ierr = PetscFree(ranks);CHKERRQ(ierr);
5215   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), numLeaves, ranksNew, PETSC_OWN_POINTER, processRanks);CHKERRQ(ierr);
5216   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfProcess);CHKERRQ(ierr);
5217   ierr = PetscSFSetFromOptions(*sfProcess);CHKERRQ(ierr);
5218   ierr = PetscSFSetGraph(*sfProcess, 1, numLeaves, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
5219   PetscFunctionReturn(0);
5220 }
5221 
5222 #undef __FUNCT__
5223 #define __FUNCT__ "CellRefinerCreateSF"
5224 PetscErrorCode CellRefinerCreateSF(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
5225 {
5226   PetscSF            sf, sfNew, sfProcess;
5227   IS                 processRanks;
5228   MPI_Datatype       depthType;
5229   PetscInt           numRoots, numLeaves, numLeavesNew = 0, l, m;
5230   const PetscInt    *localPoints, *neighbors;
5231   const PetscSFNode *remotePoints;
5232   PetscInt          *localPointsNew;
5233   PetscSFNode       *remotePointsNew;
5234   PetscInt          *depthSizeOld, *rdepthSize, *rdepthSizeOld, *rdepthMaxOld, *rvStart, *rvStartNew, *reStart, *reStartNew, *rfStart, *rfStartNew, *rcStart, *rcStartNew;
5235   PetscInt           depth, numNeighbors, pStartNew, pEndNew, cStart, cStartNew, cEnd, cMax, vStart, vStartNew, vEnd, vMax, fStart, fStartNew, fEnd, fMax, eStart, eStartNew, eEnd, eMax, r, n;
5236   PetscErrorCode     ierr;
5237 
5238   PetscFunctionBegin;
5239   ierr = DMPlexGetChart(rdm, &pStartNew, &pEndNew);CHKERRQ(ierr);
5240   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5241   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5242   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
5243   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5244   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
5245   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
5246   ierr = GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);CHKERRQ(ierr);
5247   switch (refiner) {
5248   case 3:
5249     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
5250     cMax = PetscMin(cEnd, cMax);
5251     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
5252     fMax = PetscMin(fEnd, fMax);
5253   }
5254   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
5255   ierr = DMGetPointSF(rdm, &sfNew);CHKERRQ(ierr);
5256   /* Caculate size of new SF */
5257   ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
5258   if (numRoots < 0) PetscFunctionReturn(0);
5259   for (l = 0; l < numLeaves; ++l) {
5260     const PetscInt p = localPoints[l];
5261 
5262     switch (refiner) {
5263     case 1:
5264       /* Simplicial 2D */
5265       if ((p >= vStart) && (p < vEnd)) {
5266         /* Old vertices stay the same */
5267         ++numLeavesNew;
5268       } else if ((p >= fStart) && (p < fEnd)) {
5269         /* Old faces add new faces and vertex */
5270         numLeavesNew += 1 + 2;
5271       } else if ((p >= cStart) && (p < cEnd)) {
5272         /* Old cells add new cells and interior faces */
5273         numLeavesNew += 4 + 3;
5274       }
5275       break;
5276     case 2:
5277       /* Hex 2D */
5278       if ((p >= vStart) && (p < vEnd)) {
5279         /* Old vertices stay the same */
5280         ++numLeavesNew;
5281       } else if ((p >= fStart) && (p < fEnd)) {
5282         /* Old faces add new faces and vertex */
5283         numLeavesNew += 1 + 2;
5284       } else if ((p >= cStart) && (p < cEnd)) {
5285         /* Old cells add new cells and interior faces */
5286         numLeavesNew += 4 + 4;
5287       }
5288       break;
5289     default:
5290       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5291     }
5292   }
5293   /* Communicate depthSizes for each remote rank */
5294   ierr = DMPlexCreateProcessSF(dm, sf, &processRanks, &sfProcess);CHKERRQ(ierr);
5295   ierr = ISGetLocalSize(processRanks, &numNeighbors);CHKERRQ(ierr);
5296   ierr = PetscMalloc5((depth+1)*numNeighbors,PetscInt,&rdepthSize,numNeighbors,PetscInt,&rvStartNew,numNeighbors,PetscInt,&reStartNew,numNeighbors,PetscInt,&rfStartNew,numNeighbors,PetscInt,&rcStartNew);CHKERRQ(ierr);
5297   ierr = PetscMalloc7(depth+1,PetscInt,&depthSizeOld,(depth+1)*numNeighbors,PetscInt,&rdepthSizeOld,(depth+1)*numNeighbors,PetscInt,&rdepthMaxOld,numNeighbors,PetscInt,&rvStart,numNeighbors,PetscInt,&reStart,numNeighbors,PetscInt,&rfStart,numNeighbors,PetscInt,&rcStart);CHKERRQ(ierr);
5298   ierr = MPI_Type_contiguous(depth+1, MPIU_INT, &depthType);CHKERRQ(ierr);
5299   ierr = MPI_Type_commit(&depthType);CHKERRQ(ierr);
5300   ierr = PetscSFBcastBegin(sfProcess, depthType, depthSize, rdepthSize);CHKERRQ(ierr);
5301   ierr = PetscSFBcastEnd(sfProcess, depthType, depthSize, rdepthSize);CHKERRQ(ierr);
5302   for (n = 0; n < numNeighbors; ++n) {
5303     ierr = GetDepthStart_Private(depth, &rdepthSize[n*(depth+1)], &rcStartNew[n], &rfStartNew[n], &reStartNew[n], &rvStartNew[n]);CHKERRQ(ierr);
5304   }
5305   depthSizeOld[depth]   = cMax;
5306   depthSizeOld[0]       = vMax;
5307   depthSizeOld[depth-1] = fMax;
5308   depthSizeOld[1]       = eMax;
5309 
5310   ierr = PetscSFBcastBegin(sfProcess, depthType, depthSizeOld, rdepthMaxOld);CHKERRQ(ierr);
5311   ierr = PetscSFBcastEnd(sfProcess, depthType, depthSizeOld, rdepthMaxOld);CHKERRQ(ierr);
5312 
5313   depthSizeOld[depth]   = cEnd - cStart;
5314   depthSizeOld[0]       = vEnd - vStart;
5315   depthSizeOld[depth-1] = fEnd - fStart;
5316   depthSizeOld[1]       = eEnd - eStart;
5317 
5318   ierr = PetscSFBcastBegin(sfProcess, depthType, depthSizeOld, rdepthSizeOld);CHKERRQ(ierr);
5319   ierr = PetscSFBcastEnd(sfProcess, depthType, depthSizeOld, rdepthSizeOld);CHKERRQ(ierr);
5320   for (n = 0; n < numNeighbors; ++n) {
5321     ierr = GetDepthStart_Private(depth, &rdepthSizeOld[n*(depth+1)], &rcStart[n], &rfStart[n], &reStart[n], &rvStart[n]);CHKERRQ(ierr);
5322   }
5323   ierr = MPI_Type_free(&depthType);CHKERRQ(ierr);
5324   ierr = PetscSFDestroy(&sfProcess);CHKERRQ(ierr);
5325   /* Calculate new point SF */
5326   ierr = PetscMalloc(numLeavesNew * sizeof(PetscInt),    &localPointsNew);CHKERRQ(ierr);
5327   ierr = PetscMalloc(numLeavesNew * sizeof(PetscSFNode), &remotePointsNew);CHKERRQ(ierr);
5328   ierr = ISGetIndices(processRanks, &neighbors);CHKERRQ(ierr);
5329   for (l = 0, m = 0; l < numLeaves; ++l) {
5330     PetscInt    p     = localPoints[l];
5331     PetscInt    rp    = remotePoints[l].index, n;
5332     PetscMPIInt rrank = remotePoints[l].rank;
5333 
5334     ierr = PetscFindInt(rrank, numNeighbors, neighbors, &n);CHKERRQ(ierr);
5335     if (n < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not locate remote rank %d", rrank);
5336     switch (refiner) {
5337     case 1:
5338       /* Simplicial 2D */
5339       if ((p >= vStart) && (p < vEnd)) {
5340         /* Old vertices stay the same */
5341         localPointsNew[m]        = vStartNew     + (p  - vStart);
5342         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
5343         remotePointsNew[m].rank  = rrank;
5344         ++m;
5345       } else if ((p >= fStart) && (p < fEnd)) {
5346         /* Old faces add new faces and vertex */
5347         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
5348         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
5349         remotePointsNew[m].rank  = rrank;
5350         ++m;
5351         for (r = 0; r < 2; ++r, ++m) {
5352           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
5353           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
5354           remotePointsNew[m].rank  = rrank;
5355         }
5356       } else if ((p >= cStart) && (p < cEnd)) {
5357         /* Old cells add new cells and interior faces */
5358         for (r = 0; r < 4; ++r, ++m) {
5359           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5360           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5361           remotePointsNew[m].rank  = rrank;
5362         }
5363         for (r = 0; r < 3; ++r, ++m) {
5364           localPointsNew[m]        = fStartNew     + (fEnd - fStart)*2                    + (p  - cStart)*3     + r;
5365           remotePointsNew[m].index = rfStartNew[n] + rdepthSizeOld[n*(depth+1)+depth-1]*2 + (rp - rcStart[n])*3 + r;
5366           remotePointsNew[m].rank  = rrank;
5367         }
5368       }
5369       break;
5370     case 2:
5371       /* Hex 2D */
5372       if ((p >= vStart) && (p < vEnd)) {
5373         /* Old vertices stay the same */
5374         localPointsNew[m]        = vStartNew     + (p  - vStart);
5375         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
5376         remotePointsNew[m].rank  = rrank;
5377         ++m;
5378       } else if ((p >= fStart) && (p < fEnd)) {
5379         /* Old faces add new faces and vertex */
5380         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
5381         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
5382         remotePointsNew[m].rank  = rrank;
5383         ++m;
5384         for (r = 0; r < 2; ++r, ++m) {
5385           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
5386           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
5387           remotePointsNew[m].rank  = rrank;
5388         }
5389       } else if ((p >= cStart) && (p < cEnd)) {
5390         /* Old cells add new cells and interior faces */
5391         for (r = 0; r < 4; ++r, ++m) {
5392           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5393           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5394           remotePointsNew[m].rank  = rrank;
5395         }
5396         for (r = 0; r < 4; ++r, ++m) {
5397           localPointsNew[m]        = fStartNew     + (fEnd - fStart)*2                    + (p  - cStart)*4     + r;
5398           remotePointsNew[m].index = rfStartNew[n] + rdepthSizeOld[n*(depth+1)+depth-1]*2 + (rp - rcStart[n])*4 + r;
5399           remotePointsNew[m].rank  = rrank;
5400         }
5401       }
5402       break;
5403     case 3:
5404       /* Hybrid simplicial 2D */
5405       if ((p >= vStart) && (p < vEnd)) {
5406         /* Old vertices stay the same */
5407         localPointsNew[m]        = vStartNew     + (p  - vStart);
5408         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
5409         remotePointsNew[m].rank  = rrank;
5410         ++m;
5411       } else if ((p >= fStart) && (p < fMax)) {
5412         /* Old interior faces add new faces and vertex */
5413         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
5414         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
5415         remotePointsNew[m].rank  = rrank;
5416         ++m;
5417         for (r = 0; r < 2; ++r, ++m) {
5418           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
5419           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
5420           remotePointsNew[m].rank  = rrank;
5421         }
5422       } else if ((p >= fMax) && (p < fEnd)) {
5423         /* Old hybrid faces stay the same */
5424         localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (p  - fMax);
5425         remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rp - rdepthMaxOld[n*(depth+1)+depth-1]);
5426         remotePointsNew[m].rank  = rrank;
5427         ++m;
5428       } else if ((p >= cStart) && (p < cMax)) {
5429         /* Old interior cells add new cells and interior faces */
5430         for (r = 0; r < 4; ++r, ++m) {
5431           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5432           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5433           remotePointsNew[m].rank  = rrank;
5434         }
5435         for (r = 0; r < 3; ++r, ++m) {
5436           localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (p  - cStart)*3     + r;
5437           remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rp - rcStart[n])*3 + r;
5438           remotePointsNew[m].rank  = rrank;
5439         }
5440       } else if ((p >= cStart) && (p < cMax)) {
5441         /* Old hybrid cells add new cells and hybrid face */
5442         for (r = 0; r < 2; ++r, ++m) {
5443           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5444           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5445           remotePointsNew[m].rank  = rrank;
5446         }
5447         localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (cMax                            - cStart)*3     + (p  - cMax);
5448         remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rdepthMaxOld[n*(depth+1)+depth] - rcStart[n])*3 + (rp - rdepthMaxOld[n*(depth+1)+depth]);
5449         remotePointsNew[m].rank  = rrank;
5450         ++m;
5451       }
5452       break;
5453     default:
5454       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5455     }
5456   }
5457   ierr = ISRestoreIndices(processRanks, &neighbors);CHKERRQ(ierr);
5458   ierr = ISDestroy(&processRanks);CHKERRQ(ierr);
5459   ierr = PetscSFSetGraph(sfNew, pEndNew-pStartNew, numLeavesNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
5460   ierr = PetscFree5(rdepthSize,rvStartNew,reStartNew,rfStartNew,rcStartNew);CHKERRQ(ierr);
5461   ierr = PetscFree6(depthSizeOld,rdepthSizeOld,rvStart,reStart,rfStart,rcStart);CHKERRQ(ierr);
5462   PetscFunctionReturn(0);
5463 }
5464 
5465 #undef __FUNCT__
5466 #define __FUNCT__ "CellRefinerCreateLabels"
5467 PetscErrorCode CellRefinerCreateLabels(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
5468 {
5469   PetscInt       numLabels, l;
5470   PetscInt       newp, cStart, cStartNew, cEnd, cMax, vStart, vStartNew, vEnd, vMax, fStart, fStartNew, fEnd, fMax, eStart, eEnd, eMax, r;
5471   PetscErrorCode ierr;
5472 
5473   PetscFunctionBegin;
5474   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5475   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
5476   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5477   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
5478 
5479   cStartNew = 0;
5480   vStartNew = depthSize[2];
5481   fStartNew = depthSize[2] + depthSize[0];
5482 
5483   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
5484   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
5485   switch (refiner) {
5486   case 3:
5487     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
5488     cMax = PetscMin(cEnd, cMax);
5489     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
5490     fMax = PetscMin(fEnd, fMax);
5491   }
5492   for (l = 0; l < numLabels; ++l) {
5493     DMLabel         label, labelNew;
5494     const char     *lname;
5495     PetscBool       isDepth;
5496     IS              valueIS;
5497     const PetscInt *values;
5498     PetscInt        numValues, val;
5499 
5500     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
5501     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
5502     if (isDepth) continue;
5503     ierr = DMPlexCreateLabel(rdm, lname);CHKERRQ(ierr);
5504     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
5505     ierr = DMPlexGetLabel(rdm, lname, &labelNew);CHKERRQ(ierr);
5506     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
5507     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
5508     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
5509     for (val = 0; val < numValues; ++val) {
5510       IS              pointIS;
5511       const PetscInt *points;
5512       PetscInt        numPoints, n;
5513 
5514       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
5515       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
5516       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
5517       for (n = 0; n < numPoints; ++n) {
5518         const PetscInt p = points[n];
5519         switch (refiner) {
5520         case 1:
5521           /* Simplicial 2D */
5522           if ((p >= vStart) && (p < vEnd)) {
5523             /* Old vertices stay the same */
5524             newp = vStartNew + (p - vStart);
5525             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5526           } else if ((p >= fStart) && (p < fEnd)) {
5527             /* Old faces add new faces and vertex */
5528             newp = vStartNew + (vEnd - vStart) + (p - fStart);
5529             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5530             for (r = 0; r < 2; ++r) {
5531               newp = fStartNew + (p - fStart)*2 + r;
5532               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5533             }
5534           } else if ((p >= cStart) && (p < cEnd)) {
5535             /* Old cells add new cells and interior faces */
5536             for (r = 0; r < 4; ++r) {
5537               newp = cStartNew + (p - cStart)*4 + r;
5538               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5539             }
5540             for (r = 0; r < 3; ++r) {
5541               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*3 + r;
5542               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5543             }
5544           }
5545           break;
5546         case 2:
5547           /* Hex 2D */
5548           if ((p >= vStart) && (p < vEnd)) {
5549             /* Old vertices stay the same */
5550             newp = vStartNew + (p - vStart);
5551             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5552           } else if ((p >= fStart) && (p < fEnd)) {
5553             /* Old faces add new faces and vertex */
5554             newp = vStartNew + (vEnd - vStart) + (p - fStart);
5555             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5556             for (r = 0; r < 2; ++r) {
5557               newp = fStartNew + (p - fStart)*2 + r;
5558               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5559             }
5560           } else if ((p >= cStart) && (p < cEnd)) {
5561             /* Old cells add new cells and interior faces and vertex */
5562             for (r = 0; r < 4; ++r) {
5563               newp = cStartNew + (p - cStart)*4 + r;
5564               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5565             }
5566             for (r = 0; r < 4; ++r) {
5567               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*4 + r;
5568               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5569             }
5570             newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (p - cStart);
5571             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5572           }
5573           break;
5574         case 3:
5575           /* Hybrid simplicial 2D */
5576           if ((p >= vStart) && (p < vEnd)) {
5577             /* Old vertices stay the same */
5578             newp = vStartNew + (p - vStart);
5579             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5580           } else if ((p >= fStart) && (p < fMax)) {
5581             /* Old interior faces add new faces and vertex */
5582             newp = vStartNew + (vEnd - vStart) + (p - fStart);
5583             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5584             for (r = 0; r < 2; ++r) {
5585               newp = fStartNew + (p - fStart)*2 + r;
5586               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5587             }
5588           } else if ((p >= fMax) && (p < fEnd)) {
5589             /* Old hybrid faces stay the same */
5590             newp = fStartNew + (fMax - fStart)*2 + (p - fMax);
5591             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5592           } else if ((p >= cStart) && (p < cMax)) {
5593             /* Old interior cells add new cells and interior faces */
5594             for (r = 0; r < 4; ++r) {
5595               newp = cStartNew + (p - cStart)*4 + r;
5596               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5597             }
5598             for (r = 0; r < 3; ++r) {
5599               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*3 + r;
5600               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5601             }
5602           } else if ((p >= cMax) && (p < cEnd)) {
5603             /* Old hybrid cells add new cells and hybrid face */
5604             for (r = 0; r < 2; ++r) {
5605               newp = cStartNew + (cMax - cStart)*4 + (p - cMax)*2 + r;
5606               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5607             }
5608             newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (p - cMax);
5609             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5610           }
5611           break;
5612         default:
5613           SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5614         }
5615       }
5616       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
5617       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
5618     }
5619     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
5620     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
5621     if (0) {
5622       ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
5623       ierr = DMLabelView(labelNew, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
5624       ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
5625     }
5626   }
5627   PetscFunctionReturn(0);
5628 }
5629 
5630 #undef __FUNCT__
5631 #define __FUNCT__ "DMPlexRefine_Uniform"
5632 /* This will only work for interpolated meshes */
5633 PetscErrorCode DMPlexRefine_Uniform(DM dm, CellRefiner cellRefiner, DM *dmRefined)
5634 {
5635   DM             rdm;
5636   PetscInt      *depthSize;
5637   PetscInt       dim, depth = 0, d, pStart = 0, pEnd = 0;
5638   PetscErrorCode ierr;
5639 
5640   PetscFunctionBegin;
5641   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &rdm);CHKERRQ(ierr);
5642   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
5643   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5644   ierr = DMPlexSetDimension(rdm, dim);CHKERRQ(ierr);
5645   /* Calculate number of new points of each depth */
5646   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5647   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthSize);CHKERRQ(ierr);
5648   ierr = PetscMemzero(depthSize, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
5649   ierr = CellRefinerGetSizes(cellRefiner, dm, depthSize);CHKERRQ(ierr);
5650   /* Step 1: Set chart */
5651   for (d = 0; d <= depth; ++d) pEnd += depthSize[d];
5652   ierr = DMPlexSetChart(rdm, pStart, pEnd);CHKERRQ(ierr);
5653   /* Step 2: Set cone/support sizes */
5654   ierr = CellRefinerSetConeSizes(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5655   /* Step 3: Setup refined DM */
5656   ierr = DMSetUp(rdm);CHKERRQ(ierr);
5657   /* Step 4: Set cones and supports */
5658   ierr = CellRefinerSetCones(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5659   /* Step 5: Stratify */
5660   ierr = DMPlexStratify(rdm);CHKERRQ(ierr);
5661   /* Step 6: Set coordinates for vertices */
5662   ierr = CellRefinerSetCoordinates(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5663   /* Step 7: Create pointSF */
5664   ierr = CellRefinerCreateSF(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5665   /* Step 8: Create labels */
5666   ierr = CellRefinerCreateLabels(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5667   ierr = PetscFree(depthSize);CHKERRQ(ierr);
5668 
5669   *dmRefined = rdm;
5670   PetscFunctionReturn(0);
5671 }
5672 
5673 #undef __FUNCT__
5674 #define __FUNCT__ "DMPlexSetRefinementUniform"
5675 PetscErrorCode DMPlexSetRefinementUniform(DM dm, PetscBool refinementUniform)
5676 {
5677   DM_Plex *mesh = (DM_Plex*) dm->data;
5678 
5679   PetscFunctionBegin;
5680   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5681   mesh->refinementUniform = refinementUniform;
5682   PetscFunctionReturn(0);
5683 }
5684 
5685 #undef __FUNCT__
5686 #define __FUNCT__ "DMPlexGetRefinementUniform"
5687 PetscErrorCode DMPlexGetRefinementUniform(DM dm, PetscBool *refinementUniform)
5688 {
5689   DM_Plex *mesh = (DM_Plex*) dm->data;
5690 
5691   PetscFunctionBegin;
5692   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5693   PetscValidPointer(refinementUniform,  2);
5694   *refinementUniform = mesh->refinementUniform;
5695   PetscFunctionReturn(0);
5696 }
5697 
5698 #undef __FUNCT__
5699 #define __FUNCT__ "DMPlexSetRefinementLimit"
5700 PetscErrorCode DMPlexSetRefinementLimit(DM dm, PetscReal refinementLimit)
5701 {
5702   DM_Plex *mesh = (DM_Plex*) dm->data;
5703 
5704   PetscFunctionBegin;
5705   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5706   mesh->refinementLimit = refinementLimit;
5707   PetscFunctionReturn(0);
5708 }
5709 
5710 #undef __FUNCT__
5711 #define __FUNCT__ "DMPlexGetRefinementLimit"
5712 PetscErrorCode DMPlexGetRefinementLimit(DM dm, PetscReal *refinementLimit)
5713 {
5714   DM_Plex *mesh = (DM_Plex*) dm->data;
5715 
5716   PetscFunctionBegin;
5717   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5718   PetscValidPointer(refinementLimit,  2);
5719   /* if (mesh->refinementLimit < 0) = getMaxVolume()/2.0; */
5720   *refinementLimit = mesh->refinementLimit;
5721   PetscFunctionReturn(0);
5722 }
5723 
5724 #undef __FUNCT__
5725 #define __FUNCT__ "DMPlexGetCellRefiner_Private"
5726 PetscErrorCode DMPlexGetCellRefiner_Private(DM dm, CellRefiner *cellRefiner)
5727 {
5728   PetscInt       dim, cStart, coneSize, cMax;
5729   PetscErrorCode ierr;
5730 
5731   PetscFunctionBegin;
5732   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5733   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
5734   ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
5735   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5736   switch (dim) {
5737   case 2:
5738     switch (coneSize) {
5739     case 3:
5740       if (cMax >= 0) *cellRefiner = 3; /* Hybrid */
5741       else *cellRefiner = 1; /* Triangular */
5742       break;
5743     case 4:
5744       if (cMax >= 0) *cellRefiner = 4; /* Hybrid */
5745       else *cellRefiner = 2; /* Quadrilateral */
5746       break;
5747     default:
5748       SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown coneSize %d in dimension %d for cell refiner", coneSize, dim);
5749     }
5750     break;
5751   default:
5752     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown dimension %d for cell refiner", dim);
5753   }
5754   PetscFunctionReturn(0);
5755 }
5756 
5757 #undef __FUNCT__
5758 #define __FUNCT__ "DMRefine_Plex"
5759 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
5760 {
5761   PetscReal      refinementLimit;
5762   PetscInt       dim, cStart, cEnd;
5763   char           genname[1024], *name = NULL;
5764   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
5765   PetscErrorCode ierr;
5766 
5767   PetscFunctionBegin;
5768   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
5769   if (isUniform) {
5770     CellRefiner cellRefiner;
5771 
5772     ierr = DMPlexGetCellRefiner_Private(dm, &cellRefiner);CHKERRQ(ierr);
5773     ierr = DMPlexRefine_Uniform(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
5774     PetscFunctionReturn(0);
5775   }
5776   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
5777   if (refinementLimit == 0.0) PetscFunctionReturn(0);
5778   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5779   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5780   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
5781   if (flg) name = genname;
5782   if (name) {
5783     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
5784     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
5785     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
5786   }
5787   switch (dim) {
5788   case 2:
5789     if (!name || isTriangle) {
5790 #if defined(PETSC_HAVE_TRIANGLE)
5791       double  *maxVolumes;
5792       PetscInt c;
5793 
5794       ierr = PetscMalloc((cEnd - cStart) * sizeof(double), &maxVolumes);CHKERRQ(ierr);
5795       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
5796       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
5797       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
5798 #else
5799       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
5800 #endif
5801     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
5802     break;
5803   case 3:
5804     if (!name || isCTetgen) {
5805 #if defined(PETSC_HAVE_CTETGEN)
5806       PetscReal *maxVolumes;
5807       PetscInt   c;
5808 
5809       ierr = PetscMalloc((cEnd - cStart) * sizeof(PetscReal), &maxVolumes);CHKERRQ(ierr);
5810       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
5811       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
5812 #else
5813       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
5814 #endif
5815     } else if (isTetgen) {
5816 #if defined(PETSC_HAVE_TETGEN)
5817       double  *maxVolumes;
5818       PetscInt c;
5819 
5820       ierr = PetscMalloc((cEnd - cStart) * sizeof(double), &maxVolumes);CHKERRQ(ierr);
5821       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
5822       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
5823 #else
5824       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
5825 #endif
5826     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
5827     break;
5828   default:
5829     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
5830   }
5831   PetscFunctionReturn(0);
5832 }
5833 
5834 #undef __FUNCT__
5835 #define __FUNCT__ "DMPlexGetDepthLabel"
5836 /*@
5837   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
5838 
5839   Not Collective
5840 
5841   Input Parameter:
5842 . dm    - The DMPlex object
5843 
5844   Output Parameter:
5845 . depthLabel - The DMLabel recording point depth
5846 
5847   Level: developer
5848 
5849 .keywords: mesh, points
5850 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
5851 @*/
5852 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
5853 {
5854   DM_Plex       *mesh = (DM_Plex*) dm->data;
5855   PetscErrorCode ierr;
5856 
5857   PetscFunctionBegin;
5858   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5859   PetscValidPointer(depthLabel, 2);
5860   if (!mesh->depthLabel) {
5861     ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);
5862   }
5863   *depthLabel = mesh->depthLabel;
5864   PetscFunctionReturn(0);
5865 }
5866 
5867 #undef __FUNCT__
5868 #define __FUNCT__ "DMPlexGetDepth"
5869 /*@
5870   DMPlexGetDepth - Get the depth of the DAG representing this mesh
5871 
5872   Not Collective
5873 
5874   Input Parameter:
5875 . dm    - The DMPlex object
5876 
5877   Output Parameter:
5878 . depth - The number of strata (breadth first levels) in the DAG
5879 
5880   Level: developer
5881 
5882 .keywords: mesh, points
5883 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
5884 @*/
5885 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
5886 {
5887   DMLabel        label;
5888   PetscInt       d = 0;
5889   PetscErrorCode ierr;
5890 
5891   PetscFunctionBegin;
5892   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5893   PetscValidPointer(depth, 2);
5894   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5895   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
5896   *depth = d-1;
5897   PetscFunctionReturn(0);
5898 }
5899 
5900 #undef __FUNCT__
5901 #define __FUNCT__ "DMPlexGetDepthStratum"
5902 /*@
5903   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
5904 
5905   Not Collective
5906 
5907   Input Parameters:
5908 + dm           - The DMPlex object
5909 - stratumValue - The requested depth
5910 
5911   Output Parameters:
5912 + start - The first point at this depth
5913 - end   - One beyond the last point at this depth
5914 
5915   Level: developer
5916 
5917 .keywords: mesh, points
5918 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
5919 @*/
5920 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
5921 {
5922   DMLabel        label;
5923   PetscInt       depth;
5924   PetscErrorCode ierr;
5925 
5926   PetscFunctionBegin;
5927   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5928   if (stratumValue < 0) {
5929     ierr = DMPlexGetChart(dm, start, end);CHKERRQ(ierr);
5930     PetscFunctionReturn(0);
5931   } else {
5932     PetscInt pStart, pEnd;
5933 
5934     if (start) *start = 0;
5935     if (end)   *end   = 0;
5936     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5937     if (pStart == pEnd) PetscFunctionReturn(0);
5938   }
5939   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5940   if (!label) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
5941   /* Strata are sorted and contiguous -- In addition, depth/height is either full or 1-level */
5942   depth = stratumValue;
5943   if ((depth < 0) || (depth >= label->numStrata)) {
5944     if (start) *start = 0;
5945     if (end)   *end   = 0;
5946   } else {
5947     if (start) *start = label->points[label->stratumOffsets[depth]];
5948     if (end)   *end   = label->points[label->stratumOffsets[depth]+label->stratumSizes[depth]-1]+1;
5949   }
5950   PetscFunctionReturn(0);
5951 }
5952 
5953 #undef __FUNCT__
5954 #define __FUNCT__ "DMPlexGetHeightStratum"
5955 /*@
5956   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
5957 
5958   Not Collective
5959 
5960   Input Parameters:
5961 + dm           - The DMPlex object
5962 - stratumValue - The requested height
5963 
5964   Output Parameters:
5965 + start - The first point at this height
5966 - end   - One beyond the last point at this height
5967 
5968   Level: developer
5969 
5970 .keywords: mesh, points
5971 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
5972 @*/
5973 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
5974 {
5975   DMLabel        label;
5976   PetscInt       depth;
5977   PetscErrorCode ierr;
5978 
5979   PetscFunctionBegin;
5980   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5981   if (stratumValue < 0) {
5982     ierr = DMPlexGetChart(dm, start, end);CHKERRQ(ierr);
5983   } else {
5984     PetscInt pStart, pEnd;
5985 
5986     if (start) *start = 0;
5987     if (end)   *end   = 0;
5988     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5989     if (pStart == pEnd) PetscFunctionReturn(0);
5990   }
5991   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5992   if (!label) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
5993   /* Strata are sorted and contiguous -- In addition, depth/height is either full or 1-level */
5994   depth = label->stratumValues[label->numStrata-1] - stratumValue;
5995   if ((depth < 0) || (depth >= label->numStrata)) {
5996     if (start) *start = 0;
5997     if (end)   *end   = 0;
5998   } else {
5999     if (start) *start = label->points[label->stratumOffsets[depth]];
6000     if (end)   *end   = label->points[label->stratumOffsets[depth]+label->stratumSizes[depth]-1]+1;
6001   }
6002   PetscFunctionReturn(0);
6003 }
6004 
6005 #undef __FUNCT__
6006 #define __FUNCT__ "DMPlexCreateSectionInitial"
6007 /* Set the number of dof on each point and separate by fields */
6008 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
6009 {
6010   PetscInt      *numDofTot;
6011   PetscInt       pStart = 0, pEnd = 0;
6012   PetscInt       p, d, f;
6013   PetscErrorCode ierr;
6014 
6015   PetscFunctionBegin;
6016   ierr = PetscMalloc((dim+1) * sizeof(PetscInt), &numDofTot);CHKERRQ(ierr);
6017   for (d = 0; d <= dim; ++d) {
6018     numDofTot[d] = 0;
6019     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
6020   }
6021   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
6022   if (numFields > 0) {
6023     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
6024     if (numComp) {
6025       for (f = 0; f < numFields; ++f) {
6026         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
6027       }
6028     }
6029   }
6030   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6031   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
6032   for (d = 0; d <= dim; ++d) {
6033     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
6034     for (p = pStart; p < pEnd; ++p) {
6035       for (f = 0; f < numFields; ++f) {
6036         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
6037       }
6038       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
6039     }
6040   }
6041   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
6042   PetscFunctionReturn(0);
6043 }
6044 
6045 #undef __FUNCT__
6046 #define __FUNCT__ "DMPlexCreateSectionBCDof"
6047 /* Set the number of dof on each point and separate by fields
6048    If constDof is PETSC_DETERMINE, constrain every dof on the point
6049 */
6050 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
6051 {
6052   PetscInt       numFields;
6053   PetscInt       bc;
6054   PetscErrorCode ierr;
6055 
6056   PetscFunctionBegin;
6057   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6058   for (bc = 0; bc < numBC; ++bc) {
6059     PetscInt        field = 0;
6060     const PetscInt *idx;
6061     PetscInt        n, i;
6062 
6063     if (numFields) field = bcField[bc];
6064     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
6065     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
6066     for (i = 0; i < n; ++i) {
6067       const PetscInt p        = idx[i];
6068       PetscInt       numConst = constDof;
6069 
6070       /* Constrain every dof on the point */
6071       if (numConst < 0) {
6072         if (numFields) {
6073           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
6074         } else {
6075           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
6076         }
6077       }
6078       if (numFields) {
6079         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
6080       }
6081       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
6082     }
6083     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
6084   }
6085   PetscFunctionReturn(0);
6086 }
6087 
6088 #undef __FUNCT__
6089 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
6090 /* Set the constrained indices on each point and separate by fields */
6091 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
6092 {
6093   PetscInt      *maxConstraints;
6094   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
6095   PetscErrorCode ierr;
6096 
6097   PetscFunctionBegin;
6098   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6099   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6100   ierr = PetscMalloc((numFields+1) * sizeof(PetscInt), &maxConstraints);CHKERRQ(ierr);
6101   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
6102   for (p = pStart; p < pEnd; ++p) {
6103     PetscInt cdof;
6104 
6105     if (numFields) {
6106       for (f = 0; f < numFields; ++f) {
6107         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
6108         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
6109       }
6110     } else {
6111       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
6112       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
6113     }
6114   }
6115   for (f = 0; f < numFields; ++f) {
6116     maxConstraints[numFields] += maxConstraints[f];
6117   }
6118   if (maxConstraints[numFields]) {
6119     PetscInt *indices;
6120 
6121     ierr = PetscMalloc(maxConstraints[numFields] * sizeof(PetscInt), &indices);CHKERRQ(ierr);
6122     for (p = pStart; p < pEnd; ++p) {
6123       PetscInt cdof, d;
6124 
6125       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
6126       if (cdof) {
6127         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
6128         if (numFields) {
6129           PetscInt numConst = 0, foff = 0;
6130 
6131           for (f = 0; f < numFields; ++f) {
6132             PetscInt cfdof, fdof;
6133 
6134             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
6135             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
6136             /* Change constraint numbering from absolute local dof number to field relative local dof number */
6137             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
6138             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
6139             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
6140             numConst += cfdof;
6141             foff     += fdof;
6142           }
6143           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
6144         } else {
6145           for (d = 0; d < cdof; ++d) indices[d] = d;
6146         }
6147         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
6148       }
6149     }
6150     ierr = PetscFree(indices);CHKERRQ(ierr);
6151   }
6152   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
6153   PetscFunctionReturn(0);
6154 }
6155 
6156 #undef __FUNCT__
6157 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
6158 /* Set the constrained field indices on each point */
6159 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
6160 {
6161   const PetscInt *points, *indices;
6162   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
6163   PetscErrorCode  ierr;
6164 
6165   PetscFunctionBegin;
6166   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6167   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
6168 
6169   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
6170   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
6171   if (!constraintIndices) {
6172     PetscInt *idx, i;
6173 
6174     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6175     ierr = PetscMalloc(maxDof * sizeof(PetscInt), &idx);CHKERRQ(ierr);
6176     for (i = 0; i < maxDof; ++i) idx[i] = i;
6177     for (p = 0; p < numPoints; ++p) {
6178       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
6179     }
6180     ierr = PetscFree(idx);CHKERRQ(ierr);
6181   } else {
6182     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
6183     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
6184     for (p = 0; p < numPoints; ++p) {
6185       PetscInt fcdof;
6186 
6187       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
6188       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
6189       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
6190     }
6191     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
6192   }
6193   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
6194   PetscFunctionReturn(0);
6195 }
6196 
6197 #undef __FUNCT__
6198 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
6199 /* Set the constrained indices on each point and separate by fields */
6200 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
6201 {
6202   PetscInt      *indices;
6203   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
6204   PetscErrorCode ierr;
6205 
6206   PetscFunctionBegin;
6207   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6208   ierr = PetscMalloc(maxDof * sizeof(PetscInt), &indices);CHKERRQ(ierr);
6209   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6210   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
6211   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6212   for (p = pStart; p < pEnd; ++p) {
6213     PetscInt cdof, d;
6214 
6215     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
6216     if (cdof) {
6217       PetscInt numConst = 0, foff = 0;
6218 
6219       for (f = 0; f < numFields; ++f) {
6220         const PetscInt *fcind;
6221         PetscInt        fdof, fcdof;
6222 
6223         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
6224         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
6225         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
6226         /* Change constraint numbering from field relative local dof number to absolute local dof number */
6227         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
6228         foff     += fdof;
6229         numConst += fcdof;
6230       }
6231       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
6232       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
6233     }
6234   }
6235   ierr = PetscFree(indices);CHKERRQ(ierr);
6236   PetscFunctionReturn(0);
6237 }
6238 
6239 #undef __FUNCT__
6240 #define __FUNCT__ "DMPlexCreateSection"
6241 /*@C
6242   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
6243 
6244   Not Collective
6245 
6246   Input Parameters:
6247 + dm        - The DMPlex object
6248 . dim       - The spatial dimension of the problem
6249 . numFields - The number of fields in the problem
6250 . numComp   - An array of size numFields that holds the number of components for each field
6251 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
6252 . numBC     - The number of boundary conditions
6253 . bcField   - An array of size numBC giving the field number for each boundry condition
6254 - bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
6255 
6256   Output Parameter:
6257 . section - The PetscSection object
6258 
6259   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
6260   nubmer of dof for field 0 on each edge.
6261 
6262   Level: developer
6263 
6264   Fortran Notes:
6265   A Fortran 90 version is available as DMPlexCreateSectionF90()
6266 
6267 .keywords: mesh, elements
6268 .seealso: DMPlexCreate(), PetscSectionCreate()
6269 @*/
6270 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscSection *section)
6271 {
6272   PetscErrorCode ierr;
6273 
6274   PetscFunctionBegin;
6275   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
6276   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
6277   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
6278   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
6279   {
6280     PetscBool view = PETSC_FALSE;
6281 
6282     ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-section_view", &view);CHKERRQ(ierr);
6283     if (view) {ierr = PetscSectionView(*section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
6284   }
6285   PetscFunctionReturn(0);
6286 }
6287 
6288 #undef __FUNCT__
6289 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
6290 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
6291 {
6292   PetscSection   section;
6293   PetscErrorCode ierr;
6294 
6295   PetscFunctionBegin;
6296   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
6297   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6298   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
6299   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6300   PetscFunctionReturn(0);
6301 }
6302 
6303 #undef __FUNCT__
6304 #define __FUNCT__ "DMPlexGetCoordinateSection"
6305 /*@
6306   DMPlexGetCoordinateSection - Retrieve the layout of coordinate values over the mesh.
6307 
6308   Not Collective
6309 
6310   Input Parameter:
6311 . dm - The DMPlex object
6312 
6313   Output Parameter:
6314 . section - The PetscSection object
6315 
6316   Level: intermediate
6317 
6318 .keywords: mesh, coordinates
6319 .seealso: DMGetCoordinateDM(), DMPlexGetDefaultSection(), DMPlexSetDefaultSection()
6320 @*/
6321 PetscErrorCode DMPlexGetCoordinateSection(DM dm, PetscSection *section)
6322 {
6323   DM             cdm;
6324   PetscErrorCode ierr;
6325 
6326   PetscFunctionBegin;
6327   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6328   PetscValidPointer(section, 2);
6329   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
6330   ierr = DMGetDefaultSection(cdm, section);CHKERRQ(ierr);
6331   PetscFunctionReturn(0);
6332 }
6333 
6334 #undef __FUNCT__
6335 #define __FUNCT__ "DMPlexSetCoordinateSection"
6336 /*@
6337   DMPlexSetCoordinateSection - Set the layout of coordinate values over the mesh.
6338 
6339   Not Collective
6340 
6341   Input Parameters:
6342 + dm      - The DMPlex object
6343 - section - The PetscSection object
6344 
6345   Level: intermediate
6346 
6347 .keywords: mesh, coordinates
6348 .seealso: DMPlexGetCoordinateSection(), DMPlexGetDefaultSection(), DMPlexSetDefaultSection()
6349 @*/
6350 PetscErrorCode DMPlexSetCoordinateSection(DM dm, PetscSection section)
6351 {
6352   DM             cdm;
6353   PetscErrorCode ierr;
6354 
6355   PetscFunctionBegin;
6356   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
6357   PetscValidHeaderSpecific(section,PETSC_SECTION_CLASSID,2);
6358   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
6359   ierr = DMSetDefaultSection(cdm, section);CHKERRQ(ierr);
6360   PetscFunctionReturn(0);
6361 }
6362 
6363 #undef __FUNCT__
6364 #define __FUNCT__ "DMPlexGetConeSection"
6365 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
6366 {
6367   DM_Plex *mesh = (DM_Plex*) dm->data;
6368 
6369   PetscFunctionBegin;
6370   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6371   if (section) *section = mesh->coneSection;
6372   PetscFunctionReturn(0);
6373 }
6374 
6375 #undef __FUNCT__
6376 #define __FUNCT__ "DMPlexGetCones"
6377 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
6378 {
6379   DM_Plex *mesh = (DM_Plex*) dm->data;
6380 
6381   PetscFunctionBegin;
6382   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6383   if (cones) *cones = mesh->cones;
6384   PetscFunctionReturn(0);
6385 }
6386 
6387 #undef __FUNCT__
6388 #define __FUNCT__ "DMPlexGetConeOrientations"
6389 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
6390 {
6391   DM_Plex *mesh = (DM_Plex*) dm->data;
6392 
6393   PetscFunctionBegin;
6394   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6395   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
6396   PetscFunctionReturn(0);
6397 }
6398 
6399 /******************************** FEM Support **********************************/
6400 
6401 #undef __FUNCT__
6402 #define __FUNCT__ "DMPlexVecGetClosure"
6403 /*@C
6404   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
6405 
6406   Not collective
6407 
6408   Input Parameters:
6409 + dm - The DM
6410 . section - The section describing the layout in v, or NULL to use the default section
6411 . v - The local vector
6412 - point - The sieve point in the DM
6413 
6414   Output Parameters:
6415 + csize - The number of values in the closure, or NULL
6416 - values - The array of values, which is a borrowed array and should not be freed
6417 
6418   Fortran Notes:
6419   Since it returns an array, this routine is only available in Fortran 90, and you must
6420   include petsc.h90 in your code.
6421 
6422   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
6423 
6424   Level: intermediate
6425 
6426 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
6427 @*/
6428 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6429 {
6430   PetscScalar   *array, *vArray;
6431   PetscInt      *points = NULL;
6432   PetscInt       offsets[32];
6433   PetscInt       depth, numFields, size = 0, numPoints, pStart, pEnd, p, q, f;
6434   PetscErrorCode ierr;
6435 
6436   PetscFunctionBegin;
6437   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6438   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6439   if (!section) {
6440     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
6441   }
6442   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6443   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6444   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6445   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6446   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
6447   if (depth == 1 && numFields < 2) {
6448     const PetscInt *cone, *coneO;
6449 
6450     ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6451     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6452     ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6453     if (!*values) {
6454       if ((point >= pStart) && (point < pEnd)) {
6455         PetscInt dof;
6456         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6457         size += dof;
6458       }
6459       for (p = 0; p < numPoints; ++p) {
6460         const PetscInt cp = cone[p];
6461         PetscInt       dof;
6462 
6463         if ((cp < pStart) || (cp >= pEnd)) continue;
6464         ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6465         size += dof;
6466       }
6467       ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
6468     } else {
6469       array = *values;
6470     }
6471     size = 0;
6472     ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
6473     if ((point >= pStart) && (point < pEnd)) {
6474       PetscInt     dof, off, d;
6475       PetscScalar *varr;
6476       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6477       ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6478       varr = &vArray[off];
6479       for (d = 0; d < dof; ++d, ++offsets[0]) {
6480         array[offsets[0]] = varr[d];
6481       }
6482       size += dof;
6483     }
6484     for (p = 0; p < numPoints; ++p) {
6485       const PetscInt cp = cone[p];
6486       PetscInt       o  = coneO[p];
6487       PetscInt       dof, off, d;
6488       PetscScalar   *varr;
6489 
6490       if ((cp < pStart) || (cp >= pEnd)) continue;
6491       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6492       ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
6493       varr = &vArray[off];
6494       if (o >= 0) {
6495         for (d = 0; d < dof; ++d, ++offsets[0]) {
6496           array[offsets[0]] = varr[d];
6497         }
6498       } else {
6499         for (d = dof-1; d >= 0; --d, ++offsets[0]) {
6500           array[offsets[0]] = varr[d];
6501         }
6502       }
6503       size += dof;
6504     }
6505     ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
6506     if (!*values) {
6507       if (csize) *csize = size;
6508       *values = array;
6509     } else {
6510       if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
6511       *csize = size;
6512     }
6513     PetscFunctionReturn(0);
6514   }
6515   ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6516   /* Compress out points not in the section */
6517   for (p = 0, q = 0; p < numPoints*2; p += 2) {
6518     if ((points[p] >= pStart) && (points[p] < pEnd)) {
6519       points[q*2]   = points[p];
6520       points[q*2+1] = points[p+1];
6521       ++q;
6522     }
6523   }
6524   numPoints = q;
6525   for (p = 0, size = 0; p < numPoints*2; p += 2) {
6526     PetscInt dof, fdof;
6527 
6528     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6529     for (f = 0; f < numFields; ++f) {
6530       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6531       offsets[f+1] += fdof;
6532     }
6533     size += dof;
6534   }
6535   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6536   if (numFields && offsets[numFields] != size) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], size);
6537   ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
6538   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
6539   for (p = 0; p < numPoints*2; p += 2) {
6540     PetscInt     o = points[p+1];
6541     PetscInt     dof, off, d;
6542     PetscScalar *varr;
6543 
6544     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6545     ierr = PetscSectionGetOffset(section, points[p], &off);CHKERRQ(ierr);
6546     varr = &vArray[off];
6547     if (numFields) {
6548       PetscInt fdof, foff, fcomp, f, c;
6549 
6550       for (f = 0, foff = 0; f < numFields; ++f) {
6551         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6552         if (o >= 0) {
6553           for (d = 0; d < fdof; ++d, ++offsets[f]) {
6554             array[offsets[f]] = varr[foff+d];
6555           }
6556         } else {
6557           ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
6558           for (d = fdof/fcomp-1; d >= 0; --d) {
6559             for (c = 0; c < fcomp; ++c, ++offsets[f]) {
6560               array[offsets[f]] = varr[foff+d*fcomp+c];
6561             }
6562           }
6563         }
6564         foff += fdof;
6565       }
6566     } else {
6567       if (o >= 0) {
6568         for (d = 0; d < dof; ++d, ++offsets[0]) {
6569           array[offsets[0]] = varr[d];
6570         }
6571       } else {
6572         for (d = dof-1; d >= 0; --d, ++offsets[0]) {
6573           array[offsets[0]] = varr[d];
6574         }
6575       }
6576     }
6577   }
6578   ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6579   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
6580   if (csize) *csize = size;
6581   *values = array;
6582   PetscFunctionReturn(0);
6583 }
6584 
6585 #undef __FUNCT__
6586 #define __FUNCT__ "DMPlexVecRestoreClosure"
6587 /*@C
6588   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
6589 
6590   Not collective
6591 
6592   Input Parameters:
6593 + dm - The DM
6594 . section - The section describing the layout in v, or NULL to use the default section
6595 . v - The local vector
6596 . point - The sieve point in the DM
6597 . csize - The number of values in the closure, or NULL
6598 - values - The array of values, which is a borrowed array and should not be freed
6599 
6600   Fortran Notes:
6601   Since it returns an array, this routine is only available in Fortran 90, and you must
6602   include petsc.h90 in your code.
6603 
6604   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
6605 
6606   Level: intermediate
6607 
6608 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
6609 @*/
6610 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6611 {
6612   PetscInt       size = 0;
6613   PetscErrorCode ierr;
6614 
6615   PetscFunctionBegin;
6616   /* Should work without recalculating size */
6617   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
6618   PetscFunctionReturn(0);
6619 }
6620 
6621 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
6622 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
6623 
6624 #undef __FUNCT__
6625 #define __FUNCT__ "updatePoint_private"
6626 PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
6627 {
6628   PetscInt        cdof;   /* The number of constraints on this point */
6629   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6630   PetscScalar    *a;
6631   PetscInt        off, cind = 0, k;
6632   PetscErrorCode  ierr;
6633 
6634   PetscFunctionBegin;
6635   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6636   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6637   a    = &array[off];
6638   if (!cdof || setBC) {
6639     if (orientation >= 0) {
6640       for (k = 0; k < dof; ++k) {
6641         fuse(&a[k], values[k]);
6642       }
6643     } else {
6644       for (k = 0; k < dof; ++k) {
6645         fuse(&a[k], values[dof-k-1]);
6646       }
6647     }
6648   } else {
6649     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6650     if (orientation >= 0) {
6651       for (k = 0; k < dof; ++k) {
6652         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6653         fuse(&a[k], values[k]);
6654       }
6655     } else {
6656       for (k = 0; k < dof; ++k) {
6657         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6658         fuse(&a[k], values[dof-k-1]);
6659       }
6660     }
6661   }
6662   PetscFunctionReturn(0);
6663 }
6664 
6665 #undef __FUNCT__
6666 #define __FUNCT__ "updatePointFields_private"
6667 PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt foffs[], void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
6668 {
6669   PetscScalar   *a;
6670   PetscInt       numFields, off, foff, f;
6671   PetscErrorCode ierr;
6672 
6673   PetscFunctionBegin;
6674   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6675   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6676   a    = &array[off];
6677   for (f = 0, foff = 0; f < numFields; ++f) {
6678     PetscInt        fdof, fcomp, fcdof;
6679     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6680     PetscInt        cind = 0, k, c;
6681 
6682     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
6683     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6684     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6685     if (!fcdof || setBC) {
6686       if (orientation >= 0) {
6687         for (k = 0; k < fdof; ++k) {
6688           fuse(&a[foff+k], values[foffs[f]+k]);
6689         }
6690       } else {
6691         for (k = fdof/fcomp-1; k >= 0; --k) {
6692           for (c = 0; c < fcomp; ++c) {
6693             fuse(&a[foff+(fdof/fcomp-1-k)*fcomp+c], values[foffs[f]+k*fcomp+c]);
6694           }
6695         }
6696       }
6697     } else {
6698       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6699       if (orientation >= 0) {
6700         for (k = 0; k < fdof; ++k) {
6701           if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
6702           fuse(&a[foff+k], values[foffs[f]+k]);
6703         }
6704       } else {
6705         for (k = fdof/fcomp-1; k >= 0; --k) {
6706           for (c = 0; c < fcomp; ++c) {
6707             if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
6708             fuse(&a[foff+(fdof/fcomp-1-k)*fcomp+c], values[foffs[f]+k*fcomp+c]);
6709           }
6710         }
6711       }
6712     }
6713     foff     += fdof;
6714     foffs[f] += fdof;
6715   }
6716   PetscFunctionReturn(0);
6717 }
6718 
6719 #undef __FUNCT__
6720 #define __FUNCT__ "DMPlexVecSetClosure"
6721 /*@C
6722   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6723 
6724   Not collective
6725 
6726   Input Parameters:
6727 + dm - The DM
6728 . section - The section describing the layout in v, or NULL to use the default section
6729 . v - The local vector
6730 . point - The sieve point in the DM
6731 . values - The array of values
6732 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
6733 
6734   Fortran Notes:
6735   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6736 
6737   Level: intermediate
6738 
6739 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6740 @*/
6741 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6742 {
6743   PetscScalar   *array;
6744   PetscInt      *points = NULL;
6745   PetscInt       offsets[32];
6746   PetscInt       depth, numFields, numPoints, off, dof, pStart, pEnd, p, q, f;
6747   PetscErrorCode ierr;
6748 
6749   PetscFunctionBegin;
6750   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6751   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6752   if (!section) {
6753     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
6754   }
6755   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6756   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6757   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6758   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6759   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
6760   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6761     const PetscInt *cone, *coneO;
6762 
6763     ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6764     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6765     ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6766     ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6767     for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6768       const PetscInt cp = !p ? point : cone[p-1];
6769       const PetscInt o  = !p ? 0     : coneO[p-1];
6770 
6771       if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6772       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6773       /* ADD_VALUES */
6774       {
6775         const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6776         PetscScalar    *a;
6777         PetscInt        cdof, coff, cind = 0, k;
6778 
6779         ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6780         ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6781         a    = &array[coff];
6782         if (!cdof) {
6783           if (o >= 0) {
6784             for (k = 0; k < dof; ++k) {
6785               a[k] += values[off+k];
6786             }
6787           } else {
6788             for (k = 0; k < dof; ++k) {
6789               a[k] += values[off+dof-k-1];
6790             }
6791           }
6792         } else {
6793           ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6794           if (o >= 0) {
6795             for (k = 0; k < dof; ++k) {
6796               if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6797               a[k] += values[off+k];
6798             }
6799           } else {
6800             for (k = 0; k < dof; ++k) {
6801               if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6802               a[k] += values[off+dof-k-1];
6803             }
6804           }
6805         }
6806       }
6807     }
6808     ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6809     PetscFunctionReturn(0);
6810   }
6811   ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6812   /* Compress out points not in the section */
6813   for (p = 0, q = 0; p < numPoints*2; p += 2) {
6814     if ((points[p] >= pStart) && (points[p] < pEnd)) {
6815       points[q*2]   = points[p];
6816       points[q*2+1] = points[p+1];
6817       ++q;
6818     }
6819   }
6820   numPoints = q;
6821   for (p = 0; p < numPoints*2; p += 2) {
6822     PetscInt fdof;
6823 
6824     for (f = 0; f < numFields; ++f) {
6825       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6826       offsets[f+1] += fdof;
6827     }
6828   }
6829   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6830   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6831   if (numFields) {
6832     switch (mode) {
6833     case INSERT_VALUES:
6834       for (p = 0; p < numPoints*2; p += 2) {
6835         PetscInt o = points[p+1];
6836         updatePointFields_private(section, points[p], offsets, insert, PETSC_FALSE, o, values, array);
6837       } break;
6838     case INSERT_ALL_VALUES:
6839       for (p = 0; p < numPoints*2; p += 2) {
6840         PetscInt o = points[p+1];
6841         updatePointFields_private(section, points[p], offsets, insert, PETSC_TRUE,  o, values, array);
6842       } break;
6843     case ADD_VALUES:
6844       for (p = 0; p < numPoints*2; p += 2) {
6845         PetscInt o = points[p+1];
6846         updatePointFields_private(section, points[p], offsets, add,    PETSC_FALSE, o, values, array);
6847       } break;
6848     case ADD_ALL_VALUES:
6849       for (p = 0; p < numPoints*2; p += 2) {
6850         PetscInt o = points[p+1];
6851         updatePointFields_private(section, points[p], offsets, add,    PETSC_TRUE,  o, values, array);
6852       } break;
6853     default:
6854       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
6855     }
6856   } else {
6857     switch (mode) {
6858     case INSERT_VALUES:
6859       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6860         PetscInt o = points[p+1];
6861         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6862         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
6863       } break;
6864     case INSERT_ALL_VALUES:
6865       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6866         PetscInt o = points[p+1];
6867         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6868         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
6869       } break;
6870     case ADD_VALUES:
6871       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6872         PetscInt o = points[p+1];
6873         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6874         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
6875       } break;
6876     case ADD_ALL_VALUES:
6877       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6878         PetscInt o = points[p+1];
6879         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6880         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
6881       } break;
6882     default:
6883       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
6884     }
6885   }
6886   ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6887   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6888   PetscFunctionReturn(0);
6889 }
6890 
6891 #undef __FUNCT__
6892 #define __FUNCT__ "DMPlexPrintMatSetValues"
6893 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numIndices, const PetscInt indices[], const PetscScalar values[])
6894 {
6895   PetscMPIInt    rank;
6896   PetscInt       i, j;
6897   PetscErrorCode ierr;
6898 
6899   PetscFunctionBegin;
6900   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
6901   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
6902   for (i = 0; i < numIndices; i++) {
6903     ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat indices[%D] = %D\n", rank, i, indices[i]);CHKERRQ(ierr);
6904   }
6905   for (i = 0; i < numIndices; i++) {
6906     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
6907     for (j = 0; j < numIndices; j++) {
6908 #if defined(PETSC_USE_COMPLEX)
6909       ierr = PetscViewerASCIIPrintf(viewer, " (%G,%G)", PetscRealPart(values[i*numIndices+j]), PetscImaginaryPart(values[i*numIndices+j]));CHKERRQ(ierr);
6910 #else
6911       ierr = PetscViewerASCIIPrintf(viewer, " %G", values[i*numIndices+j]);CHKERRQ(ierr);
6912 #endif
6913     }
6914     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6915   }
6916   PetscFunctionReturn(0);
6917 }
6918 
6919 #undef __FUNCT__
6920 #define __FUNCT__ "indicesPoint_private"
6921 /* . off - The global offset of this point */
6922 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
6923 {
6924   PetscInt        dof;    /* The number of unknowns on this point */
6925   PetscInt        cdof;   /* The number of constraints on this point */
6926   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6927   PetscInt        cind = 0, k;
6928   PetscErrorCode  ierr;
6929 
6930   PetscFunctionBegin;
6931   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6932   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6933   if (!cdof || setBC) {
6934     if (orientation >= 0) {
6935       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
6936     } else {
6937       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
6938     }
6939   } else {
6940     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6941     if (orientation >= 0) {
6942       for (k = 0; k < dof; ++k) {
6943         if ((cind < cdof) && (k == cdofs[cind])) {
6944           /* Insert check for returning constrained indices */
6945           indices[*loff+k] = -(off+k+1);
6946           ++cind;
6947         } else {
6948           indices[*loff+k] = off+k-cind;
6949         }
6950       }
6951     } else {
6952       for (k = 0; k < dof; ++k) {
6953         if ((cind < cdof) && (k == cdofs[cind])) {
6954           /* Insert check for returning constrained indices */
6955           indices[*loff+dof-k-1] = -(off+k+1);
6956           ++cind;
6957         } else {
6958           indices[*loff+dof-k-1] = off+k-cind;
6959         }
6960       }
6961     }
6962   }
6963   *loff += dof;
6964   PetscFunctionReturn(0);
6965 }
6966 
6967 #undef __FUNCT__
6968 #define __FUNCT__ "indicesPointFields_private"
6969 /* . off - The global offset of this point */
6970 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
6971 {
6972   PetscInt       numFields, foff, f;
6973   PetscErrorCode ierr;
6974 
6975   PetscFunctionBegin;
6976   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6977   for (f = 0, foff = 0; f < numFields; ++f) {
6978     PetscInt        fdof, fcomp, cfdof;
6979     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6980     PetscInt        cind = 0, k, c;
6981 
6982     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
6983     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6984     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6985     if (!cfdof || setBC) {
6986       if (orientation >= 0) {
6987         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
6988       } else {
6989         for (k = fdof/fcomp-1; k >= 0; --k) {
6990           for (c = 0; c < fcomp; ++c) {
6991             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
6992           }
6993         }
6994       }
6995     } else {
6996       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6997       if (orientation >= 0) {
6998         for (k = 0; k < fdof; ++k) {
6999           if ((cind < cfdof) && (k == fcdofs[cind])) {
7000             indices[foffs[f]+k] = -(off+foff+k+1);
7001             ++cind;
7002           } else {
7003             indices[foffs[f]+k] = off+foff+k-cind;
7004           }
7005         }
7006       } else {
7007         for (k = fdof/fcomp-1; k >= 0; --k) {
7008           for (c = 0; c < fcomp; ++c) {
7009             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
7010               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
7011               ++cind;
7012             } else {
7013               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
7014             }
7015           }
7016         }
7017       }
7018     }
7019     foff     += fdof - cfdof;
7020     foffs[f] += fdof;
7021   }
7022   PetscFunctionReturn(0);
7023 }
7024 
7025 #undef __FUNCT__
7026 #define __FUNCT__ "DMPlexMatSetClosure"
7027 /*@C
7028   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7029 
7030   Not collective
7031 
7032   Input Parameters:
7033 + dm - The DM
7034 . section - The section describing the layout in v
7035 . globalSection - The section describing the layout in v
7036 . A - The matrix
7037 . point - The sieve point in the DM
7038 . values - The array of values
7039 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7040 
7041   Fortran Notes:
7042   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7043 
7044   Level: intermediate
7045 
7046 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
7047 @*/
7048 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7049 {
7050   DM_Plex       *mesh   = (DM_Plex*) dm->data;
7051   PetscInt      *points = NULL;
7052   PetscInt      *indices;
7053   PetscInt       offsets[32];
7054   PetscInt       numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
7055   PetscErrorCode ierr;
7056 
7057   PetscFunctionBegin;
7058   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7059   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7060   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7061   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7062   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
7063   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7064   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
7065   ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
7066   /* Compress out points not in the section */
7067   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
7068   for (p = 0, q = 0; p < numPoints*2; p += 2) {
7069     if ((points[p] >= pStart) && (points[p] < pEnd)) {
7070       points[q*2]   = points[p];
7071       points[q*2+1] = points[p+1];
7072       ++q;
7073     }
7074   }
7075   numPoints = q;
7076   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
7077     PetscInt fdof;
7078 
7079     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7080     for (f = 0; f < numFields; ++f) {
7081       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7082       offsets[f+1] += fdof;
7083     }
7084     numIndices += dof;
7085   }
7086   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
7087 
7088   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
7089   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
7090   if (numFields) {
7091     for (p = 0; p < numPoints*2; p += 2) {
7092       PetscInt o = points[p+1];
7093       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
7094       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
7095     }
7096   } else {
7097     for (p = 0, off = 0; p < numPoints*2; p += 2) {
7098       PetscInt o = points[p+1];
7099       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
7100       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
7101     }
7102   }
7103   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, values);CHKERRQ(ierr);}
7104   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7105   if (ierr) {
7106     PetscMPIInt    rank;
7107     PetscErrorCode ierr2;
7108 
7109     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
7110     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7111     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, values);CHKERRQ(ierr2);
7112     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
7113     CHKERRQ(ierr);
7114   }
7115   ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
7116   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
7117   PetscFunctionReturn(0);
7118 }
7119 
7120 #undef __FUNCT__
7121 #define __FUNCT__ "DMPlexGetHybridBounds"
7122 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
7123 {
7124   DM_Plex       *mesh = (DM_Plex*) dm->data;
7125   PetscInt       dim;
7126   PetscErrorCode ierr;
7127 
7128   PetscFunctionBegin;
7129   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7130   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
7131   if (cMax) *cMax = mesh->hybridPointMax[dim];
7132   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
7133   if (eMax) *eMax = mesh->hybridPointMax[1];
7134   if (vMax) *vMax = mesh->hybridPointMax[0];
7135   PetscFunctionReturn(0);
7136 }
7137 
7138 #undef __FUNCT__
7139 #define __FUNCT__ "DMPlexSetHybridBounds"
7140 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
7141 {
7142   DM_Plex       *mesh = (DM_Plex*) dm->data;
7143   PetscInt       dim;
7144   PetscErrorCode ierr;
7145 
7146   PetscFunctionBegin;
7147   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7148   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
7149   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
7150   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
7151   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
7152   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
7153   PetscFunctionReturn(0);
7154 }
7155 
7156 #undef __FUNCT__
7157 #define __FUNCT__ "DMPlexGetVTKCellHeight"
7158 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7159 {
7160   DM_Plex *mesh = (DM_Plex*) dm->data;
7161 
7162   PetscFunctionBegin;
7163   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7164   PetscValidPointer(cellHeight, 2);
7165   *cellHeight = mesh->vtkCellHeight;
7166   PetscFunctionReturn(0);
7167 }
7168 
7169 #undef __FUNCT__
7170 #define __FUNCT__ "DMPlexSetVTKCellHeight"
7171 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7172 {
7173   DM_Plex *mesh = (DM_Plex*) dm->data;
7174 
7175   PetscFunctionBegin;
7176   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7177   mesh->vtkCellHeight = cellHeight;
7178   PetscFunctionReturn(0);
7179 }
7180 
7181 #undef __FUNCT__
7182 #define __FUNCT__ "DMPlexCreateNumbering_Private"
7183 /* We can easily have a form that takes an IS instead */
7184 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
7185 {
7186   PetscSection   section, globalSection;
7187   PetscInt      *numbers, p;
7188   PetscErrorCode ierr;
7189 
7190   PetscFunctionBegin;
7191   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7192   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7193   for (p = pStart; p < pEnd; ++p) {
7194     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7195   }
7196   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7197   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7198   ierr = PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &numbers);CHKERRQ(ierr);
7199   for (p = pStart; p < pEnd; ++p) {
7200     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7201   }
7202   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7203   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7204   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7205   PetscFunctionReturn(0);
7206 }
7207 
7208 #undef __FUNCT__
7209 #define __FUNCT__ "DMPlexGetCellNumbering"
7210 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7211 {
7212   DM_Plex       *mesh = (DM_Plex*) dm->data;
7213   PetscInt       cellHeight, cStart, cEnd, cMax;
7214   PetscErrorCode ierr;
7215 
7216   PetscFunctionBegin;
7217   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7218   if (!mesh->globalCellNumbers) {
7219     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7220     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
7221     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7222     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
7223     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
7224   }
7225   *globalCellNumbers = mesh->globalCellNumbers;
7226   PetscFunctionReturn(0);
7227 }
7228 
7229 #undef __FUNCT__
7230 #define __FUNCT__ "DMPlexGetVertexNumbering"
7231 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7232 {
7233   DM_Plex       *mesh = (DM_Plex*) dm->data;
7234   PetscInt       vStart, vEnd, vMax;
7235   PetscErrorCode ierr;
7236 
7237   PetscFunctionBegin;
7238   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7239   if (!mesh->globalVertexNumbers) {
7240     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7241     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
7242     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
7243     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
7244   }
7245   *globalVertexNumbers = mesh->globalVertexNumbers;
7246   PetscFunctionReturn(0);
7247 }
7248 
7249 
7250 #undef __FUNCT__
7251 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
7252 /*@C
7253   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
7254   the local section and an SF describing the section point overlap.
7255 
7256   Input Parameters:
7257   + s - The PetscSection for the local field layout
7258   . sf - The SF describing parallel layout of the section points
7259   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
7260   . label - The label specifying the points
7261   - labelValue - The label stratum specifying the points
7262 
7263   Output Parameter:
7264   . gsection - The PetscSection for the global field layout
7265 
7266   Note: This gives negative sizes and offsets to points not owned by this process
7267 
7268   Level: developer
7269 
7270 .seealso: PetscSectionCreate()
7271 @*/
7272 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
7273 {
7274   PetscInt      *neg = NULL, *tmpOff = NULL;
7275   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
7276   PetscErrorCode ierr;
7277 
7278   PetscFunctionBegin;
7279   ierr = PetscSectionCreate(s->atlasLayout.comm, gsection);CHKERRQ(ierr);
7280   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
7281   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
7282   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
7283   if (nroots >= 0) {
7284     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
7285     ierr = PetscMalloc(nroots * sizeof(PetscInt), &neg);CHKERRQ(ierr);
7286     ierr = PetscMemzero(neg, nroots * sizeof(PetscInt));CHKERRQ(ierr);
7287     if (nroots > pEnd-pStart) {
7288       ierr = PetscMalloc(nroots * sizeof(PetscInt), &tmpOff);CHKERRQ(ierr);
7289       ierr = PetscMemzero(tmpOff, nroots * sizeof(PetscInt));CHKERRQ(ierr);
7290     } else {
7291       tmpOff = &(*gsection)->atlasDof[-pStart];
7292     }
7293   }
7294   /* Mark ghost points with negative dof */
7295   for (p = pStart; p < pEnd; ++p) {
7296     PetscInt value;
7297 
7298     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
7299     if (value != labelValue) continue;
7300     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
7301     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
7302     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
7303     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
7304     if (neg) neg[p] = -(dof+1);
7305   }
7306   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
7307   if (nroots >= 0) {
7308     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7309     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7310     if (nroots > pEnd-pStart) {
7311       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
7312     }
7313   }
7314   /* Calculate new sizes, get proccess offset, and calculate point offsets */
7315   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
7316     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
7317     (*gsection)->atlasOff[p] = off;
7318     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
7319   }
7320   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, s->atlasLayout.comm);CHKERRQ(ierr);
7321   globalOff -= off;
7322   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
7323     (*gsection)->atlasOff[p] += globalOff;
7324     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
7325   }
7326   /* Put in negative offsets for ghost points */
7327   if (nroots >= 0) {
7328     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7329     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7330     if (nroots > pEnd-pStart) {
7331       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
7332     }
7333   }
7334   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
7335   ierr = PetscFree(neg);CHKERRQ(ierr);
7336   PetscFunctionReturn(0);
7337 }
7338