xref: /petsc/src/dm/impls/plex/plex.c (revision 238e5e170671a8e6856da8ddb6395fb529b04210) !
1 #include <petsc-private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2 #include <../src/sys/utils/hash.h>
3 #include <petsc-private/isimpl.h>
4 #include <petscsf.h>
5 
6 /* Logging support */
7 PetscLogEvent DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_Stratify;
8 
9 PETSC_EXTERN PetscErrorCode VecView_Seq(Vec, PetscViewer);
10 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
11 
12 #undef __FUNCT__
13 #define __FUNCT__ "VecView_Plex_Local"
14 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
15 {
16   DM             dm;
17   PetscBool      isvtk;
18   PetscErrorCode ierr;
19 
20   PetscFunctionBegin;
21   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
22   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
23   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);CHKERRQ(ierr);
24   if (isvtk) {
25     PetscViewerVTKFieldType ft = PETSC_VTK_POINT_FIELD;
26     PetscSection            section;
27     PetscInt                dim, pStart, pEnd, cStart, fStart, vStart, cdof = 0, fdof = 0, vdof = 0;
28 
29     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
30     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
31     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
32     ierr = DMPlexGetHeightStratum(dm, 1, &fStart, NULL);CHKERRQ(ierr);
33     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, NULL);CHKERRQ(ierr);
34     ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
35     /* Assumes that numer of dofs per point of each stratum is constant, natural for VTK */
36     if ((cStart >= pStart) && (cStart < pEnd)) {ierr = PetscSectionGetDof(section, cStart, &cdof);CHKERRQ(ierr);}
37     if ((fStart >= pStart) && (fStart < pEnd)) {ierr = PetscSectionGetDof(section, fStart, &fdof);CHKERRQ(ierr);}
38     if ((vStart >= pStart) && (vStart < pEnd)) {ierr = PetscSectionGetDof(section, vStart, &vdof);CHKERRQ(ierr);}
39     if (cdof && fdof && vdof) { /* Actually Q2 or some such, but visualize as Q1 */
40       ft = (cdof == dim) ? PETSC_VTK_POINT_VECTOR_FIELD : PETSC_VTK_POINT_FIELD;
41     } else if (cdof && vdof) {
42       SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"No support for viewing mixed space with dofs at both vertices and cells");
43     } else if (cdof) {
44       /* TODO: This assumption should be removed when there is a way of identifying whether a space is conceptually a
45        * vector or just happens to have the same number of dofs as the dimension. */
46       if (cdof == dim) {
47         ft = PETSC_VTK_CELL_VECTOR_FIELD;
48       } else {
49         ft = PETSC_VTK_CELL_FIELD;
50       }
51     } else if (vdof) {
52       if (vdof == dim) {
53         ft = PETSC_VTK_POINT_VECTOR_FIELD;
54       } else {
55         ft = PETSC_VTK_POINT_FIELD;
56       }
57     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Could not classify input Vec for VTK");
58 
59     ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr); /* viewer drops reference */
60     ierr = PetscObjectReference((PetscObject) v);CHKERRQ(ierr);  /* viewer drops reference */
61     ierr = PetscViewerVTKAddField(viewer, (PetscObject) dm, DMPlexVTKWriteAll, ft, (PetscObject) v);CHKERRQ(ierr);
62   } else {
63     PetscBool isseq;
64 
65     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
66     if (isseq) {
67       ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);
68     } else {
69       ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);
70     }
71   }
72   PetscFunctionReturn(0);
73 }
74 
75 #undef __FUNCT__
76 #define __FUNCT__ "VecView_Plex"
77 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
78 {
79   DM             dm;
80   PetscBool      isvtk;
81   PetscErrorCode ierr;
82 
83   PetscFunctionBegin;
84   ierr = VecGetDM(v, &dm);CHKERRQ(ierr);
85   if (!dm) SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
86   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK, &isvtk);CHKERRQ(ierr);
87   if (isvtk) {
88     Vec         locv;
89     const char *name;
90 
91     ierr = DMGetLocalVector(dm, &locv);CHKERRQ(ierr);
92     ierr = PetscObjectGetName((PetscObject) v, &name);CHKERRQ(ierr);
93     ierr = PetscObjectSetName((PetscObject) locv, name);CHKERRQ(ierr);
94     ierr = DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
95     ierr = DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv);CHKERRQ(ierr);
96     ierr = VecView_Plex_Local(locv, viewer);CHKERRQ(ierr);
97     ierr = DMRestoreLocalVector(dm, &locv);CHKERRQ(ierr);
98   } else {
99     PetscBool isseq;
100 
101     ierr = PetscObjectTypeCompare((PetscObject) v, VECSEQ, &isseq);CHKERRQ(ierr);
102     if (isseq) {
103       ierr = VecView_Seq(v, viewer);CHKERRQ(ierr);
104     } else {
105       ierr = VecView_MPI(v, viewer);CHKERRQ(ierr);
106     }
107   }
108   PetscFunctionReturn(0);
109 }
110 
111 #undef __FUNCT__
112 #define __FUNCT__ "DMPlexView_Ascii"
113 PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
114 {
115   DM_Plex          *mesh = (DM_Plex*) dm->data;
116   DM                cdm;
117   DMLabel           markers;
118   PetscSection      coordSection;
119   Vec               coordinates;
120   PetscViewerFormat format;
121   PetscErrorCode    ierr;
122 
123   PetscFunctionBegin;
124   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
125   ierr = DMGetDefaultSection(cdm, &coordSection);CHKERRQ(ierr);
126   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
127   ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
128   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
129     const char *name;
130     PetscInt    maxConeSize, maxSupportSize;
131     PetscInt    pStart, pEnd, p;
132     PetscMPIInt rank, size;
133 
134     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
135     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
136     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
137     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
138     ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
139     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
140     ierr = PetscViewerASCIIPrintf(viewer, "Mesh '%s':\n", name);CHKERRQ(ierr);
141     ierr = PetscViewerASCIISynchronizedPrintf(viewer, "Max sizes cone: %D support: %D\n", maxConeSize, maxSupportSize);CHKERRQ(ierr);
142     ierr = PetscViewerASCIIPrintf(viewer, "orientation is missing\n", name);CHKERRQ(ierr);
143     ierr = PetscViewerASCIIPrintf(viewer, "cap --> base:\n", name);CHKERRQ(ierr);
144     for (p = pStart; p < pEnd; ++p) {
145       PetscInt dof, off, s;
146 
147       ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
148       ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
149       for (s = off; s < off+dof; ++s) {
150         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D ----> %D\n", rank, p, mesh->supports[s]);CHKERRQ(ierr);
151       }
152     }
153     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
154     ierr = PetscViewerASCIIPrintf(viewer, "base <-- cap:\n", name);CHKERRQ(ierr);
155     for (p = pStart; p < pEnd; ++p) {
156       PetscInt dof, off, c;
157 
158       ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
159       ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
160       for (c = off; c < off+dof; ++c) {
161         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "[%D]: %D <---- %D (%D)\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]);CHKERRQ(ierr);
162       }
163     }
164     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
165     ierr = PetscSectionGetChart(coordSection, &pStart, NULL);CHKERRQ(ierr);
166     if (pStart >= 0) {ierr = PetscSectionVecView(coordSection, coordinates, viewer);CHKERRQ(ierr);}
167     ierr = DMPlexGetLabel(dm, "marker", &markers);CHKERRQ(ierr);
168     ierr = DMLabelView(markers,viewer);CHKERRQ(ierr);
169     if (size > 1) {
170       PetscSF sf;
171 
172       ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
173       ierr = PetscSFView(sf, viewer);CHKERRQ(ierr);
174     }
175     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
176   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
177     const char  *name;
178     const char  *colors[3] = {"red", "blue", "green"};
179     const int    numColors  = 3;
180     PetscReal    scale      = 2.0;
181     PetscScalar *coords;
182     PetscInt     depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, e, p;
183     PetscMPIInt  rank, size;
184 
185     ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
186     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
187     ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
188     ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
189     ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE);CHKERRQ(ierr);
190     ierr = PetscViewerASCIIPrintf(viewer, "\
191 \\documentclass[crop,multi=false]{standalone}\n\n\
192 \\usepackage{tikz}\n\
193 \\usepackage{pgflibraryshapes}\n\
194 \\usetikzlibrary{backgrounds}\n\
195 \\usetikzlibrary{arrows}\n\
196 \\begin{document}\n\
197 \\section{%s}\n\
198 \\begin{center}\n", name, 8.0/scale);CHKERRQ(ierr);
199     ierr = PetscViewerASCIIPrintf(viewer, "Mesh for process ");CHKERRQ(ierr);
200     for (p = 0; p < size; ++p) {
201       if (p > 0 && p == size-1) {
202         ierr = PetscViewerASCIIPrintf(viewer, ", and ", colors[p%numColors], p);CHKERRQ(ierr);
203       } else if (p > 0) {
204         ierr = PetscViewerASCIIPrintf(viewer, ", ", colors[p%numColors], p);CHKERRQ(ierr);
205       }
206       ierr = PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%D}", colors[p%numColors], p);CHKERRQ(ierr);
207     }
208     ierr = PetscViewerASCIIPrintf(viewer, ".\n\n\n\
209 \\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n");CHKERRQ(ierr);
210     /* Plot vertices */
211     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
212     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
213     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
214     for (v = vStart; v < vEnd; ++v) {
215       PetscInt off, dof, d;
216 
217       ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
218       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
219       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
220       for (d = 0; d < dof; ++d) {
221         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
222         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%G", scale*PetscRealPart(coords[off+d]));CHKERRQ(ierr);
223       }
224       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", v, rank, colors[rank%numColors], v);CHKERRQ(ierr);
225     }
226     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
227     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
228     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
229     /* Plot edges */
230     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
231     ierr = PetscViewerASCIIPrintf(viewer, "\\path\n");CHKERRQ(ierr);
232     if (depth > 1) {ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);}
233     for (e = eStart; e < eEnd; ++e) {
234       const PetscInt *cone;
235       PetscInt        coneSize, offA, offB, dof, d;
236 
237       ierr = DMPlexGetConeSize(dm, e, &coneSize);CHKERRQ(ierr);
238       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %d cone should have two vertices, not %d", e, coneSize);
239       ierr = DMPlexGetCone(dm, e, &cone);CHKERRQ(ierr);
240       ierr = PetscSectionGetDof(coordSection, cone[0], &dof);CHKERRQ(ierr);
241       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
242       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
243       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(");CHKERRQ(ierr);
244       for (d = 0; d < dof; ++d) {
245         if (d > 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, ",");CHKERRQ(ierr);}
246         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "%G", scale*0.5*PetscRealPart(coords[offA+d]+coords[offB+d]));CHKERRQ(ierr);
247       }
248       ierr = PetscViewerASCIISynchronizedPrintf(viewer, ") node(%D_%D) [draw,shape=circle,color=%s] {%D} --\n", e, rank, colors[rank%numColors], e);CHKERRQ(ierr);
249     }
250     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
251     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
252     ierr = PetscViewerASCIIPrintf(viewer, "(0,0);\n");CHKERRQ(ierr);
253     /* Plot cells */
254     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
255     for (c = cStart; c < cEnd; ++c) {
256       PetscInt *closure = NULL;
257       PetscInt  closureSize, firstPoint = -1;
258 
259       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
260       ierr = PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank%numColors]);CHKERRQ(ierr);
261       for (p = 0; p < closureSize*2; p += 2) {
262         const PetscInt point = closure[p];
263 
264         if ((point < vStart) || (point >= vEnd)) continue;
265         if (firstPoint >= 0) {ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- ");CHKERRQ(ierr);}
266         ierr = PetscViewerASCIISynchronizedPrintf(viewer, "(%D_%D)", point, rank);CHKERRQ(ierr);
267         if (firstPoint < 0) firstPoint = point;
268       }
269       /* Why doesn't this work? ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- cycle;\n");CHKERRQ(ierr); */
270       ierr = PetscViewerASCIISynchronizedPrintf(viewer, " -- (%D_%D);\n", firstPoint, rank);CHKERRQ(ierr);
271       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
272     }
273     ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
274     ierr = PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n\\end{center}\n");CHKERRQ(ierr);
275     ierr = PetscViewerASCIIPrintf(viewer, "\\end{document}\n", name);CHKERRQ(ierr);
276   } else {
277     MPI_Comm    comm;
278     PetscInt   *sizes;
279     PetscInt    locDepth, depth, dim, d;
280     PetscInt    pStart, pEnd, p;
281     PetscInt    numLabels, l;
282     PetscMPIInt size;
283 
284     ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
285     ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
286     ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
287     ierr = PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dim);CHKERRQ(ierr);
288     ierr = DMPlexGetDepth(dm, &locDepth);CHKERRQ(ierr);
289     ierr = MPI_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
290     ierr = PetscMalloc(size * sizeof(PetscInt), &sizes);CHKERRQ(ierr);
291     if (depth == 1) {
292       ierr = DMPlexGetDepthStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
293       pEnd = pEnd - pStart;
294       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
295       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", 0);CHKERRQ(ierr);
296       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
297       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
298       ierr = DMPlexGetHeightStratum(dm, 0, &pStart, &pEnd);CHKERRQ(ierr);
299       pEnd = pEnd - pStart;
300       ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
301       ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", dim);CHKERRQ(ierr);
302       for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
303       ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
304     } else {
305       for (d = 0; d <= dim; d++) {
306         ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
307         pEnd = pEnd - pStart;
308         ierr = MPI_Gather(&pEnd, 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
309         ierr = PetscViewerASCIIPrintf(viewer, "  %D-cells:", d);CHKERRQ(ierr);
310         for (p = 0; p < size; ++p) {ierr = PetscViewerASCIIPrintf(viewer, " %D", sizes[p]);CHKERRQ(ierr);}
311         ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
312       }
313     }
314     ierr = PetscFree(sizes);CHKERRQ(ierr);
315     ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
316     if (numLabels) {ierr = PetscViewerASCIIPrintf(viewer, "Labels:\n");CHKERRQ(ierr);}
317     for (l = 0; l < numLabels; ++l) {
318       DMLabel         label;
319       const char     *name;
320       IS              valueIS;
321       const PetscInt *values;
322       PetscInt        numValues, v;
323 
324       ierr = DMPlexGetLabelName(dm, l, &name);CHKERRQ(ierr);
325       ierr = DMPlexGetLabel(dm, name, &label);CHKERRQ(ierr);
326       ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
327       ierr = PetscViewerASCIIPrintf(viewer, "  %s: %d strata of sizes (", name, numValues);CHKERRQ(ierr);
328       ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
329       ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
330       for (v = 0; v < numValues; ++v) {
331         PetscInt size;
332 
333         ierr = DMLabelGetStratumSize(label, values[v], &size);CHKERRQ(ierr);
334         if (v > 0) {ierr = PetscViewerASCIIPrintf(viewer, ", ");CHKERRQ(ierr);}
335         ierr = PetscViewerASCIIPrintf(viewer, "%d", size);CHKERRQ(ierr);
336       }
337       ierr = PetscViewerASCIIPrintf(viewer, ")\n");CHKERRQ(ierr);
338       ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
339       ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
340     }
341   }
342   PetscFunctionReturn(0);
343 }
344 
345 #undef __FUNCT__
346 #define __FUNCT__ "DMView_Plex"
347 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
348 {
349   PetscBool      iascii, isbinary;
350   PetscErrorCode ierr;
351 
352   PetscFunctionBegin;
353   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
354   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
355   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);CHKERRQ(ierr);
356   ierr = PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERBINARY, &isbinary);CHKERRQ(ierr);
357   if (iascii) {
358     ierr = DMPlexView_Ascii(dm, viewer);CHKERRQ(ierr);
359 #if 0
360   } else if (isbinary) {
361     ierr = DMPlexView_Binary(dm, viewer);CHKERRQ(ierr);
362 #endif
363   }
364   PetscFunctionReturn(0);
365 }
366 
367 #undef __FUNCT__
368 #define __FUNCT__ "DMDestroy_Plex"
369 PetscErrorCode DMDestroy_Plex(DM dm)
370 {
371   DM_Plex       *mesh = (DM_Plex*) dm->data;
372   DMLabel        next  = mesh->labels;
373   PetscErrorCode ierr;
374 
375   PetscFunctionBegin;
376   if (--mesh->refct > 0) PetscFunctionReturn(0);
377   ierr = PetscSectionDestroy(&mesh->coneSection);CHKERRQ(ierr);
378   ierr = PetscFree(mesh->cones);CHKERRQ(ierr);
379   ierr = PetscFree(mesh->coneOrientations);CHKERRQ(ierr);
380   ierr = PetscSectionDestroy(&mesh->supportSection);CHKERRQ(ierr);
381   ierr = PetscFree(mesh->supports);CHKERRQ(ierr);
382   ierr = PetscFree(mesh->facesTmp);CHKERRQ(ierr);
383   while (next) {
384     DMLabel tmp = next->next;
385 
386     ierr = DMLabelDestroy(&next);CHKERRQ(ierr);
387     next = tmp;
388   }
389   ierr = DMLabelDestroy(&mesh->subpointMap);CHKERRQ(ierr);
390   ierr = ISDestroy(&mesh->globalVertexNumbers);CHKERRQ(ierr);
391   ierr = ISDestroy(&mesh->globalCellNumbers);CHKERRQ(ierr);
392   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
393   ierr = PetscFree(mesh);CHKERRQ(ierr);
394   PetscFunctionReturn(0);
395 }
396 
397 #undef __FUNCT__
398 #define __FUNCT__ "DMCreateMatrix_Plex"
399 PetscErrorCode DMCreateMatrix_Plex(DM dm, MatType mtype, Mat *J)
400 {
401   PetscSection   section, sectionGlobal;
402   PetscInt       bs = -1;
403   PetscInt       localSize;
404   PetscBool      isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock;
405   PetscErrorCode ierr;
406 
407   PetscFunctionBegin;
408 #if !defined(PETSC_USE_DYNAMIC_LIBRARIES)
409   ierr = MatInitializePackage();CHKERRQ(ierr);
410 #endif
411   if (!mtype) mtype = MATAIJ;
412   ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
413   ierr = DMGetDefaultGlobalSection(dm, &sectionGlobal);CHKERRQ(ierr);
414   /* ierr = PetscSectionGetStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr); */
415   ierr = PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);CHKERRQ(ierr);
416   ierr = MatCreate(PetscObjectComm((PetscObject)dm), J);CHKERRQ(ierr);
417   ierr = MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
418   ierr = MatSetType(*J, mtype);CHKERRQ(ierr);
419   ierr = MatSetFromOptions(*J);CHKERRQ(ierr);
420   ierr = PetscStrcmp(mtype, MATSHELL, &isShell);CHKERRQ(ierr);
421   ierr = PetscStrcmp(mtype, MATBAIJ, &isBlock);CHKERRQ(ierr);
422   ierr = PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);CHKERRQ(ierr);
423   ierr = PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);CHKERRQ(ierr);
424   ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
425   ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
426   ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
427   if (!isShell) {
428     PetscBool fillMatrix = (PetscBool) !dm->prealloc_only;
429     PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal, bsMax, bsMin;
430 
431     if (bs < 0) {
432       if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
433         PetscInt pStart, pEnd, p, dof, cdof;
434 
435         ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
436         for (p = pStart; p < pEnd; ++p) {
437           ierr = PetscSectionGetDof(sectionGlobal, p, &dof);CHKERRQ(ierr);
438           ierr = PetscSectionGetConstraintDof(sectionGlobal, p, &cdof);CHKERRQ(ierr);
439           if (dof-cdof) {
440             if (bs < 0) {
441               bs = dof-cdof;
442             } else if (bs != dof-cdof) {
443               /* Layout does not admit a pointwise block size */
444               bs = 1;
445               break;
446             }
447           }
448         }
449         /* Must have same blocksize on all procs (some might have no points) */
450         bsLocal = bs;
451         ierr = MPI_Allreduce(&bsLocal, &bsMax, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
452         bsLocal = bs < 0 ? bsMax : bs;
453         ierr = MPI_Allreduce(&bsLocal, &bsMin, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
454         if (bsMin != bsMax) {
455           bs = 1;
456         } else {
457           bs = bsMax;
458         }
459       } else {
460         bs = 1;
461       }
462     }
463     ierr = PetscMalloc4(localSize/bs, PetscInt, &dnz, localSize/bs, PetscInt, &onz, localSize/bs, PetscInt, &dnzu, localSize/bs, PetscInt, &onzu);CHKERRQ(ierr);
464     ierr = PetscMemzero(dnz,  localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
465     ierr = PetscMemzero(onz,  localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
466     ierr = PetscMemzero(dnzu, localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
467     ierr = PetscMemzero(onzu, localSize/bs * sizeof(PetscInt));CHKERRQ(ierr);
468     ierr = DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix);CHKERRQ(ierr);
469     ierr = PetscFree4(dnz, onz, dnzu, onzu);CHKERRQ(ierr);
470   }
471   PetscFunctionReturn(0);
472 }
473 
474 #undef __FUNCT__
475 #define __FUNCT__ "DMPlexGetDimension"
476 /*@
477   DMPlexGetDimension - Return the topological mesh dimension
478 
479   Not collective
480 
481   Input Parameter:
482 . mesh - The DMPlex
483 
484   Output Parameter:
485 . dim - The topological mesh dimension
486 
487   Level: beginner
488 
489 .seealso: DMPlexCreate()
490 @*/
491 PetscErrorCode DMPlexGetDimension(DM dm, PetscInt *dim)
492 {
493   DM_Plex *mesh = (DM_Plex*) dm->data;
494 
495   PetscFunctionBegin;
496   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
497   PetscValidPointer(dim, 2);
498   *dim = mesh->dim;
499   PetscFunctionReturn(0);
500 }
501 
502 #undef __FUNCT__
503 #define __FUNCT__ "DMPlexSetDimension"
504 /*@
505   DMPlexSetDimension - Set the topological mesh dimension
506 
507   Collective on mesh
508 
509   Input Parameters:
510 + mesh - The DMPlex
511 - dim - The topological mesh dimension
512 
513   Level: beginner
514 
515 .seealso: DMPlexCreate()
516 @*/
517 PetscErrorCode DMPlexSetDimension(DM dm, PetscInt dim)
518 {
519   DM_Plex *mesh = (DM_Plex*) dm->data;
520 
521   PetscFunctionBegin;
522   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
523   PetscValidLogicalCollectiveInt(dm, dim, 2);
524   mesh->dim               = dim;
525   mesh->preallocCenterDim = dim;
526   PetscFunctionReturn(0);
527 }
528 
529 #undef __FUNCT__
530 #define __FUNCT__ "DMPlexGetChart"
531 /*@
532   DMPlexGetChart - Return the interval for all mesh points [pStart, pEnd)
533 
534   Not collective
535 
536   Input Parameter:
537 . mesh - The DMPlex
538 
539   Output Parameters:
540 + pStart - The first mesh point
541 - pEnd   - The upper bound for mesh points
542 
543   Level: beginner
544 
545 .seealso: DMPlexCreate(), DMPlexSetChart()
546 @*/
547 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
548 {
549   DM_Plex       *mesh = (DM_Plex*) dm->data;
550   PetscErrorCode ierr;
551 
552   PetscFunctionBegin;
553   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
554   ierr = PetscSectionGetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
555   PetscFunctionReturn(0);
556 }
557 
558 #undef __FUNCT__
559 #define __FUNCT__ "DMPlexSetChart"
560 /*@
561   DMPlexSetChart - Set the interval for all mesh points [pStart, pEnd)
562 
563   Not collective
564 
565   Input Parameters:
566 + mesh - The DMPlex
567 . pStart - The first mesh point
568 - pEnd   - The upper bound for mesh points
569 
570   Output Parameters:
571 
572   Level: beginner
573 
574 .seealso: DMPlexCreate(), DMPlexGetChart()
575 @*/
576 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
577 {
578   DM_Plex       *mesh = (DM_Plex*) dm->data;
579   PetscErrorCode ierr;
580 
581   PetscFunctionBegin;
582   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
583   ierr = PetscSectionSetChart(mesh->coneSection, pStart, pEnd);CHKERRQ(ierr);
584   ierr = PetscSectionSetChart(mesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
585   PetscFunctionReturn(0);
586 }
587 
588 #undef __FUNCT__
589 #define __FUNCT__ "DMPlexGetConeSize"
590 /*@
591   DMPlexGetConeSize - Return the number of in-edges for this point in the Sieve DAG
592 
593   Not collective
594 
595   Input Parameters:
596 + mesh - The DMPlex
597 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
598 
599   Output Parameter:
600 . size - The cone size for point p
601 
602   Level: beginner
603 
604 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
605 @*/
606 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
607 {
608   DM_Plex       *mesh = (DM_Plex*) dm->data;
609   PetscErrorCode ierr;
610 
611   PetscFunctionBegin;
612   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
613   PetscValidPointer(size, 3);
614   ierr = PetscSectionGetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
615   PetscFunctionReturn(0);
616 }
617 
618 #undef __FUNCT__
619 #define __FUNCT__ "DMPlexSetConeSize"
620 /*@
621   DMPlexSetConeSize - Set the number of in-edges for this point in the Sieve DAG
622 
623   Not collective
624 
625   Input Parameters:
626 + mesh - The DMPlex
627 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
628 - size - The cone size for point p
629 
630   Output Parameter:
631 
632   Note:
633   This should be called after DMPlexSetChart().
634 
635   Level: beginner
636 
637 .seealso: DMPlexCreate(), DMPlexGetConeSize(), DMPlexSetChart()
638 @*/
639 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
640 {
641   DM_Plex       *mesh = (DM_Plex*) dm->data;
642   PetscErrorCode ierr;
643 
644   PetscFunctionBegin;
645   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
646   ierr = PetscSectionSetDof(mesh->coneSection, p, size);CHKERRQ(ierr);
647 
648   mesh->maxConeSize = PetscMax(mesh->maxConeSize, size);
649   PetscFunctionReturn(0);
650 }
651 
652 #undef __FUNCT__
653 #define __FUNCT__ "DMPlexGetCone"
654 /*@C
655   DMPlexGetCone - Return the points on the in-edges for this point in the Sieve DAG
656 
657   Not collective
658 
659   Input Parameters:
660 + mesh - The DMPlex
661 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
662 
663   Output Parameter:
664 . cone - An array of points which are on the in-edges for point p
665 
666   Level: beginner
667 
668   Fortran Notes:
669   Since it returns an array, this routine is only available in Fortran 90, and you must
670   include petsc.h90 in your code.
671 
672   You must also call DMPlexRestoreCone() after you finish using the returned array.
673 
674 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart()
675 @*/
676 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
677 {
678   DM_Plex       *mesh = (DM_Plex*) dm->data;
679   PetscInt       off;
680   PetscErrorCode ierr;
681 
682   PetscFunctionBegin;
683   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
684   PetscValidPointer(cone, 3);
685   ierr  = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
686   *cone = &mesh->cones[off];
687   PetscFunctionReturn(0);
688 }
689 
690 #undef __FUNCT__
691 #define __FUNCT__ "DMPlexSetCone"
692 /*@
693   DMPlexSetCone - Set the points on the in-edges for this point in the Sieve DAG
694 
695   Not collective
696 
697   Input Parameters:
698 + mesh - The DMPlex
699 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
700 - cone - An array of points which are on the in-edges for point p
701 
702   Output Parameter:
703 
704   Note:
705   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
706 
707   Level: beginner
708 
709 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
710 @*/
711 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
712 {
713   DM_Plex       *mesh = (DM_Plex*) dm->data;
714   PetscInt       pStart, pEnd;
715   PetscInt       dof, off, c;
716   PetscErrorCode ierr;
717 
718   PetscFunctionBegin;
719   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
720   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
721   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
722   if (dof) PetscValidPointer(cone, 3);
723   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
724   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
725   for (c = 0; c < dof; ++c) {
726     if ((cone[c] < pStart) || (cone[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", cone[c], pStart, pEnd);
727     mesh->cones[off+c] = cone[c];
728   }
729   PetscFunctionReturn(0);
730 }
731 
732 #undef __FUNCT__
733 #define __FUNCT__ "DMPlexGetConeOrientation"
734 /*@C
735   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the Sieve DAG
736 
737   Not collective
738 
739   Input Parameters:
740 + mesh - The DMPlex
741 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
742 
743   Output Parameter:
744 . coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
745                     integer giving the prescription for cone traversal. If it is negative, the cone is
746                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
747                     the index of the cone point on which to start.
748 
749   Level: beginner
750 
751   Fortran Notes:
752   Since it returns an array, this routine is only available in Fortran 90, and you must
753   include petsc.h90 in your code.
754 
755   You must also call DMPlexRestoreConeOrientation() after you finish using the returned array.
756 
757 .seealso: DMPlexCreate(), DMPlexGetCone(), DMPlexSetCone(), DMPlexSetChart()
758 @*/
759 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
760 {
761   DM_Plex       *mesh = (DM_Plex*) dm->data;
762   PetscInt       off;
763   PetscErrorCode ierr;
764 
765   PetscFunctionBegin;
766   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
767 #if defined(PETSC_USE_DEBUG)
768   {
769     PetscInt dof;
770     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
771     if (dof) PetscValidPointer(coneOrientation, 3);
772   }
773 #endif
774   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
775 
776   *coneOrientation = &mesh->coneOrientations[off];
777   PetscFunctionReturn(0);
778 }
779 
780 #undef __FUNCT__
781 #define __FUNCT__ "DMPlexSetConeOrientation"
782 /*@
783   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the Sieve DAG
784 
785   Not collective
786 
787   Input Parameters:
788 + mesh - The DMPlex
789 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
790 - coneOrientation - An array of orientations which are on the in-edges for point p. An orientation is an
791                     integer giving the prescription for cone traversal. If it is negative, the cone is
792                     traversed in the opposite direction. Its value 'o', or if negative '-(o+1)', gives
793                     the index of the cone point on which to start.
794 
795   Output Parameter:
796 
797   Note:
798   This should be called after all calls to DMPlexSetConeSize() and DMSetUp().
799 
800   Level: beginner
801 
802 .seealso: DMPlexCreate(), DMPlexGetConeOrientation(), DMPlexSetCone(), DMPlexSetChart(), DMPlexSetConeSize(), DMSetUp()
803 @*/
804 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
805 {
806   DM_Plex       *mesh = (DM_Plex*) dm->data;
807   PetscInt       pStart, pEnd;
808   PetscInt       dof, off, c;
809   PetscErrorCode ierr;
810 
811   PetscFunctionBegin;
812   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
813   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
814   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
815   if (dof) PetscValidPointer(coneOrientation, 3);
816   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
817   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
818   for (c = 0; c < dof; ++c) {
819     PetscInt cdof, o = coneOrientation[c];
820 
821     ierr = PetscSectionGetDof(mesh->coneSection, mesh->cones[off+c], &cdof);CHKERRQ(ierr);
822     if (o && ((o < -(cdof+1)) || (o >= cdof))) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %D is not in the valid range [%D. %D)", o, -(cdof+1), cdof);
823     mesh->coneOrientations[off+c] = o;
824   }
825   PetscFunctionReturn(0);
826 }
827 
828 #undef __FUNCT__
829 #define __FUNCT__ "DMPlexInsertCone"
830 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
831 {
832   DM_Plex       *mesh = (DM_Plex*) dm->data;
833   PetscInt       pStart, pEnd;
834   PetscInt       dof, off;
835   PetscErrorCode ierr;
836 
837   PetscFunctionBegin;
838   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
839   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
840   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
841   if ((conePoint < pStart) || (conePoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %D is not in the valid range [%D, %D)", conePoint, pStart, pEnd);
842   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
843   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
844   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
845   mesh->cones[off+conePos] = conePoint;
846   PetscFunctionReturn(0);
847 }
848 
849 #undef __FUNCT__
850 #define __FUNCT__ "DMPlexInsertConeOrientation"
851 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
852 {
853   DM_Plex       *mesh = (DM_Plex*) dm->data;
854   PetscInt       pStart, pEnd;
855   PetscInt       dof, off;
856   PetscErrorCode ierr;
857 
858   PetscFunctionBegin;
859   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
860   ierr = PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
861   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
862   ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
863   ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
864   if ((conePos < 0) || (conePos >= dof)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %D of point %D is not in the valid range [0, %D)", conePos, p, dof);
865   mesh->coneOrientations[off+conePos] = coneOrientation;
866   PetscFunctionReturn(0);
867 }
868 
869 #undef __FUNCT__
870 #define __FUNCT__ "DMPlexGetSupportSize"
871 /*@
872   DMPlexGetSupportSize - Return the number of out-edges for this point in the Sieve DAG
873 
874   Not collective
875 
876   Input Parameters:
877 + mesh - The DMPlex
878 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
879 
880   Output Parameter:
881 . size - The support size for point p
882 
883   Level: beginner
884 
885 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart(), DMPlexGetConeSize()
886 @*/
887 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
888 {
889   DM_Plex       *mesh = (DM_Plex*) dm->data;
890   PetscErrorCode ierr;
891 
892   PetscFunctionBegin;
893   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
894   PetscValidPointer(size, 3);
895   ierr = PetscSectionGetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
896   PetscFunctionReturn(0);
897 }
898 
899 #undef __FUNCT__
900 #define __FUNCT__ "DMPlexSetSupportSize"
901 /*@
902   DMPlexSetSupportSize - Set the number of out-edges for this point in the Sieve DAG
903 
904   Not collective
905 
906   Input Parameters:
907 + mesh - The DMPlex
908 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
909 - size - The support size for point p
910 
911   Output Parameter:
912 
913   Note:
914   This should be called after DMPlexSetChart().
915 
916   Level: beginner
917 
918 .seealso: DMPlexCreate(), DMPlexGetSupportSize(), DMPlexSetChart()
919 @*/
920 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
921 {
922   DM_Plex       *mesh = (DM_Plex*) dm->data;
923   PetscErrorCode ierr;
924 
925   PetscFunctionBegin;
926   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
927   ierr = PetscSectionSetDof(mesh->supportSection, p, size);CHKERRQ(ierr);
928 
929   mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, size);
930   PetscFunctionReturn(0);
931 }
932 
933 #undef __FUNCT__
934 #define __FUNCT__ "DMPlexGetSupport"
935 /*@C
936   DMPlexGetSupport - Return the points on the out-edges for this point in the Sieve DAG
937 
938   Not collective
939 
940   Input Parameters:
941 + mesh - The DMPlex
942 - p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
943 
944   Output Parameter:
945 . support - An array of points which are on the out-edges for point p
946 
947   Level: beginner
948 
949   Fortran Notes:
950   Since it returns an array, this routine is only available in Fortran 90, and you must
951   include petsc.h90 in your code.
952 
953   You must also call DMPlexRestoreSupport() after you finish using the returned array.
954 
955 .seealso: DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
956 @*/
957 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
958 {
959   DM_Plex       *mesh = (DM_Plex*) dm->data;
960   PetscInt       off;
961   PetscErrorCode ierr;
962 
963   PetscFunctionBegin;
964   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
965   PetscValidPointer(support, 3);
966   ierr     = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
967   *support = &mesh->supports[off];
968   PetscFunctionReturn(0);
969 }
970 
971 #undef __FUNCT__
972 #define __FUNCT__ "DMPlexSetSupport"
973 /*@
974   DMPlexSetSupport - Set the points on the out-edges for this point in the Sieve DAG
975 
976   Not collective
977 
978   Input Parameters:
979 + mesh - The DMPlex
980 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
981 - support - An array of points which are on the in-edges for point p
982 
983   Output Parameter:
984 
985   Note:
986   This should be called after all calls to DMPlexSetSupportSize() and DMSetUp().
987 
988   Level: beginner
989 
990 .seealso: DMPlexCreate(), DMPlexGetSupport(), DMPlexSetChart(), DMPlexSetSupportSize(), DMSetUp()
991 @*/
992 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
993 {
994   DM_Plex       *mesh = (DM_Plex*) dm->data;
995   PetscInt       pStart, pEnd;
996   PetscInt       dof, off, c;
997   PetscErrorCode ierr;
998 
999   PetscFunctionBegin;
1000   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1001   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1002   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1003   if (dof) PetscValidPointer(support, 3);
1004   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1005   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1006   for (c = 0; c < dof; ++c) {
1007     if ((support[c] < pStart) || (support[c] >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", support[c], pStart, pEnd);
1008     mesh->supports[off+c] = support[c];
1009   }
1010   PetscFunctionReturn(0);
1011 }
1012 
1013 #undef __FUNCT__
1014 #define __FUNCT__ "DMPlexInsertSupport"
1015 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
1016 {
1017   DM_Plex       *mesh = (DM_Plex*) dm->data;
1018   PetscInt       pStart, pEnd;
1019   PetscInt       dof, off;
1020   PetscErrorCode ierr;
1021 
1022   PetscFunctionBegin;
1023   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1024   ierr = PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
1025   ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1026   ierr = PetscSectionGetOffset(mesh->supportSection, p, &off);CHKERRQ(ierr);
1027   if ((p < pStart) || (p >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %D is not in the valid range [%D, %D)", p, pStart, pEnd);
1028   if ((supportPoint < pStart) || (supportPoint >= pEnd)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %D is not in the valid range [%D, %D)", supportPoint, pStart, pEnd);
1029   if (supportPos >= dof) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %D of point %D is not in the valid range [0, %D)", supportPos, p, dof);
1030   mesh->supports[off+supportPos] = supportPoint;
1031   PetscFunctionReturn(0);
1032 }
1033 
1034 #undef __FUNCT__
1035 #define __FUNCT__ "DMPlexGetTransitiveClosure"
1036 /*@C
1037   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1038 
1039   Not collective
1040 
1041   Input Parameters:
1042 + mesh - The DMPlex
1043 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1044 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1045 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1046 
1047   Output Parameters:
1048 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1049 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1050 
1051   Note:
1052   If using internal storage (points is NULL on input), each call overwrites the last output.
1053 
1054   Fortran Notes:
1055   Since it returns an array, this routine is only available in Fortran 90, and you must
1056   include petsc.h90 in your code.
1057 
1058   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1059 
1060   Level: beginner
1061 
1062 .seealso: DMPlexRestoreTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1063 @*/
1064 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1065 {
1066   DM_Plex        *mesh = (DM_Plex*) dm->data;
1067   PetscInt       *closure, *fifo;
1068   const PetscInt *tmp = NULL, *tmpO = NULL;
1069   PetscInt        tmpSize, t;
1070   PetscInt        depth       = 0, maxSize;
1071   PetscInt        closureSize = 2, fifoSize = 0, fifoStart = 0;
1072   PetscErrorCode  ierr;
1073 
1074   PetscFunctionBegin;
1075   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1076   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1077   /* This is only 1-level */
1078   if (useCone) {
1079     ierr = DMPlexGetConeSize(dm, p, &tmpSize);CHKERRQ(ierr);
1080     ierr = DMPlexGetCone(dm, p, &tmp);CHKERRQ(ierr);
1081     ierr = DMPlexGetConeOrientation(dm, p, &tmpO);CHKERRQ(ierr);
1082   } else {
1083     ierr = DMPlexGetSupportSize(dm, p, &tmpSize);CHKERRQ(ierr);
1084     ierr = DMPlexGetSupport(dm, p, &tmp);CHKERRQ(ierr);
1085   }
1086   if (depth == 1) {
1087     if (*points) {
1088       closure = *points;
1089     } else {
1090       maxSize = 2*(PetscMax(mesh->maxConeSize, mesh->maxSupportSize)+1);
1091       ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1092     }
1093     closure[0] = p; closure[1] = 0;
1094     for (t = 0; t < tmpSize; ++t, closureSize += 2) {
1095       closure[closureSize]   = tmp[t];
1096       closure[closureSize+1] = tmpO ? tmpO[t] : 0;
1097     }
1098     if (numPoints) *numPoints = closureSize/2;
1099     if (points)    *points    = closure;
1100     PetscFunctionReturn(0);
1101   }
1102   maxSize = 2*PetscMax(PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1)),depth+1);
1103   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1104   if (*points) {
1105     closure = *points;
1106   } else {
1107     ierr = DMGetWorkArray(dm, maxSize, PETSC_INT, &closure);CHKERRQ(ierr);
1108   }
1109   closure[0] = p; closure[1] = 0;
1110   for (t = 0; t < tmpSize; ++t, closureSize += 2, fifoSize += 2) {
1111     const PetscInt cp = tmp[t];
1112     const PetscInt co = tmpO ? tmpO[t] : 0;
1113 
1114     closure[closureSize]   = cp;
1115     closure[closureSize+1] = co;
1116     fifo[fifoSize]         = cp;
1117     fifo[fifoSize+1]       = co;
1118   }
1119   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
1120   while (fifoSize - fifoStart) {
1121     const PetscInt q   = fifo[fifoStart];
1122     const PetscInt o   = fifo[fifoStart+1];
1123     const PetscInt rev = o >= 0 ? 0 : 1;
1124     const PetscInt off = rev ? -(o+1) : o;
1125 
1126     if (useCone) {
1127       ierr = DMPlexGetConeSize(dm, q, &tmpSize);CHKERRQ(ierr);
1128       ierr = DMPlexGetCone(dm, q, &tmp);CHKERRQ(ierr);
1129       ierr = DMPlexGetConeOrientation(dm, q, &tmpO);CHKERRQ(ierr);
1130     } else {
1131       ierr = DMPlexGetSupportSize(dm, q, &tmpSize);CHKERRQ(ierr);
1132       ierr = DMPlexGetSupport(dm, q, &tmp);CHKERRQ(ierr);
1133       tmpO = NULL;
1134     }
1135     for (t = 0; t < tmpSize; ++t) {
1136       const PetscInt i  = ((rev ? tmpSize-t : t) + off)%tmpSize;
1137       const PetscInt cp = tmp[i];
1138       /* Must propogate orientation: When we reverse orientation, we both reverse the direction of iteration and start at the other end of the chain. */
1139       /* HACK: It is worse to get the size here, than to change the interpretation of -(*+1)
1140        const PetscInt co = tmpO ? (rev ? -(tmpO[i]+1) : tmpO[i]) : 0; */
1141       PetscInt       co = tmpO ? tmpO[i] : 0;
1142       PetscInt       c;
1143 
1144       if (rev) {
1145         PetscInt childSize, coff;
1146         ierr = DMPlexGetConeSize(dm, cp, &childSize);CHKERRQ(ierr);
1147         coff = tmpO[i] < 0 ? -(tmpO[i]+1) : tmpO[i];
1148         co   = childSize ? -(((coff+childSize-1)%childSize)+1) : 0;
1149       }
1150       /* Check for duplicate */
1151       for (c = 0; c < closureSize; c += 2) {
1152         if (closure[c] == cp) break;
1153       }
1154       if (c == closureSize) {
1155         closure[closureSize]   = cp;
1156         closure[closureSize+1] = co;
1157         fifo[fifoSize]         = cp;
1158         fifo[fifoSize+1]       = co;
1159         closureSize           += 2;
1160         fifoSize              += 2;
1161       }
1162     }
1163     fifoStart += 2;
1164   }
1165   if (numPoints) *numPoints = closureSize/2;
1166   if (points)    *points    = closure;
1167   ierr = DMRestoreWorkArray(dm, maxSize, PETSC_INT, &fifo);CHKERRQ(ierr);
1168   PetscFunctionReturn(0);
1169 }
1170 
1171 #undef __FUNCT__
1172 #define __FUNCT__ "DMPlexRestoreTransitiveClosure"
1173 /*@C
1174   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the Sieve DAG
1175 
1176   Not collective
1177 
1178   Input Parameters:
1179 + mesh - The DMPlex
1180 . p - The Sieve point, which must lie in the chart set with DMPlexSetChart()
1181 . useCone - PETSC_TRUE for in-edges,  otherwise use out-edges
1182 - points - If points is NULL on input, internal storage will be returned, otherwise the provided array is used
1183 
1184   Output Parameters:
1185 + numPoints - The number of points in the closure, so points[] is of size 2*numPoints
1186 - points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
1187 
1188   Note:
1189   If not using internal storage (points is not NULL on input), this call is unnecessary
1190 
1191   Fortran Notes:
1192   Since it returns an array, this routine is only available in Fortran 90, and you must
1193   include petsc.h90 in your code.
1194 
1195   The numPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1196 
1197   Level: beginner
1198 
1199 .seealso: DMPlexGetTransitiveClosure(), DMPlexCreate(), DMPlexSetCone(), DMPlexSetChart(), DMPlexGetCone()
1200 @*/
1201 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
1202 {
1203   PetscErrorCode ierr;
1204 
1205   PetscFunctionBegin;
1206   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1207   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, points);CHKERRQ(ierr);
1208   PetscFunctionReturn(0);
1209 }
1210 
1211 #undef __FUNCT__
1212 #define __FUNCT__ "DMPlexGetMaxSizes"
1213 /*@
1214   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the Sieve DAG
1215 
1216   Not collective
1217 
1218   Input Parameter:
1219 . mesh - The DMPlex
1220 
1221   Output Parameters:
1222 + maxConeSize - The maximum number of in-edges
1223 - maxSupportSize - The maximum number of out-edges
1224 
1225   Level: beginner
1226 
1227 .seealso: DMPlexCreate(), DMPlexSetConeSize(), DMPlexSetChart()
1228 @*/
1229 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
1230 {
1231   DM_Plex *mesh = (DM_Plex*) dm->data;
1232 
1233   PetscFunctionBegin;
1234   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1235   if (maxConeSize)    *maxConeSize    = mesh->maxConeSize;
1236   if (maxSupportSize) *maxSupportSize = mesh->maxSupportSize;
1237   PetscFunctionReturn(0);
1238 }
1239 
1240 #undef __FUNCT__
1241 #define __FUNCT__ "DMSetUp_Plex"
1242 PetscErrorCode DMSetUp_Plex(DM dm)
1243 {
1244   DM_Plex       *mesh = (DM_Plex*) dm->data;
1245   PetscInt       size;
1246   PetscErrorCode ierr;
1247 
1248   PetscFunctionBegin;
1249   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1250   ierr = PetscSectionSetUp(mesh->coneSection);CHKERRQ(ierr);
1251   ierr = PetscSectionGetStorageSize(mesh->coneSection, &size);CHKERRQ(ierr);
1252   ierr = PetscMalloc(size * sizeof(PetscInt), &mesh->cones);CHKERRQ(ierr);
1253   ierr = PetscMalloc(size * sizeof(PetscInt), &mesh->coneOrientations);CHKERRQ(ierr);
1254   ierr = PetscMemzero(mesh->coneOrientations, size * sizeof(PetscInt));CHKERRQ(ierr);
1255   if (mesh->maxSupportSize) {
1256     ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1257     ierr = PetscSectionGetStorageSize(mesh->supportSection, &size);CHKERRQ(ierr);
1258     ierr = PetscMalloc(size * sizeof(PetscInt), &mesh->supports);CHKERRQ(ierr);
1259   }
1260   PetscFunctionReturn(0);
1261 }
1262 
1263 #undef __FUNCT__
1264 #define __FUNCT__ "DMCreateSubDM_Plex"
1265 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
1266 {
1267   PetscErrorCode ierr;
1268 
1269   PetscFunctionBegin;
1270   if (subdm) {ierr = DMClone(dm, subdm);CHKERRQ(ierr);}
1271   ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
1272   PetscFunctionReturn(0);
1273 }
1274 
1275 #undef __FUNCT__
1276 #define __FUNCT__ "DMPlexSymmetrize"
1277 /*@
1278   DMPlexSymmetrize - Creates support (out-edge) information from cone (in-edge) inoformation
1279 
1280   Not collective
1281 
1282   Input Parameter:
1283 . mesh - The DMPlex
1284 
1285   Output Parameter:
1286 
1287   Note:
1288   This should be called after all calls to DMPlexSetCone()
1289 
1290   Level: beginner
1291 
1292 .seealso: DMPlexCreate(), DMPlexSetChart(), DMPlexSetConeSize(), DMPlexSetCone()
1293 @*/
1294 PetscErrorCode DMPlexSymmetrize(DM dm)
1295 {
1296   DM_Plex       *mesh = (DM_Plex*) dm->data;
1297   PetscInt      *offsets;
1298   PetscInt       supportSize;
1299   PetscInt       pStart, pEnd, p;
1300   PetscErrorCode ierr;
1301 
1302   PetscFunctionBegin;
1303   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1304   if (mesh->supports) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
1305   /* Calculate support sizes */
1306   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1307   for (p = pStart; p < pEnd; ++p) {
1308     PetscInt dof, off, c;
1309 
1310     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1311     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1312     for (c = off; c < off+dof; ++c) {
1313       ierr = PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1);CHKERRQ(ierr);
1314     }
1315   }
1316   for (p = pStart; p < pEnd; ++p) {
1317     PetscInt dof;
1318 
1319     ierr = PetscSectionGetDof(mesh->supportSection, p, &dof);CHKERRQ(ierr);
1320 
1321     mesh->maxSupportSize = PetscMax(mesh->maxSupportSize, dof);
1322   }
1323   ierr = PetscSectionSetUp(mesh->supportSection);CHKERRQ(ierr);
1324   /* Calculate supports */
1325   ierr = PetscSectionGetStorageSize(mesh->supportSection, &supportSize);CHKERRQ(ierr);
1326   ierr = PetscMalloc(supportSize * sizeof(PetscInt), &mesh->supports);CHKERRQ(ierr);
1327   ierr = PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &offsets);CHKERRQ(ierr);
1328   ierr = PetscMemzero(offsets, (pEnd - pStart) * sizeof(PetscInt));CHKERRQ(ierr);
1329   for (p = pStart; p < pEnd; ++p) {
1330     PetscInt dof, off, c;
1331 
1332     ierr = PetscSectionGetDof(mesh->coneSection, p, &dof);CHKERRQ(ierr);
1333     ierr = PetscSectionGetOffset(mesh->coneSection, p, &off);CHKERRQ(ierr);
1334     for (c = off; c < off+dof; ++c) {
1335       const PetscInt q = mesh->cones[c];
1336       PetscInt       offS;
1337 
1338       ierr = PetscSectionGetOffset(mesh->supportSection, q, &offS);CHKERRQ(ierr);
1339 
1340       mesh->supports[offS+offsets[q]] = p;
1341       ++offsets[q];
1342     }
1343   }
1344   ierr = PetscFree(offsets);CHKERRQ(ierr);
1345   PetscFunctionReturn(0);
1346 }
1347 
1348 #undef __FUNCT__
1349 #define __FUNCT__ "DMPlexStratify"
1350 /*@
1351   DMPlexStratify - The Sieve DAG for most topologies is a graded poset (http://en.wikipedia.org/wiki/Graded_poset), and
1352   can be illustrated by Hasse Diagram (a http://en.wikipedia.org/wiki/Hasse_diagram). The strata group all points of the
1353   same grade, and this function calculates the strata. This grade can be seen as the height (or depth) of the point in
1354   the DAG.
1355 
1356   Not collective
1357 
1358   Input Parameter:
1359 . mesh - The DMPlex
1360 
1361   Output Parameter:
1362 
1363   Notes:
1364   The normal association for the point grade is element dimension (or co-dimension). For instance, all vertices would
1365   have depth 0, and all edges depth 1. Likewise, all cells heights would have height 0, and all faces height 1.
1366 
1367   This should be called after all calls to DMPlexSymmetrize()
1368 
1369   Level: beginner
1370 
1371 .seealso: DMPlexCreate(), DMPlexSymmetrize()
1372 @*/
1373 PetscErrorCode DMPlexStratify(DM dm)
1374 {
1375   DMLabel        label;
1376   PetscInt       pStart, pEnd, p;
1377   PetscInt       numRoots = 0, numLeaves = 0;
1378   PetscErrorCode ierr;
1379 
1380   PetscFunctionBegin;
1381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1382   ierr = PetscLogEventBegin(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1383   /* Calculate depth */
1384   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
1385   ierr = DMPlexCreateLabel(dm, "depth");CHKERRQ(ierr);
1386   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
1387   /* Initialize roots and count leaves */
1388   for (p = pStart; p < pEnd; ++p) {
1389     PetscInt coneSize, supportSize;
1390 
1391     ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1392     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1393     if (!coneSize && supportSize) {
1394       ++numRoots;
1395       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1396     } else if (!supportSize && coneSize) {
1397       ++numLeaves;
1398     } else if (!supportSize && !coneSize) {
1399       /* Isolated points */
1400       ierr = DMLabelSetValue(label, p, 0);CHKERRQ(ierr);
1401     }
1402   }
1403   if (numRoots + numLeaves == (pEnd - pStart)) {
1404     for (p = pStart; p < pEnd; ++p) {
1405       PetscInt coneSize, supportSize;
1406 
1407       ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
1408       ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
1409       if (!supportSize && coneSize) {
1410         ierr = DMLabelSetValue(label, p, 1);CHKERRQ(ierr);
1411       }
1412     }
1413   } else {
1414     IS       pointIS;
1415     PetscInt numPoints = 0, level = 0;
1416 
1417     ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1418     if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1419     while (numPoints) {
1420       const PetscInt *points;
1421       const PetscInt  newLevel = level+1;
1422 
1423       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
1424       for (p = 0; p < numPoints; ++p) {
1425         const PetscInt  point = points[p];
1426         const PetscInt *support;
1427         PetscInt        supportSize, s;
1428 
1429         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1430         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1431         for (s = 0; s < supportSize; ++s) {
1432           ierr = DMLabelSetValue(label, support[s], newLevel);CHKERRQ(ierr);
1433         }
1434       }
1435       ++level;
1436       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1437       ierr = DMLabelGetStratumIS(label, level, &pointIS);CHKERRQ(ierr);
1438       if (pointIS) {ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);}
1439       else         {numPoints = 0;}
1440     }
1441     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
1442   }
1443   ierr = PetscLogEventEnd(DMPLEX_Stratify,dm,0,0,0);CHKERRQ(ierr);
1444   PetscFunctionReturn(0);
1445 }
1446 
1447 #undef __FUNCT__
1448 #define __FUNCT__ "DMPlexGetJoin"
1449 /*@C
1450   DMPlexGetJoin - Get an array for the join of the set of points
1451 
1452   Not Collective
1453 
1454   Input Parameters:
1455 + dm - The DMPlex object
1456 . numPoints - The number of input points for the join
1457 - points - The input points
1458 
1459   Output Parameters:
1460 + numCoveredPoints - The number of points in the join
1461 - coveredPoints - The points in the join
1462 
1463   Level: intermediate
1464 
1465   Note: Currently, this is restricted to a single level join
1466 
1467   Fortran Notes:
1468   Since it returns an array, this routine is only available in Fortran 90, and you must
1469   include petsc.h90 in your code.
1470 
1471   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1472 
1473 .keywords: mesh
1474 .seealso: DMPlexRestoreJoin(), DMPlexGetMeet()
1475 @*/
1476 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1477 {
1478   DM_Plex       *mesh = (DM_Plex*) dm->data;
1479   PetscInt      *join[2];
1480   PetscInt       joinSize, i = 0;
1481   PetscInt       dof, off, p, c, m;
1482   PetscErrorCode ierr;
1483 
1484   PetscFunctionBegin;
1485   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1486   PetscValidPointer(points, 2);
1487   PetscValidPointer(numCoveredPoints, 3);
1488   PetscValidPointer(coveredPoints, 4);
1489   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1490   ierr = DMGetWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1491   /* Copy in support of first point */
1492   ierr = PetscSectionGetDof(mesh->supportSection, points[0], &dof);CHKERRQ(ierr);
1493   ierr = PetscSectionGetOffset(mesh->supportSection, points[0], &off);CHKERRQ(ierr);
1494   for (joinSize = 0; joinSize < dof; ++joinSize) {
1495     join[i][joinSize] = mesh->supports[off+joinSize];
1496   }
1497   /* Check each successive support */
1498   for (p = 1; p < numPoints; ++p) {
1499     PetscInt newJoinSize = 0;
1500 
1501     ierr = PetscSectionGetDof(mesh->supportSection, points[p], &dof);CHKERRQ(ierr);
1502     ierr = PetscSectionGetOffset(mesh->supportSection, points[p], &off);CHKERRQ(ierr);
1503     for (c = 0; c < dof; ++c) {
1504       const PetscInt point = mesh->supports[off+c];
1505 
1506       for (m = 0; m < joinSize; ++m) {
1507         if (point == join[i][m]) {
1508           join[1-i][newJoinSize++] = point;
1509           break;
1510         }
1511       }
1512     }
1513     joinSize = newJoinSize;
1514     i        = 1-i;
1515   }
1516   *numCoveredPoints = joinSize;
1517   *coveredPoints    = join[i];
1518   ierr              = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1519   PetscFunctionReturn(0);
1520 }
1521 
1522 #undef __FUNCT__
1523 #define __FUNCT__ "DMPlexRestoreJoin"
1524 /*@C
1525   DMPlexRestoreJoin - Restore an array for the join of the set of points
1526 
1527   Not Collective
1528 
1529   Input Parameters:
1530 + dm - The DMPlex object
1531 . numPoints - The number of input points for the join
1532 - points - The input points
1533 
1534   Output Parameters:
1535 + numCoveredPoints - The number of points in the join
1536 - coveredPoints - The points in the join
1537 
1538   Fortran Notes:
1539   Since it returns an array, this routine is only available in Fortran 90, and you must
1540   include petsc.h90 in your code.
1541 
1542   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1543 
1544   Level: intermediate
1545 
1546 .keywords: mesh
1547 .seealso: DMPlexGetJoin(), DMPlexGetFullJoin(), DMPlexGetMeet()
1548 @*/
1549 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1550 {
1551   PetscErrorCode ierr;
1552 
1553   PetscFunctionBegin;
1554   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1555   PetscValidPointer(coveredPoints, 4);
1556   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
1557   PetscFunctionReturn(0);
1558 }
1559 
1560 #undef __FUNCT__
1561 #define __FUNCT__ "DMPlexGetFullJoin"
1562 /*@C
1563   DMPlexGetFullJoin - Get an array for the join of the set of points
1564 
1565   Not Collective
1566 
1567   Input Parameters:
1568 + dm - The DMPlex object
1569 . numPoints - The number of input points for the join
1570 - points - The input points
1571 
1572   Output Parameters:
1573 + numCoveredPoints - The number of points in the join
1574 - coveredPoints - The points in the join
1575 
1576   Fortran Notes:
1577   Since it returns an array, this routine is only available in Fortran 90, and you must
1578   include petsc.h90 in your code.
1579 
1580   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1581 
1582   Level: intermediate
1583 
1584 .keywords: mesh
1585 .seealso: DMPlexGetJoin(), DMPlexRestoreJoin(), DMPlexGetMeet()
1586 @*/
1587 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1588 {
1589   DM_Plex       *mesh = (DM_Plex*) dm->data;
1590   PetscInt      *offsets, **closures;
1591   PetscInt      *join[2];
1592   PetscInt       depth = 0, maxSize, joinSize = 0, i = 0;
1593   PetscInt       p, d, c, m;
1594   PetscErrorCode ierr;
1595 
1596   PetscFunctionBegin;
1597   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1598   PetscValidPointer(points, 2);
1599   PetscValidPointer(numCoveredPoints, 3);
1600   PetscValidPointer(coveredPoints, 4);
1601 
1602   ierr    = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1603   ierr    = PetscMalloc(numPoints * sizeof(PetscInt*), &closures);CHKERRQ(ierr);
1604   ierr    = PetscMemzero(closures,numPoints*sizeof(PetscInt*));CHKERRQ(ierr);
1605   ierr    = DMGetWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1606   maxSize = PetscPowInt(mesh->maxSupportSize,depth+1);
1607   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[0]);CHKERRQ(ierr);
1608   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &join[1]);CHKERRQ(ierr);
1609 
1610   for (p = 0; p < numPoints; ++p) {
1611     PetscInt closureSize;
1612 
1613     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]);CHKERRQ(ierr);
1614 
1615     offsets[p*(depth+2)+0] = 0;
1616     for (d = 0; d < depth+1; ++d) {
1617       PetscInt pStart, pEnd, i;
1618 
1619       ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
1620       for (i = offsets[p*(depth+2)+d]; i < closureSize; ++i) {
1621         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
1622           offsets[p*(depth+2)+d+1] = i;
1623           break;
1624         }
1625       }
1626       if (i == closureSize) offsets[p*(depth+2)+d+1] = i;
1627     }
1628     if (offsets[p*(depth+2)+depth+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(depth+2)+depth+1], closureSize);
1629   }
1630   for (d = 0; d < depth+1; ++d) {
1631     PetscInt dof;
1632 
1633     /* Copy in support of first point */
1634     dof = offsets[d+1] - offsets[d];
1635     for (joinSize = 0; joinSize < dof; ++joinSize) {
1636       join[i][joinSize] = closures[0][(offsets[d]+joinSize)*2];
1637     }
1638     /* Check each successive cone */
1639     for (p = 1; p < numPoints && joinSize; ++p) {
1640       PetscInt newJoinSize = 0;
1641 
1642       dof = offsets[p*(depth+2)+d+1] - offsets[p*(depth+2)+d];
1643       for (c = 0; c < dof; ++c) {
1644         const PetscInt point = closures[p][(offsets[p*(depth+2)+d]+c)*2];
1645 
1646         for (m = 0; m < joinSize; ++m) {
1647           if (point == join[i][m]) {
1648             join[1-i][newJoinSize++] = point;
1649             break;
1650           }
1651         }
1652       }
1653       joinSize = newJoinSize;
1654       i        = 1-i;
1655     }
1656     if (joinSize) break;
1657   }
1658   *numCoveredPoints = joinSize;
1659   *coveredPoints    = join[i];
1660   for (p = 0; p < numPoints; ++p) {
1661     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]);CHKERRQ(ierr);
1662   }
1663   ierr = PetscFree(closures);CHKERRQ(ierr);
1664   ierr = DMRestoreWorkArray(dm, numPoints*(depth+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1665   ierr = DMRestoreWorkArray(dm, mesh->maxSupportSize, PETSC_INT, &join[1-i]);CHKERRQ(ierr);
1666   PetscFunctionReturn(0);
1667 }
1668 
1669 #undef __FUNCT__
1670 #define __FUNCT__ "DMPlexGetMeet"
1671 /*@C
1672   DMPlexGetMeet - Get an array for the meet of the set of points
1673 
1674   Not Collective
1675 
1676   Input Parameters:
1677 + dm - The DMPlex object
1678 . numPoints - The number of input points for the meet
1679 - points - The input points
1680 
1681   Output Parameters:
1682 + numCoveredPoints - The number of points in the meet
1683 - coveredPoints - The points in the meet
1684 
1685   Level: intermediate
1686 
1687   Note: Currently, this is restricted to a single level meet
1688 
1689   Fortran Notes:
1690   Since it returns an array, this routine is only available in Fortran 90, and you must
1691   include petsc.h90 in your code.
1692 
1693   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1694 
1695 .keywords: mesh
1696 .seealso: DMPlexRestoreMeet(), DMPlexGetJoin()
1697 @*/
1698 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt **coveringPoints)
1699 {
1700   DM_Plex       *mesh = (DM_Plex*) dm->data;
1701   PetscInt      *meet[2];
1702   PetscInt       meetSize, i = 0;
1703   PetscInt       dof, off, p, c, m;
1704   PetscErrorCode ierr;
1705 
1706   PetscFunctionBegin;
1707   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1708   PetscValidPointer(points, 2);
1709   PetscValidPointer(numCoveringPoints, 3);
1710   PetscValidPointer(coveringPoints, 4);
1711   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
1712   ierr = DMGetWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
1713   /* Copy in cone of first point */
1714   ierr = PetscSectionGetDof(mesh->coneSection, points[0], &dof);CHKERRQ(ierr);
1715   ierr = PetscSectionGetOffset(mesh->coneSection, points[0], &off);CHKERRQ(ierr);
1716   for (meetSize = 0; meetSize < dof; ++meetSize) {
1717     meet[i][meetSize] = mesh->cones[off+meetSize];
1718   }
1719   /* Check each successive cone */
1720   for (p = 1; p < numPoints; ++p) {
1721     PetscInt newMeetSize = 0;
1722 
1723     ierr = PetscSectionGetDof(mesh->coneSection, points[p], &dof);CHKERRQ(ierr);
1724     ierr = PetscSectionGetOffset(mesh->coneSection, points[p], &off);CHKERRQ(ierr);
1725     for (c = 0; c < dof; ++c) {
1726       const PetscInt point = mesh->cones[off+c];
1727 
1728       for (m = 0; m < meetSize; ++m) {
1729         if (point == meet[i][m]) {
1730           meet[1-i][newMeetSize++] = point;
1731           break;
1732         }
1733       }
1734     }
1735     meetSize = newMeetSize;
1736     i        = 1-i;
1737   }
1738   *numCoveringPoints = meetSize;
1739   *coveringPoints    = meet[i];
1740   ierr               = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
1741   PetscFunctionReturn(0);
1742 }
1743 
1744 #undef __FUNCT__
1745 #define __FUNCT__ "DMPlexRestoreMeet"
1746 /*@C
1747   DMPlexRestoreMeet - Restore an array for the meet of the set of points
1748 
1749   Not Collective
1750 
1751   Input Parameters:
1752 + dm - The DMPlex object
1753 . numPoints - The number of input points for the meet
1754 - points - The input points
1755 
1756   Output Parameters:
1757 + numCoveredPoints - The number of points in the meet
1758 - coveredPoints - The points in the meet
1759 
1760   Level: intermediate
1761 
1762   Fortran Notes:
1763   Since it returns an array, this routine is only available in Fortran 90, and you must
1764   include petsc.h90 in your code.
1765 
1766   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1767 
1768 .keywords: mesh
1769 .seealso: DMPlexGetMeet(), DMPlexGetFullMeet(), DMPlexGetJoin()
1770 @*/
1771 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1772 {
1773   PetscErrorCode ierr;
1774 
1775   PetscFunctionBegin;
1776   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1777   PetscValidPointer(coveredPoints, 4);
1778   ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, (void*) coveredPoints);CHKERRQ(ierr);
1779   PetscFunctionReturn(0);
1780 }
1781 
1782 #undef __FUNCT__
1783 #define __FUNCT__ "DMPlexGetFullMeet"
1784 /*@C
1785   DMPlexGetFullMeet - Get an array for the meet of the set of points
1786 
1787   Not Collective
1788 
1789   Input Parameters:
1790 + dm - The DMPlex object
1791 . numPoints - The number of input points for the meet
1792 - points - The input points
1793 
1794   Output Parameters:
1795 + numCoveredPoints - The number of points in the meet
1796 - coveredPoints - The points in the meet
1797 
1798   Level: intermediate
1799 
1800   Fortran Notes:
1801   Since it returns an array, this routine is only available in Fortran 90, and you must
1802   include petsc.h90 in your code.
1803 
1804   The numCoveredPoints argument is not present in the Fortran 90 binding since it is internal to the array.
1805 
1806 .keywords: mesh
1807 .seealso: DMPlexGetMeet(), DMPlexRestoreMeet(), DMPlexGetJoin()
1808 @*/
1809 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt **coveredPoints)
1810 {
1811   DM_Plex       *mesh = (DM_Plex*) dm->data;
1812   PetscInt      *offsets, **closures;
1813   PetscInt      *meet[2];
1814   PetscInt       height = 0, maxSize, meetSize = 0, i = 0;
1815   PetscInt       p, h, c, m;
1816   PetscErrorCode ierr;
1817 
1818   PetscFunctionBegin;
1819   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1820   PetscValidPointer(points, 2);
1821   PetscValidPointer(numCoveredPoints, 3);
1822   PetscValidPointer(coveredPoints, 4);
1823 
1824   ierr    = DMPlexGetDepth(dm, &height);CHKERRQ(ierr);
1825   ierr    = PetscMalloc(numPoints * sizeof(PetscInt*), &closures);CHKERRQ(ierr);
1826   ierr    = DMGetWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1827   maxSize = PetscPowInt(mesh->maxConeSize,height+1);
1828   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[0]);CHKERRQ(ierr);
1829   ierr    = DMGetWorkArray(dm, maxSize, PETSC_INT, &meet[1]);CHKERRQ(ierr);
1830 
1831   for (p = 0; p < numPoints; ++p) {
1832     PetscInt closureSize;
1833 
1834     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]);CHKERRQ(ierr);
1835 
1836     offsets[p*(height+2)+0] = 0;
1837     for (h = 0; h < height+1; ++h) {
1838       PetscInt pStart, pEnd, i;
1839 
1840       ierr = DMPlexGetHeightStratum(dm, h, &pStart, &pEnd);CHKERRQ(ierr);
1841       for (i = offsets[p*(height+2)+h]; i < closureSize; ++i) {
1842         if ((pStart > closures[p][i*2]) || (pEnd <= closures[p][i*2])) {
1843           offsets[p*(height+2)+h+1] = i;
1844           break;
1845         }
1846       }
1847       if (i == closureSize) offsets[p*(height+2)+h+1] = i;
1848     }
1849     if (offsets[p*(height+2)+height+1] != closureSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %D should be %D", offsets[p*(height+2)+height+1], closureSize);
1850   }
1851   for (h = 0; h < height+1; ++h) {
1852     PetscInt dof;
1853 
1854     /* Copy in cone of first point */
1855     dof = offsets[h+1] - offsets[h];
1856     for (meetSize = 0; meetSize < dof; ++meetSize) {
1857       meet[i][meetSize] = closures[0][(offsets[h]+meetSize)*2];
1858     }
1859     /* Check each successive cone */
1860     for (p = 1; p < numPoints && meetSize; ++p) {
1861       PetscInt newMeetSize = 0;
1862 
1863       dof = offsets[p*(height+2)+h+1] - offsets[p*(height+2)+h];
1864       for (c = 0; c < dof; ++c) {
1865         const PetscInt point = closures[p][(offsets[p*(height+2)+h]+c)*2];
1866 
1867         for (m = 0; m < meetSize; ++m) {
1868           if (point == meet[i][m]) {
1869             meet[1-i][newMeetSize++] = point;
1870             break;
1871           }
1872         }
1873       }
1874       meetSize = newMeetSize;
1875       i        = 1-i;
1876     }
1877     if (meetSize) break;
1878   }
1879   *numCoveredPoints = meetSize;
1880   *coveredPoints    = meet[i];
1881   for (p = 0; p < numPoints; ++p) {
1882     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]);CHKERRQ(ierr);
1883   }
1884   ierr = PetscFree(closures);CHKERRQ(ierr);
1885   ierr = DMRestoreWorkArray(dm, numPoints*(height+2), PETSC_INT, &offsets);CHKERRQ(ierr);
1886   ierr = DMRestoreWorkArray(dm, mesh->maxConeSize, PETSC_INT, &meet[1-i]);CHKERRQ(ierr);
1887   PetscFunctionReturn(0);
1888 }
1889 
1890 #undef __FUNCT__
1891 #define __FUNCT__ "DMPlexGetNumFaceVertices"
1892 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
1893 {
1894   MPI_Comm       comm;
1895   PetscErrorCode ierr;
1896 
1897   PetscFunctionBegin;
1898   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1899   PetscValidPointer(numFaceVertices,3);
1900   switch (cellDim) {
1901   case 0:
1902     *numFaceVertices = 0;
1903     break;
1904   case 1:
1905     *numFaceVertices = 1;
1906     break;
1907   case 2:
1908     switch (numCorners) {
1909     case 3: /* triangle */
1910       *numFaceVertices = 2; /* Edge has 2 vertices */
1911       break;
1912     case 4: /* quadrilateral */
1913       *numFaceVertices = 2; /* Edge has 2 vertices */
1914       break;
1915     case 6: /* quadratic triangle, tri and quad cohesive Lagrange cells */
1916       *numFaceVertices = 3; /* Edge has 3 vertices */
1917       break;
1918     case 9: /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
1919       *numFaceVertices = 3; /* Edge has 3 vertices */
1920       break;
1921     default:
1922       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
1923     }
1924     break;
1925   case 3:
1926     switch (numCorners) {
1927     case 4: /* tetradehdron */
1928       *numFaceVertices = 3; /* Face has 3 vertices */
1929       break;
1930     case 6: /* tet cohesive cells */
1931       *numFaceVertices = 4; /* Face has 4 vertices */
1932       break;
1933     case 8: /* hexahedron */
1934       *numFaceVertices = 4; /* Face has 4 vertices */
1935       break;
1936     case 9: /* tet cohesive Lagrange cells */
1937       *numFaceVertices = 6; /* Face has 6 vertices */
1938       break;
1939     case 10: /* quadratic tetrahedron */
1940       *numFaceVertices = 6; /* Face has 6 vertices */
1941       break;
1942     case 12: /* hex cohesive Lagrange cells */
1943       *numFaceVertices = 6; /* Face has 6 vertices */
1944       break;
1945     case 18: /* quadratic tet cohesive Lagrange cells */
1946       *numFaceVertices = 6; /* Face has 6 vertices */
1947       break;
1948     case 27: /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
1949       *numFaceVertices = 9; /* Face has 9 vertices */
1950       break;
1951     default:
1952       SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %d for dimension %d", numCorners, cellDim);
1953     }
1954     break;
1955   default:
1956     SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %d", cellDim);
1957   }
1958   PetscFunctionReturn(0);
1959 }
1960 
1961 #undef __FUNCT__
1962 #define __FUNCT__ "DMPlexOrient"
1963 /* Trys to give the mesh a consistent orientation */
1964 PetscErrorCode DMPlexOrient(DM dm)
1965 {
1966   PetscBT        seenCells, flippedCells, seenFaces;
1967   PetscInt      *faceFIFO, fTop, fBottom;
1968   PetscInt       dim, h, cStart, cEnd, c, fStart, fEnd, face, maxConeSize, *revcone, *revconeO;
1969   PetscErrorCode ierr;
1970 
1971   PetscFunctionBegin;
1972   /* Truth Table
1973      mismatch    flips   do action   mismatch   flipA ^ flipB   action
1974          F       0 flips     no         F             F           F
1975          F       1 flip      yes        F             T           T
1976          F       2 flips     no         T             F           T
1977          T       0 flips     yes        T             T           F
1978          T       1 flip      no
1979          T       2 flips     yes
1980   */
1981   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
1982   ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
1983   ierr = DMPlexGetHeightStratum(dm, h,   &cStart, &cEnd);CHKERRQ(ierr);
1984   ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
1985   ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
1986   ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
1987   ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
1988   ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
1989   ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
1990   ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
1991   ierr = PetscMalloc((fEnd - fStart) * sizeof(PetscInt), &faceFIFO);CHKERRQ(ierr);
1992   fTop = fBottom = 0;
1993   /* Initialize FIFO with first cell */
1994   if (cEnd > cStart) {
1995     const PetscInt *cone;
1996     PetscInt        coneSize;
1997 
1998     ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
1999     ierr = DMPlexGetCone(dm, cStart, &cone);CHKERRQ(ierr);
2000     for (c = 0; c < coneSize; ++c) {
2001       faceFIFO[fBottom++] = cone[c];
2002       ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
2003     }
2004   }
2005   /* Consider each face in FIFO */
2006   while (fTop < fBottom) {
2007     const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
2008     PetscInt        supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
2009     PetscInt        seenA, flippedA, seenB, flippedB, mismatch;
2010 
2011     face = faceFIFO[fTop++];
2012     ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2013     ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2014     if (supportSize < 2) continue;
2015     if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
2016     seenA    = PetscBTLookup(seenCells,    support[0]-cStart);
2017     flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
2018     seenB    = PetscBTLookup(seenCells,    support[1]-cStart);
2019     flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
2020 
2021     ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
2022     ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
2023     ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
2024     ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
2025     ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
2026     ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
2027     for (c = 0; c < coneSizeA; ++c) {
2028       if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
2029         faceFIFO[fBottom++] = coneA[c];
2030         ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
2031       }
2032       if (coneA[c] == face) posA = c;
2033       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2034     }
2035     if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
2036     for (c = 0; c < coneSizeB; ++c) {
2037       if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
2038         faceFIFO[fBottom++] = coneB[c];
2039         ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
2040       }
2041       if (coneB[c] == face) posB = c;
2042       if (fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], fBottom, fEnd-fStart);
2043     }
2044     if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
2045 
2046     if (dim == 1) {
2047       mismatch = posA == posB;
2048     } else {
2049       mismatch = coneOA[posA] == coneOB[posB];
2050     }
2051 
2052     if (mismatch ^ (flippedA ^ flippedB)) {
2053       if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
2054       if (!seenA && !flippedA) {
2055         ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
2056       } else if (!seenB && !flippedB) {
2057         ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
2058       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
2059     } else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
2060     ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
2061     ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
2062   }
2063 
2064   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2065   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2066   ierr = DMGetWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2067   for (c = cStart; c < cEnd; ++c) {
2068     const PetscInt *cone, *coneO, *support;
2069     PetscInt        coneSize, supportSize, faceSize, cp, sp;
2070 
2071     if (!PetscBTLookup(flippedCells, c-cStart)) continue;
2072     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2073     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2074     ierr = DMPlexGetConeOrientation(dm, c, &coneO);CHKERRQ(ierr);
2075     for (cp = 0; cp < coneSize; ++cp) {
2076       const PetscInt rcp = coneSize-cp-1;
2077 
2078       ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
2079       revcone[cp]  = cone[rcp];
2080       revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
2081     }
2082     ierr = DMPlexSetCone(dm, c, revcone);CHKERRQ(ierr);
2083     ierr = DMPlexSetConeOrientation(dm, c, revconeO);CHKERRQ(ierr);
2084     /* Reverse orientations of support */
2085     faceSize = coneSize;
2086     ierr = DMPlexGetSupportSize(dm, c, &supportSize);CHKERRQ(ierr);
2087     ierr = DMPlexGetSupport(dm, c, &support);CHKERRQ(ierr);
2088     for (sp = 0; sp < supportSize; ++sp) {
2089       ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
2090       ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
2091       ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
2092       for (cp = 0; cp < coneSize; ++cp) {
2093         if (cone[cp] != c) continue;
2094         ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
2095       }
2096     }
2097   }
2098   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revcone);CHKERRQ(ierr);
2099   ierr = DMRestoreWorkArray(dm, maxConeSize, PETSC_INT, &revconeO);CHKERRQ(ierr);
2100   ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
2101   ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
2102   ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
2103   ierr = PetscFree(faceFIFO);CHKERRQ(ierr);
2104   PetscFunctionReturn(0);
2105 }
2106 
2107 #undef __FUNCT__
2108 #define __FUNCT__ "DMPlexGetAdjacencySingleLevel_Internal"
2109 static PetscErrorCode DMPlexGetAdjacencySingleLevel_Internal(DM dm, PetscInt p, PetscBool useClosure, const PetscInt *tmpClosure, PetscInt *adjSize, PetscInt adj[])
2110 {
2111   const PetscInt *support = NULL;
2112   PetscInt        numAdj   = 0, maxAdjSize = *adjSize, supportSize, s;
2113   PetscErrorCode  ierr;
2114 
2115   PetscFunctionBegin;
2116   if (useClosure) {
2117     ierr = DMPlexGetConeSize(dm, p, &supportSize);CHKERRQ(ierr);
2118     ierr = DMPlexGetCone(dm, p, &support);CHKERRQ(ierr);
2119     for (s = 0; s < supportSize; ++s) {
2120       const PetscInt *cone = NULL;
2121       PetscInt        coneSize, c, q;
2122 
2123       ierr = DMPlexGetSupportSize(dm, support[s], &coneSize);CHKERRQ(ierr);
2124       ierr = DMPlexGetSupport(dm, support[s], &cone);CHKERRQ(ierr);
2125       for (c = 0; c < coneSize; ++c) {
2126         for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
2127           if (cone[c] == adj[q]) break;
2128         }
2129         if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
2130       }
2131     }
2132   } else {
2133     ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
2134     ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
2135     for (s = 0; s < supportSize; ++s) {
2136       const PetscInt *cone = NULL;
2137       PetscInt        coneSize, c, q;
2138 
2139       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
2140       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
2141       for (c = 0; c < coneSize; ++c) {
2142         for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
2143           if (cone[c] == adj[q]) break;
2144         }
2145         if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
2146       }
2147     }
2148   }
2149   *adjSize = numAdj;
2150   PetscFunctionReturn(0);
2151 }
2152 
2153 #undef __FUNCT__
2154 #define __FUNCT__ "DMPlexCreateNeighborCSR"
2155 PetscErrorCode DMPlexCreateNeighborCSR(DM dm, PetscInt cellHeight, PetscInt *numVertices, PetscInt **offsets, PetscInt **adjacency)
2156 {
2157   const PetscInt maxFaceCases = 30;
2158   PetscInt       numFaceCases = 0;
2159   PetscInt       numFaceVertices[30]; /* maxFaceCases, C89 sucks sucks sucks */
2160   PetscInt      *off, *adj;
2161   PetscInt      *neighborCells, *tmpClosure;
2162   PetscInt       maxConeSize, maxSupportSize, maxClosure, maxNeighbors;
2163   PetscInt       dim, cellDim, depth = 0, faceDepth, cStart, cEnd, c, numCells, cell;
2164   PetscErrorCode ierr;
2165 
2166   PetscFunctionBegin;
2167   /* For parallel partitioning, I think you have to communicate supports */
2168   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2169   cellDim = dim - cellHeight;
2170   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2171   ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
2172   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
2173   if (cEnd - cStart == 0) {
2174     if (numVertices) *numVertices = 0;
2175     if (offsets)   *offsets   = NULL;
2176     if (adjacency) *adjacency = NULL;
2177     PetscFunctionReturn(0);
2178   }
2179   numCells  = cEnd - cStart;
2180   faceDepth = depth - cellHeight;
2181   /* Setup face recognition */
2182   if (faceDepth == 1) {
2183     PetscInt cornersSeen[30] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Could use PetscBT */
2184 
2185     for (c = cStart; c < cEnd; ++c) {
2186       PetscInt corners;
2187 
2188       ierr = DMPlexGetConeSize(dm, c, &corners);CHKERRQ(ierr);
2189       if (!cornersSeen[corners]) {
2190         PetscInt nFV;
2191 
2192         if (numFaceCases >= maxFaceCases) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Exceeded maximum number of face recognition cases");
2193         cornersSeen[corners] = 1;
2194 
2195         ierr = DMPlexGetNumFaceVertices(dm, cellDim, corners, &nFV);CHKERRQ(ierr);
2196 
2197         numFaceVertices[numFaceCases++] = nFV;
2198       }
2199     }
2200   }
2201   maxClosure   = 2*PetscMax(PetscPowInt(maxConeSize,depth+1),PetscPowInt(maxSupportSize,depth+1));
2202   maxNeighbors = PetscPowInt(maxConeSize,depth+1)*PetscPowInt(maxSupportSize,depth+1);
2203   ierr         = PetscMalloc2(maxNeighbors,PetscInt,&neighborCells,maxClosure,PetscInt,&tmpClosure);CHKERRQ(ierr);
2204   ierr         = PetscMalloc((numCells+1) * sizeof(PetscInt), &off);CHKERRQ(ierr);
2205   ierr         = PetscMemzero(off, (numCells+1) * sizeof(PetscInt));CHKERRQ(ierr);
2206   /* Count neighboring cells */
2207   for (cell = cStart; cell < cEnd; ++cell) {
2208     PetscInt numNeighbors = maxNeighbors, n;
2209 
2210     ierr = DMPlexGetAdjacencySingleLevel_Internal(dm, cell, PETSC_TRUE, tmpClosure, &numNeighbors, neighborCells);CHKERRQ(ierr);
2211     /* Get meet with each cell, and check with recognizer (could optimize to check each pair only once) */
2212     for (n = 0; n < numNeighbors; ++n) {
2213       PetscInt        cellPair[2];
2214       PetscBool       found    = faceDepth > 1 ? PETSC_TRUE : PETSC_FALSE;
2215       PetscInt        meetSize = 0;
2216       const PetscInt *meet    = NULL;
2217 
2218       cellPair[0] = cell; cellPair[1] = neighborCells[n];
2219       if (cellPair[0] == cellPair[1]) continue;
2220       if (!found) {
2221         ierr = DMPlexGetMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2222         if (meetSize) {
2223           PetscInt f;
2224 
2225           for (f = 0; f < numFaceCases; ++f) {
2226             if (numFaceVertices[f] == meetSize) {
2227               found = PETSC_TRUE;
2228               break;
2229             }
2230           }
2231         }
2232         ierr = DMPlexRestoreMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2233       }
2234       if (found) ++off[cell-cStart+1];
2235     }
2236   }
2237   /* Prefix sum */
2238   for (cell = 1; cell <= numCells; ++cell) off[cell] += off[cell-1];
2239 
2240   if (adjacency) {
2241     ierr = PetscMalloc(off[numCells] * sizeof(PetscInt), &adj);CHKERRQ(ierr);
2242     /* Get neighboring cells */
2243     for (cell = cStart; cell < cEnd; ++cell) {
2244       PetscInt numNeighbors = maxNeighbors, n;
2245       PetscInt cellOffset   = 0;
2246 
2247       ierr = DMPlexGetAdjacencySingleLevel_Internal(dm, cell, PETSC_TRUE, tmpClosure, &numNeighbors, neighborCells);CHKERRQ(ierr);
2248       /* Get meet with each cell, and check with recognizer (could optimize to check each pair only once) */
2249       for (n = 0; n < numNeighbors; ++n) {
2250         PetscInt        cellPair[2];
2251         PetscBool       found    = faceDepth > 1 ? PETSC_TRUE : PETSC_FALSE;
2252         PetscInt        meetSize = 0;
2253         const PetscInt *meet    = NULL;
2254 
2255         cellPair[0] = cell; cellPair[1] = neighborCells[n];
2256         if (cellPair[0] == cellPair[1]) continue;
2257         if (!found) {
2258           ierr = DMPlexGetMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2259           if (meetSize) {
2260             PetscInt f;
2261 
2262             for (f = 0; f < numFaceCases; ++f) {
2263               if (numFaceVertices[f] == meetSize) {
2264                 found = PETSC_TRUE;
2265                 break;
2266               }
2267             }
2268           }
2269           ierr = DMPlexRestoreMeet(dm, 2, cellPair, &meetSize, &meet);CHKERRQ(ierr);
2270         }
2271         if (found) {
2272           adj[off[cell-cStart]+cellOffset] = neighborCells[n];
2273           ++cellOffset;
2274         }
2275       }
2276     }
2277   }
2278   ierr = PetscFree2(neighborCells,tmpClosure);CHKERRQ(ierr);
2279   if (numVertices) *numVertices = numCells;
2280   if (offsets)   *offsets   = off;
2281   if (adjacency) *adjacency = adj;
2282   PetscFunctionReturn(0);
2283 }
2284 
2285 #if defined(PETSC_HAVE_CHACO)
2286 #if defined(PETSC_HAVE_UNISTD_H)
2287 #include <unistd.h>
2288 #endif
2289 /* Chaco does not have an include file */
2290 PETSC_EXTERN int interface(int nvtxs, int *start, int *adjacency, int *vwgts,
2291                        float *ewgts, float *x, float *y, float *z, char *outassignname,
2292                        char *outfilename, short *assignment, int architecture, int ndims_tot,
2293                        int mesh_dims[3], double *goal, int global_method, int local_method,
2294                        int rqi_flag, int vmax, int ndims, double eigtol, long seed);
2295 
2296 extern int FREE_GRAPH;
2297 
2298 #undef __FUNCT__
2299 #define __FUNCT__ "DMPlexPartition_Chaco"
2300 PetscErrorCode DMPlexPartition_Chaco(DM dm, PetscInt numVertices, PetscInt start[], PetscInt adjacency[], PetscSection *partSection, IS *partition)
2301 {
2302   enum {DEFAULT_METHOD = 1, INERTIAL_METHOD = 3};
2303   MPI_Comm       comm;
2304   int            nvtxs          = numVertices; /* number of vertices in full graph */
2305   int           *vwgts          = NULL;   /* weights for all vertices */
2306   float         *ewgts          = NULL;   /* weights for all edges */
2307   float         *x              = NULL, *y = NULL, *z = NULL; /* coordinates for inertial method */
2308   char          *outassignname  = NULL;   /*  name of assignment output file */
2309   char          *outfilename    = NULL;   /* output file name */
2310   int            architecture   = 1;      /* 0 => hypercube, d => d-dimensional mesh */
2311   int            ndims_tot      = 0;      /* total number of cube dimensions to divide */
2312   int            mesh_dims[3];            /* dimensions of mesh of processors */
2313   double        *goal          = NULL;    /* desired set sizes for each set */
2314   int            global_method = 1;       /* global partitioning algorithm */
2315   int            local_method  = 1;       /* local partitioning algorithm */
2316   int            rqi_flag      = 0;       /* should I use RQI/Symmlq eigensolver? */
2317   int            vmax          = 200;     /* how many vertices to coarsen down to? */
2318   int            ndims         = 1;       /* number of eigenvectors (2^d sets) */
2319   double         eigtol        = 0.001;   /* tolerance on eigenvectors */
2320   long           seed          = 123636512; /* for random graph mutations */
2321   short int     *assignment;              /* Output partition */
2322   int            fd_stdout, fd_pipe[2];
2323   PetscInt      *points;
2324   PetscMPIInt    commSize;
2325   int            i, v, p;
2326   PetscErrorCode ierr;
2327 
2328   PetscFunctionBegin;
2329   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2330   ierr = MPI_Comm_size(comm, &commSize);CHKERRQ(ierr);
2331   if (!numVertices) {
2332     ierr = PetscSectionCreate(comm, partSection);CHKERRQ(ierr);
2333     ierr = PetscSectionSetChart(*partSection, 0, commSize);CHKERRQ(ierr);
2334     ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2335     ierr = ISCreateGeneral(comm, 0, NULL, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2336     PetscFunctionReturn(0);
2337   }
2338   FREE_GRAPH = 0;                         /* Do not let Chaco free my memory */
2339   for (i = 0; i < start[numVertices]; ++i) ++adjacency[i];
2340 
2341   if (global_method == INERTIAL_METHOD) {
2342     /* manager.createCellCoordinates(nvtxs, &x, &y, &z); */
2343     SETERRQ(comm, PETSC_ERR_SUP, "Inertial partitioning not yet supported");
2344   }
2345   mesh_dims[0] = commSize;
2346   mesh_dims[1] = 1;
2347   mesh_dims[2] = 1;
2348   ierr = PetscMalloc(nvtxs * sizeof(short int), &assignment);CHKERRQ(ierr);
2349   /* Chaco outputs to stdout. We redirect this to a buffer. */
2350   /* TODO: check error codes for UNIX calls */
2351 #if defined(PETSC_HAVE_UNISTD_H)
2352   {
2353     int piperet;
2354     piperet = pipe(fd_pipe);
2355     if (piperet) SETERRQ(comm,PETSC_ERR_SYS,"Could not create pipe");
2356     fd_stdout = dup(1);
2357     close(1);
2358     dup2(fd_pipe[1], 1);
2359   }
2360 #endif
2361   ierr = interface(nvtxs, (int*) start, (int*) adjacency, vwgts, ewgts, x, y, z, outassignname, outfilename,
2362                    assignment, architecture, ndims_tot, mesh_dims, goal, global_method, local_method, rqi_flag,
2363                    vmax, ndims, eigtol, seed);
2364 #if defined(PETSC_HAVE_UNISTD_H)
2365   {
2366     char msgLog[10000];
2367     int  count;
2368 
2369     fflush(stdout);
2370     count = read(fd_pipe[0], msgLog, (10000-1)*sizeof(char));
2371     if (count < 0) count = 0;
2372     msgLog[count] = 0;
2373     close(1);
2374     dup2(fd_stdout, 1);
2375     close(fd_stdout);
2376     close(fd_pipe[0]);
2377     close(fd_pipe[1]);
2378     if (ierr) SETERRQ1(comm, PETSC_ERR_LIB, "Error in Chaco library: %s", msgLog);
2379   }
2380 #endif
2381   /* Convert to PetscSection+IS */
2382   ierr = PetscSectionCreate(comm, partSection);CHKERRQ(ierr);
2383   ierr = PetscSectionSetChart(*partSection, 0, commSize);CHKERRQ(ierr);
2384   for (v = 0; v < nvtxs; ++v) {
2385     ierr = PetscSectionAddDof(*partSection, assignment[v], 1);CHKERRQ(ierr);
2386   }
2387   ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2388   ierr = PetscMalloc(nvtxs * sizeof(PetscInt), &points);CHKERRQ(ierr);
2389   for (p = 0, i = 0; p < commSize; ++p) {
2390     for (v = 0; v < nvtxs; ++v) {
2391       if (assignment[v] == p) points[i++] = v;
2392     }
2393   }
2394   if (i != nvtxs) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of points %D should be %D", i, nvtxs);
2395   ierr = ISCreateGeneral(comm, nvtxs, points, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2396   if (global_method == INERTIAL_METHOD) {
2397     /* manager.destroyCellCoordinates(nvtxs, &x, &y, &z); */
2398   }
2399   ierr = PetscFree(assignment);CHKERRQ(ierr);
2400   for (i = 0; i < start[numVertices]; ++i) --adjacency[i];
2401   PetscFunctionReturn(0);
2402 }
2403 #endif
2404 
2405 #if defined(PETSC_HAVE_PARMETIS)
2406 #undef __FUNCT__
2407 #define __FUNCT__ "DMPlexPartition_ParMetis"
2408 PetscErrorCode DMPlexPartition_ParMetis(DM dm, PetscInt numVertices, PetscInt start[], PetscInt adjacency[], PetscSection *partSection, IS *partition)
2409 {
2410   PetscFunctionBegin;
2411   SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "ParMetis not yet supported");
2412   PetscFunctionReturn(0);
2413 }
2414 #endif
2415 
2416 #undef __FUNCT__
2417 #define __FUNCT__ "DMPlexEnlargePartition"
2418 /* Expand the partition by BFS on the adjacency graph */
2419 PetscErrorCode DMPlexEnlargePartition(DM dm, const PetscInt start[], const PetscInt adjacency[], PetscSection origPartSection, IS origPartition, PetscSection *partSection, IS *partition)
2420 {
2421   PetscHashI      h;
2422   const PetscInt *points;
2423   PetscInt      **tmpPoints, *newPoints, totPoints = 0;
2424   PetscInt        pStart, pEnd, part, q;
2425   PetscErrorCode  ierr;
2426 
2427   PetscFunctionBegin;
2428   PetscHashICreate(h);
2429   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), partSection);CHKERRQ(ierr);
2430   ierr = PetscSectionGetChart(origPartSection, &pStart, &pEnd);CHKERRQ(ierr);
2431   ierr = PetscSectionSetChart(*partSection, pStart, pEnd);CHKERRQ(ierr);
2432   ierr = ISGetIndices(origPartition, &points);CHKERRQ(ierr);
2433   ierr = PetscMalloc((pEnd - pStart) * sizeof(PetscInt*), &tmpPoints);CHKERRQ(ierr);
2434   for (part = pStart; part < pEnd; ++part) {
2435     PetscInt numPoints, nP, numNewPoints, off, p, n = 0;
2436 
2437     PetscHashIClear(h);
2438     ierr = PetscSectionGetDof(origPartSection, part, &numPoints);CHKERRQ(ierr);
2439     ierr = PetscSectionGetOffset(origPartSection, part, &off);CHKERRQ(ierr);
2440     /* Add all existing points to h */
2441     for (p = 0; p < numPoints; ++p) {
2442       const PetscInt point = points[off+p];
2443       PetscHashIAdd(h, point, 1);
2444     }
2445     PetscHashISize(h, nP);
2446     if (nP != numPoints) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid partition has %d points, but only %d were unique", numPoints, nP);
2447     /* Add all points in next BFS level */
2448     /*   TODO We are brute forcing here, but could check the adjacency size to find the boundary */
2449     for (p = 0; p < numPoints; ++p) {
2450       const PetscInt point = points[off+p];
2451       PetscInt       s     = start[point], e = start[point+1], a;
2452 
2453       for (a = s; a < e; ++a) PetscHashIAdd(h, adjacency[a], 1);
2454     }
2455     PetscHashISize(h, numNewPoints);
2456     ierr = PetscSectionSetDof(*partSection, part, numNewPoints);CHKERRQ(ierr);
2457     ierr = PetscMalloc(numNewPoints * sizeof(PetscInt), &tmpPoints[part]);CHKERRQ(ierr);
2458     if (numNewPoints) PetscHashIGetKeys(h, n, tmpPoints[part]); /* Should not need this conditional */
2459     totPoints += numNewPoints;
2460   }
2461   ierr = ISRestoreIndices(origPartition, &points);CHKERRQ(ierr);
2462   PetscHashIDestroy(h);
2463   ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2464   ierr = PetscMalloc(totPoints * sizeof(PetscInt), &newPoints);CHKERRQ(ierr);
2465   for (part = pStart, q = 0; part < pEnd; ++part) {
2466     PetscInt numPoints, p;
2467 
2468     ierr = PetscSectionGetDof(*partSection, part, &numPoints);CHKERRQ(ierr);
2469     for (p = 0; p < numPoints; ++p, ++q) newPoints[q] = tmpPoints[part][p];
2470     ierr = PetscFree(tmpPoints[part]);CHKERRQ(ierr);
2471   }
2472   ierr = PetscFree(tmpPoints);CHKERRQ(ierr);
2473   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), totPoints, newPoints, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2474   PetscFunctionReturn(0);
2475 }
2476 
2477 #undef __FUNCT__
2478 #define __FUNCT__ "DMPlexCreatePartition"
2479 /*
2480   DMPlexCreatePartition - Create a non-overlapping partition of the points at the given height
2481 
2482   Collective on DM
2483 
2484   Input Parameters:
2485   + dm - The DM
2486   . height - The height for points in the partition
2487   - enlarge - Expand each partition with neighbors
2488 
2489   Output Parameters:
2490   + partSection - The PetscSection giving the division of points by partition
2491   . partition - The list of points by partition
2492   . origPartSection - If enlarge is true, the PetscSection giving the division of points before enlarging by partition, otherwise NULL
2493   - origPartition - If enlarge is true, the list of points before enlarging by partition, otherwise NULL
2494 
2495   Level: developer
2496 
2497 .seealso DMPlexDistribute()
2498 */
2499 PetscErrorCode DMPlexCreatePartition(DM dm, PetscInt height, PetscBool enlarge, PetscSection *partSection, IS *partition, PetscSection *origPartSection, IS *origPartition)
2500 {
2501   PetscMPIInt    size;
2502   PetscErrorCode ierr;
2503 
2504   PetscFunctionBegin;
2505   ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
2506 
2507   *origPartSection = NULL;
2508   *origPartition   = NULL;
2509   if (size == 1) {
2510     PetscInt *points;
2511     PetscInt  cStart, cEnd, c;
2512 
2513     ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
2514     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), partSection);CHKERRQ(ierr);
2515     ierr = PetscSectionSetChart(*partSection, 0, size);CHKERRQ(ierr);
2516     ierr = PetscSectionSetDof(*partSection, 0, cEnd-cStart);CHKERRQ(ierr);
2517     ierr = PetscSectionSetUp(*partSection);CHKERRQ(ierr);
2518     ierr = PetscMalloc((cEnd - cStart) * sizeof(PetscInt), &points);CHKERRQ(ierr);
2519     for (c = cStart; c < cEnd; ++c) points[c] = c;
2520     ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), cEnd-cStart, points, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2521     PetscFunctionReturn(0);
2522   }
2523   if (height == 0) {
2524     PetscInt  numVertices;
2525     PetscInt *start     = NULL;
2526     PetscInt *adjacency = NULL;
2527 
2528     ierr = DMPlexCreateNeighborCSR(dm, 0, &numVertices, &start, &adjacency);CHKERRQ(ierr);
2529     if (1) {
2530 #if defined(PETSC_HAVE_CHACO)
2531       ierr = DMPlexPartition_Chaco(dm, numVertices, start, adjacency, partSection, partition);CHKERRQ(ierr);
2532 #endif
2533     } else {
2534 #if defined(PETSC_HAVE_PARMETIS)
2535       ierr = DMPlexPartition_ParMetis(dm, numVertices, start, adjacency, partSection, partition);CHKERRQ(ierr);
2536 #endif
2537     }
2538     if (enlarge) {
2539       *origPartSection = *partSection;
2540       *origPartition   = *partition;
2541 
2542       ierr = DMPlexEnlargePartition(dm, start, adjacency, *origPartSection, *origPartition, partSection, partition);CHKERRQ(ierr);
2543     }
2544     ierr = PetscFree(start);CHKERRQ(ierr);
2545     ierr = PetscFree(adjacency);CHKERRQ(ierr);
2546 # if 0
2547   } else if (height == 1) {
2548     /* Build the dual graph for faces and partition the hypergraph */
2549     PetscInt numEdges;
2550 
2551     buildFaceCSRV(mesh, mesh->getFactory()->getNumbering(mesh, mesh->depth()-1), &numEdges, &start, &adjacency, GraphPartitioner::zeroBase());
2552     GraphPartitioner().partition(numEdges, start, adjacency, partition, manager);
2553     destroyCSR(numEdges, start, adjacency);
2554 #endif
2555   } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid partition height %D", height);
2556   PetscFunctionReturn(0);
2557 }
2558 
2559 #undef __FUNCT__
2560 #define __FUNCT__ "DMPlexCreatePartitionClosure"
2561 PetscErrorCode DMPlexCreatePartitionClosure(DM dm, PetscSection pointSection, IS pointPartition, PetscSection *section, IS *partition)
2562 {
2563   /* const PetscInt  height = 0; */
2564   const PetscInt *partArray;
2565   PetscInt       *allPoints, *packPoints;
2566   PetscInt        rStart, rEnd, rank, pStart, pEnd, newSize;
2567   PetscErrorCode  ierr;
2568   PetscBT         bt;
2569   PetscSegBuffer  segpack,segpart;
2570 
2571   PetscFunctionBegin;
2572   ierr = PetscSectionGetChart(pointSection, &rStart, &rEnd);CHKERRQ(ierr);
2573   ierr = ISGetIndices(pointPartition, &partArray);CHKERRQ(ierr);
2574   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
2575   ierr = PetscSectionSetChart(*section, rStart, rEnd);CHKERRQ(ierr);
2576   ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
2577   ierr = PetscBTCreate(pEnd-pStart,&bt);CHKERRQ(ierr);
2578   ierr = PetscSegBufferCreate(sizeof(PetscInt),1000,&segpack);CHKERRQ(ierr);
2579   ierr = PetscSegBufferCreate(sizeof(PetscInt),1000,&segpart);CHKERRQ(ierr);
2580   for (rank = rStart; rank < rEnd; ++rank) {
2581     PetscInt partSize = 0, numPoints, offset, p, *PETSC_RESTRICT placePoints;
2582 
2583     ierr = PetscSectionGetDof(pointSection, rank, &numPoints);CHKERRQ(ierr);
2584     ierr = PetscSectionGetOffset(pointSection, rank, &offset);CHKERRQ(ierr);
2585     for (p = 0; p < numPoints; ++p) {
2586       PetscInt  point   = partArray[offset+p], closureSize, c;
2587       PetscInt *closure = NULL;
2588 
2589       /* TODO Include support for height > 0 case */
2590       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2591       for (c=0; c<closureSize; c++) {
2592         PetscInt cpoint = closure[c*2];
2593         if (!PetscBTLookupSet(bt,cpoint-pStart)) {
2594           PetscInt *PETSC_RESTRICT pt;
2595           partSize++;
2596           ierr = PetscSegBufferGetInts(segpart,1,&pt);CHKERRQ(ierr);
2597           *pt = cpoint;
2598         }
2599       }
2600       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2601     }
2602     ierr = PetscSectionSetDof(*section, rank, partSize);CHKERRQ(ierr);
2603     ierr = PetscSegBufferGetInts(segpack,partSize,&placePoints);CHKERRQ(ierr);
2604     ierr = PetscSegBufferExtractTo(segpart,placePoints);CHKERRQ(ierr);
2605     ierr = PetscSortInt(partSize,placePoints);CHKERRQ(ierr);
2606     for (p=0; p<partSize; p++) {ierr = PetscBTClear(bt,placePoints[p]-pStart);CHKERRQ(ierr);}
2607   }
2608   ierr = PetscBTDestroy(&bt);CHKERRQ(ierr);
2609   ierr = PetscSegBufferDestroy(&segpart);CHKERRQ(ierr);
2610 
2611   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
2612   ierr = PetscSectionGetStorageSize(*section, &newSize);CHKERRQ(ierr);
2613   ierr = PetscMalloc(newSize * sizeof(PetscInt), &allPoints);CHKERRQ(ierr);
2614 
2615   ierr = PetscSegBufferExtractInPlace(segpack,&packPoints);CHKERRQ(ierr);
2616   for (rank = rStart; rank < rEnd; ++rank) {
2617     PetscInt numPoints, offset;
2618 
2619     ierr = PetscSectionGetDof(*section, rank, &numPoints);CHKERRQ(ierr);
2620     ierr = PetscSectionGetOffset(*section, rank, &offset);CHKERRQ(ierr);
2621     ierr = PetscMemcpy(&allPoints[offset], packPoints, numPoints * sizeof(PetscInt));CHKERRQ(ierr);
2622     packPoints += numPoints;
2623   }
2624 
2625   ierr = PetscSegBufferDestroy(&segpack);CHKERRQ(ierr);
2626   ierr = ISRestoreIndices(pointPartition, &partArray);CHKERRQ(ierr);
2627   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), newSize, allPoints, PETSC_OWN_POINTER, partition);CHKERRQ(ierr);
2628   PetscFunctionReturn(0);
2629 }
2630 
2631 #undef __FUNCT__
2632 #define __FUNCT__ "DMPlexDistributeField"
2633 /*
2634   Input Parameters:
2635 . originalSection
2636 , originalVec
2637 
2638   Output Parameters:
2639 . newSection
2640 . newVec
2641 */
2642 PetscErrorCode DMPlexDistributeField(DM dm, PetscSF pointSF, PetscSection originalSection, Vec originalVec, PetscSection newSection, Vec newVec)
2643 {
2644   PetscSF        fieldSF;
2645   PetscInt      *remoteOffsets, fieldSize;
2646   PetscScalar   *originalValues, *newValues;
2647   PetscErrorCode ierr;
2648 
2649   PetscFunctionBegin;
2650   ierr = PetscSFDistributeSection(pointSF, originalSection, &remoteOffsets, newSection);CHKERRQ(ierr);
2651 
2652   ierr = PetscSectionGetStorageSize(newSection, &fieldSize);CHKERRQ(ierr);
2653   ierr = VecSetSizes(newVec, fieldSize, PETSC_DETERMINE);CHKERRQ(ierr);
2654   ierr = VecSetFromOptions(newVec);CHKERRQ(ierr);
2655 
2656   ierr = VecGetArray(originalVec, &originalValues);CHKERRQ(ierr);
2657   ierr = VecGetArray(newVec, &newValues);CHKERRQ(ierr);
2658   ierr = PetscSFCreateSectionSF(pointSF, originalSection, remoteOffsets, newSection, &fieldSF);CHKERRQ(ierr);
2659   ierr = PetscSFBcastBegin(fieldSF, MPIU_SCALAR, originalValues, newValues);CHKERRQ(ierr);
2660   ierr = PetscSFBcastEnd(fieldSF, MPIU_SCALAR, originalValues, newValues);CHKERRQ(ierr);
2661   ierr = PetscSFDestroy(&fieldSF);CHKERRQ(ierr);
2662   ierr = VecRestoreArray(newVec, &newValues);CHKERRQ(ierr);
2663   ierr = VecRestoreArray(originalVec, &originalValues);CHKERRQ(ierr);
2664   PetscFunctionReturn(0);
2665 }
2666 
2667 #undef __FUNCT__
2668 #define __FUNCT__ "DMPlexDistribute"
2669 /*@C
2670   DMPlexDistribute - Distributes the mesh and any associated sections.
2671 
2672   Not Collective
2673 
2674   Input Parameter:
2675 + dm  - The original DMPlex object
2676 . partitioner - The partitioning package, or NULL for the default
2677 - overlap - The overlap of partitions, 0 is the default
2678 
2679   Output Parameter:
2680 . parallelMesh - The distributed DMPlex object, or NULL
2681 
2682   Note: If the mesh was not distributed, the return value is NULL
2683 
2684   Level: intermediate
2685 
2686 .keywords: mesh, elements
2687 .seealso: DMPlexCreate(), DMPlexDistributeByFace()
2688 @*/
2689 PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, DM *dmParallel)
2690 {
2691   DM_Plex               *mesh   = (DM_Plex*) dm->data, *pmesh;
2692   MPI_Comm               comm;
2693   const PetscInt         height = 0;
2694   PetscInt               dim, numRemoteRanks;
2695   IS                     origCellPart,        cellPart,        part;
2696   PetscSection           origCellPartSection, cellPartSection, partSection;
2697   PetscSFNode           *remoteRanks;
2698   PetscSF                partSF, pointSF, coneSF;
2699   ISLocalToGlobalMapping renumbering;
2700   PetscSection           originalConeSection, newConeSection;
2701   PetscInt              *remoteOffsets;
2702   PetscInt              *cones, *newCones, newConesSize;
2703   PetscBool              flg;
2704   PetscMPIInt            rank, numProcs, p;
2705   PetscErrorCode         ierr;
2706 
2707   PetscFunctionBegin;
2708   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2709   PetscValidPointer(dmParallel,4);
2710 
2711   ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
2712   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2713   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2714   ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
2715 
2716   *dmParallel = NULL;
2717   if (numProcs == 1) PetscFunctionReturn(0);
2718 
2719   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
2720   /* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
2721   ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
2722   if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
2723   ierr = DMPlexCreatePartition(dm, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
2724   /* Create SF assuming a serial partition for all processes: Could check for IS length here */
2725   if (!rank) numRemoteRanks = numProcs;
2726   else       numRemoteRanks = 0;
2727   ierr = PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);CHKERRQ(ierr);
2728   for (p = 0; p < numRemoteRanks; ++p) {
2729     remoteRanks[p].rank  = p;
2730     remoteRanks[p].index = 0;
2731   }
2732   ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
2733   ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
2734   ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
2735   if (flg) {
2736     ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
2737     ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2738     ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
2739     if (origCellPart) {
2740       ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
2741       ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2742       ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
2743     }
2744     ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
2745   }
2746   /* Close the partition over the mesh */
2747   ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
2748   ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
2749   ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
2750   /* Create new mesh */
2751   ierr  = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
2752   ierr  = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
2753   ierr  = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
2754   pmesh = (DM_Plex*) (*dmParallel)->data;
2755   /* Distribute sieve points and the global point numbering (replaces creating remote bases) */
2756   ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
2757   if (flg) {
2758     ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
2759     ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2760     ierr = ISView(part, NULL);CHKERRQ(ierr);
2761     ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
2762     ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
2763     ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
2764   }
2765   ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
2766   /* Distribute cone section */
2767   ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
2768   ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
2769   ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
2770   ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
2771   {
2772     PetscInt pStart, pEnd, p;
2773 
2774     ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
2775     for (p = pStart; p < pEnd; ++p) {
2776       PetscInt coneSize;
2777       ierr               = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
2778       pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
2779     }
2780   }
2781   /* Communicate and renumber cones */
2782   ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
2783   ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
2784   ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
2785   ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2786   ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2787   ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
2788   ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
2789   ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
2790   if (flg) {
2791     ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
2792     ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2793     ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
2794     ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
2795     ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
2796   }
2797   ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
2798   ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
2799   ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2800   ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
2801   ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
2802   /* Create supports and stratify sieve */
2803   {
2804     PetscInt pStart, pEnd;
2805 
2806     ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
2807     ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
2808   }
2809   ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
2810   ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
2811   /* Distribute Coordinates */
2812   {
2813     PetscSection originalCoordSection, newCoordSection;
2814     Vec          originalCoordinates, newCoordinates;
2815     const char  *name;
2816 
2817     ierr = DMPlexGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
2818     ierr = DMPlexGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
2819     ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
2820     ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
2821     ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
2822     ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
2823 
2824     ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
2825     ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
2826     ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
2827   }
2828   /* Distribute labels */
2829   ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
2830   {
2831     DMLabel  next      = mesh->labels, newNext = pmesh->labels;
2832     PetscInt numLabels = 0, l;
2833 
2834     /* Bcast number of labels */
2835     while (next) {
2836       ++numLabels; next = next->next;
2837     }
2838     ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
2839     next = mesh->labels;
2840     for (l = 0; l < numLabels; ++l) {
2841       DMLabel         newLabel;
2842       const PetscInt *partArray;
2843       char           *name;
2844       PetscInt       *stratumSizes = NULL, *points = NULL;
2845       PetscMPIInt    *sendcnts     = NULL, *offsets = NULL, *displs = NULL;
2846       PetscInt        nameSize, s, p;
2847       PetscBool       isdepth;
2848       size_t          len = 0;
2849 
2850       /* Bcast name (could filter for no points) */
2851       if (!rank) {ierr = PetscStrlen(next->name, &len);CHKERRQ(ierr);}
2852       nameSize = len;
2853       ierr     = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
2854       ierr     = PetscMalloc(nameSize+1, &name);CHKERRQ(ierr);
2855       if (!rank) {ierr = PetscMemcpy(name, next->name, nameSize+1);CHKERRQ(ierr);}
2856       ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
2857       ierr = PetscStrcmp(name, "depth", &isdepth);CHKERRQ(ierr);
2858       if (isdepth) {            /* skip because "depth" is not distributed */
2859         ierr = PetscFree(name);CHKERRQ(ierr);
2860         if (!rank) next = next->next;
2861         continue;
2862       }
2863       ierr           = PetscNew(struct _n_DMLabel, &newLabel);CHKERRQ(ierr);
2864       newLabel->name = name;
2865       /* Bcast numStrata (could filter for no points in stratum) */
2866       if (!rank) newLabel->numStrata = next->numStrata;
2867       ierr = MPI_Bcast(&newLabel->numStrata, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
2868       ierr = PetscMalloc3(newLabel->numStrata,PetscInt,&newLabel->stratumValues,
2869                           newLabel->numStrata,PetscInt,&newLabel->stratumSizes,
2870                           newLabel->numStrata+1,PetscInt,&newLabel->stratumOffsets);CHKERRQ(ierr);
2871       /* Bcast stratumValues (could filter for no points in stratum) */
2872       if (!rank) {ierr = PetscMemcpy(newLabel->stratumValues, next->stratumValues, next->numStrata * sizeof(PetscInt));CHKERRQ(ierr);}
2873       ierr = MPI_Bcast(newLabel->stratumValues, newLabel->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
2874       /* Find size on each process and Scatter */
2875       if (!rank) {
2876         ierr = ISGetIndices(part, &partArray);CHKERRQ(ierr);
2877         ierr = PetscMalloc(numProcs*next->numStrata * sizeof(PetscInt), &stratumSizes);CHKERRQ(ierr);
2878         ierr = PetscMemzero(stratumSizes, numProcs*next->numStrata * sizeof(PetscInt));CHKERRQ(ierr);
2879         for (s = 0; s < next->numStrata; ++s) {
2880           for (p = next->stratumOffsets[s]; p < next->stratumOffsets[s]+next->stratumSizes[s]; ++p) {
2881             const PetscInt point = next->points[p];
2882             PetscInt       proc;
2883 
2884             for (proc = 0; proc < numProcs; ++proc) {
2885               PetscInt dof, off, pPart;
2886 
2887               ierr = PetscSectionGetDof(partSection, proc, &dof);CHKERRQ(ierr);
2888               ierr = PetscSectionGetOffset(partSection, proc, &off);CHKERRQ(ierr);
2889               for (pPart = off; pPart < off+dof; ++pPart) {
2890                 if (partArray[pPart] == point) {
2891                   ++stratumSizes[proc*next->numStrata+s];
2892                   break;
2893                 }
2894               }
2895             }
2896           }
2897         }
2898         ierr = ISRestoreIndices(part, &partArray);CHKERRQ(ierr);
2899       }
2900       ierr = MPI_Scatter(stratumSizes, newLabel->numStrata, MPIU_INT, newLabel->stratumSizes, newLabel->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
2901       /* Calculate stratumOffsets */
2902       newLabel->stratumOffsets[0] = 0;
2903       for (s = 0; s < newLabel->numStrata; ++s) {
2904         newLabel->stratumOffsets[s+1] = newLabel->stratumSizes[s] + newLabel->stratumOffsets[s];
2905       }
2906       /* Pack points and Scatter */
2907       if (!rank) {
2908         ierr = PetscMalloc3(numProcs,PetscMPIInt,&sendcnts,numProcs,PetscMPIInt,&offsets,numProcs+1,PetscMPIInt,&displs);CHKERRQ(ierr);
2909         displs[0] = 0;
2910         for (p = 0; p < numProcs; ++p) {
2911           sendcnts[p] = 0;
2912           for (s = 0; s < next->numStrata; ++s) {
2913             sendcnts[p] += stratumSizes[p*next->numStrata+s];
2914           }
2915           offsets[p]  = displs[p];
2916           displs[p+1] = displs[p] + sendcnts[p];
2917         }
2918         ierr = PetscMalloc(displs[numProcs] * sizeof(PetscInt), &points);CHKERRQ(ierr);
2919         for (s = 0; s < next->numStrata; ++s) {
2920           for (p = next->stratumOffsets[s]; p < next->stratumOffsets[s]+next->stratumSizes[s]; ++p) {
2921             const PetscInt point = next->points[p];
2922             PetscInt       proc;
2923 
2924             for (proc = 0; proc < numProcs; ++proc) {
2925               PetscInt dof, off, pPart;
2926 
2927               ierr = PetscSectionGetDof(partSection, proc, &dof);CHKERRQ(ierr);
2928               ierr = PetscSectionGetOffset(partSection, proc, &off);CHKERRQ(ierr);
2929               for (pPart = off; pPart < off+dof; ++pPart) {
2930                 if (partArray[pPart] == point) {
2931                   points[offsets[proc]++] = point;
2932                   break;
2933                 }
2934               }
2935             }
2936           }
2937         }
2938       }
2939       ierr = PetscMalloc(newLabel->stratumOffsets[newLabel->numStrata] * sizeof(PetscInt), &newLabel->points);CHKERRQ(ierr);
2940       ierr = MPI_Scatterv(points, sendcnts, displs, MPIU_INT, newLabel->points, newLabel->stratumOffsets[newLabel->numStrata], MPIU_INT, 0, comm);CHKERRQ(ierr);
2941       ierr = PetscFree(points);CHKERRQ(ierr);
2942       ierr = PetscFree3(sendcnts,offsets,displs);CHKERRQ(ierr);
2943       ierr = PetscFree(stratumSizes);CHKERRQ(ierr);
2944       /* Renumber points */
2945       ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newLabel->stratumOffsets[newLabel->numStrata], newLabel->points, NULL, newLabel->points);CHKERRQ(ierr);
2946       /* Sort points */
2947       for (s = 0; s < newLabel->numStrata; ++s) {
2948         ierr = PetscSortInt(newLabel->stratumSizes[s], &newLabel->points[newLabel->stratumOffsets[s]]);CHKERRQ(ierr);
2949       }
2950       /* Insert into list */
2951       if (newNext) newNext->next = newLabel;
2952       else pmesh->labels = newLabel;
2953       newNext = newLabel;
2954       if (!rank) next = next->next;
2955     }
2956   }
2957   ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
2958   /* Cleanup Partition */
2959   ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
2960   ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
2961   ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
2962   ierr = ISDestroy(&part);CHKERRQ(ierr);
2963   /* Create point SF for parallel mesh */
2964   ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
2965   {
2966     const PetscInt *leaves;
2967     PetscSFNode    *remotePoints, *rowners, *lowners;
2968     PetscInt        numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
2969     PetscInt        pStart, pEnd;
2970 
2971     ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
2972     ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
2973     ierr = PetscMalloc2(numRoots,PetscSFNode,&rowners,numLeaves,PetscSFNode,&lowners);CHKERRQ(ierr);
2974     for (p=0; p<numRoots; p++) {
2975       rowners[p].rank  = -1;
2976       rowners[p].index = -1;
2977     }
2978     if (origCellPart) {
2979       /* Make sure cells in the original partition are not assigned to other procs */
2980       const PetscInt *origCells;
2981 
2982       ierr = ISGetIndices(origCellPart, &origCells);CHKERRQ(ierr);
2983       for (p = 0; p < numProcs; ++p) {
2984         PetscInt dof, off, d;
2985 
2986         ierr = PetscSectionGetDof(origCellPartSection, p, &dof);CHKERRQ(ierr);
2987         ierr = PetscSectionGetOffset(origCellPartSection, p, &off);CHKERRQ(ierr);
2988         for (d = off; d < off+dof; ++d) {
2989           rowners[origCells[d]].rank = p;
2990         }
2991       }
2992       ierr = ISRestoreIndices(origCellPart, &origCells);CHKERRQ(ierr);
2993     }
2994     ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
2995     ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
2996 
2997     ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
2998     ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
2999     for (p = 0; p < numLeaves; ++p) {
3000       if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
3001         lowners[p].rank  = rank;
3002         lowners[p].index = leaves ? leaves[p] : p;
3003       } else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
3004         lowners[p].rank  = -2;
3005         lowners[p].index = -2;
3006       }
3007     }
3008     for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
3009       rowners[p].rank  = -3;
3010       rowners[p].index = -3;
3011     }
3012     ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
3013     ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
3014     ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
3015     ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
3016     for (p = 0; p < numLeaves; ++p) {
3017       if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
3018       if (lowners[p].rank != rank) ++numGhostPoints;
3019     }
3020     ierr = PetscMalloc(numGhostPoints * sizeof(PetscInt),    &ghostPoints);CHKERRQ(ierr);
3021     ierr = PetscMalloc(numGhostPoints * sizeof(PetscSFNode), &remotePoints);CHKERRQ(ierr);
3022     for (p = 0, gp = 0; p < numLeaves; ++p) {
3023       if (lowners[p].rank != rank) {
3024         ghostPoints[gp]        = leaves ? leaves[p] : p;
3025         remotePoints[gp].rank  = lowners[p].rank;
3026         remotePoints[gp].index = lowners[p].index;
3027         ++gp;
3028       }
3029     }
3030     ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
3031     ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3032     ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
3033   }
3034   ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
3035   /* Cleanup */
3036   ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);
3037   ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
3038   ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
3039   PetscFunctionReturn(0);
3040 }
3041 
3042 #undef __FUNCT__
3043 #define __FUNCT__ "DMPlexInvertCell"
3044 /*@C
3045   DMPlexInvertCell - This flips tetrahedron and hexahedron orientation since Plex stores them internally with outward normals. Other cells are left untouched.
3046 
3047   Input Parameters:
3048 + numCorners - The number of vertices in a cell
3049 - cone - The incoming cone
3050 
3051   Output Parameter:
3052 . cone - The inverted cone (in-place)
3053 
3054   Level: developer
3055 
3056 .seealso: DMPlexGenerate()
3057 @*/
3058 PetscErrorCode DMPlexInvertCell(PetscInt dim, PetscInt numCorners, int cone[])
3059 {
3060   int tmpc;
3061 
3062   PetscFunctionBegin;
3063   if (dim != 3) PetscFunctionReturn(0);
3064   switch (numCorners) {
3065   case 4:
3066     tmpc    = cone[0];
3067     cone[0] = cone[1];
3068     cone[1] = tmpc;
3069     break;
3070   case 8:
3071     tmpc    = cone[1];
3072     cone[1] = cone[3];
3073     cone[3] = tmpc;
3074     break;
3075   default: break;
3076   }
3077   PetscFunctionReturn(0);
3078 }
3079 
3080 #undef __FUNCT__
3081 #define __FUNCT__ "DMPlexInvertCells_Internal"
3082 /* This is to fix the tetrahedron orientation from TetGen */
3083 PETSC_UNUSED static PetscErrorCode DMPlexInvertCells_Internal(PetscInt dim, PetscInt numCells, PetscInt numCorners, int cells[])
3084 {
3085   PetscInt       bound = numCells*numCorners, coff;
3086   PetscErrorCode ierr;
3087 
3088   PetscFunctionBegin;
3089   for (coff = 0; coff < bound; coff += numCorners) {
3090     ierr = DMPlexInvertCell(dim, numCorners, &cells[coff]);CHKERRQ(ierr);
3091   }
3092   PetscFunctionReturn(0);
3093 }
3094 
3095 #if defined(PETSC_HAVE_TRIANGLE)
3096 #include <triangle.h>
3097 
3098 #undef __FUNCT__
3099 #define __FUNCT__ "InitInput_Triangle"
3100 PetscErrorCode InitInput_Triangle(struct triangulateio *inputCtx)
3101 {
3102   PetscFunctionBegin;
3103   inputCtx->numberofpoints             = 0;
3104   inputCtx->numberofpointattributes    = 0;
3105   inputCtx->pointlist                  = NULL;
3106   inputCtx->pointattributelist         = NULL;
3107   inputCtx->pointmarkerlist            = NULL;
3108   inputCtx->numberofsegments           = 0;
3109   inputCtx->segmentlist                = NULL;
3110   inputCtx->segmentmarkerlist          = NULL;
3111   inputCtx->numberoftriangleattributes = 0;
3112   inputCtx->trianglelist               = NULL;
3113   inputCtx->numberofholes              = 0;
3114   inputCtx->holelist                   = NULL;
3115   inputCtx->numberofregions            = 0;
3116   inputCtx->regionlist                 = NULL;
3117   PetscFunctionReturn(0);
3118 }
3119 
3120 #undef __FUNCT__
3121 #define __FUNCT__ "InitOutput_Triangle"
3122 PetscErrorCode InitOutput_Triangle(struct triangulateio *outputCtx)
3123 {
3124   PetscFunctionBegin;
3125   outputCtx->numberofpoints        = 0;
3126   outputCtx->pointlist             = NULL;
3127   outputCtx->pointattributelist    = NULL;
3128   outputCtx->pointmarkerlist       = NULL;
3129   outputCtx->numberoftriangles     = 0;
3130   outputCtx->trianglelist          = NULL;
3131   outputCtx->triangleattributelist = NULL;
3132   outputCtx->neighborlist          = NULL;
3133   outputCtx->segmentlist           = NULL;
3134   outputCtx->segmentmarkerlist     = NULL;
3135   outputCtx->numberofedges         = 0;
3136   outputCtx->edgelist              = NULL;
3137   outputCtx->edgemarkerlist        = NULL;
3138   PetscFunctionReturn(0);
3139 }
3140 
3141 #undef __FUNCT__
3142 #define __FUNCT__ "FiniOutput_Triangle"
3143 PetscErrorCode FiniOutput_Triangle(struct triangulateio *outputCtx)
3144 {
3145   PetscFunctionBegin;
3146   free(outputCtx->pointmarkerlist);
3147   free(outputCtx->edgelist);
3148   free(outputCtx->edgemarkerlist);
3149   free(outputCtx->trianglelist);
3150   free(outputCtx->neighborlist);
3151   PetscFunctionReturn(0);
3152 }
3153 
3154 #undef __FUNCT__
3155 #define __FUNCT__ "DMPlexGenerate_Triangle"
3156 PetscErrorCode DMPlexGenerate_Triangle(DM boundary, PetscBool interpolate, DM *dm)
3157 {
3158   MPI_Comm             comm;
3159   PetscInt             dim              = 2;
3160   const PetscBool      createConvexHull = PETSC_FALSE;
3161   const PetscBool      constrained      = PETSC_FALSE;
3162   struct triangulateio in;
3163   struct triangulateio out;
3164   PetscInt             vStart, vEnd, v, eStart, eEnd, e;
3165   PetscMPIInt          rank;
3166   PetscErrorCode       ierr;
3167 
3168   PetscFunctionBegin;
3169   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3170   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3171   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3172   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3173   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3174 
3175   in.numberofpoints = vEnd - vStart;
3176   if (in.numberofpoints > 0) {
3177     PetscSection coordSection;
3178     Vec          coordinates;
3179     PetscScalar *array;
3180 
3181     ierr = PetscMalloc(in.numberofpoints*dim * sizeof(double), &in.pointlist);CHKERRQ(ierr);
3182     ierr = PetscMalloc(in.numberofpoints * sizeof(int), &in.pointmarkerlist);CHKERRQ(ierr);
3183     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3184     ierr = DMPlexGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3185     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3186     for (v = vStart; v < vEnd; ++v) {
3187       const PetscInt idx = v - vStart;
3188       PetscInt       off, d;
3189 
3190       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3191       for (d = 0; d < dim; ++d) {
3192         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3193       }
3194       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3195     }
3196     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3197   }
3198   ierr  = DMPlexGetHeightStratum(boundary, 0, &eStart, &eEnd);CHKERRQ(ierr);
3199   in.numberofsegments = eEnd - eStart;
3200   if (in.numberofsegments > 0) {
3201     ierr = PetscMalloc(in.numberofsegments*2 * sizeof(int), &in.segmentlist);CHKERRQ(ierr);
3202     ierr = PetscMalloc(in.numberofsegments   * sizeof(int), &in.segmentmarkerlist);CHKERRQ(ierr);
3203     for (e = eStart; e < eEnd; ++e) {
3204       const PetscInt  idx = e - eStart;
3205       const PetscInt *cone;
3206 
3207       ierr = DMPlexGetCone(boundary, e, &cone);CHKERRQ(ierr);
3208 
3209       in.segmentlist[idx*2+0] = cone[0] - vStart;
3210       in.segmentlist[idx*2+1] = cone[1] - vStart;
3211 
3212       ierr = DMPlexGetLabelValue(boundary, "marker", e, &in.segmentmarkerlist[idx]);CHKERRQ(ierr);
3213     }
3214   }
3215 #if 0 /* Do not currently support holes */
3216   PetscReal *holeCoords;
3217   PetscInt   h, d;
3218 
3219   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3220   if (in.numberofholes > 0) {
3221     ierr = PetscMalloc(in.numberofholes*dim * sizeof(double), &in.holelist);CHKERRQ(ierr);
3222     for (h = 0; h < in.numberofholes; ++h) {
3223       for (d = 0; d < dim; ++d) {
3224         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3225       }
3226     }
3227   }
3228 #endif
3229   if (!rank) {
3230     char args[32];
3231 
3232     /* Take away 'Q' for verbose output */
3233     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3234     if (createConvexHull) {
3235       ierr = PetscStrcat(args, "c");CHKERRQ(ierr);
3236     }
3237     if (constrained) {
3238       ierr = PetscStrcpy(args, "zepDQ");CHKERRQ(ierr);
3239     }
3240     triangulate(args, &in, &out, NULL);
3241   }
3242   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3243   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3244   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3245   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3246   ierr = PetscFree(in.holelist);CHKERRQ(ierr);
3247 
3248   {
3249     const PetscInt numCorners  = 3;
3250     const PetscInt numCells    = out.numberoftriangles;
3251     const PetscInt numVertices = out.numberofpoints;
3252     const int     *cells      = out.trianglelist;
3253     const double  *meshCoords = out.pointlist;
3254 
3255     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3256     /* Set labels */
3257     for (v = 0; v < numVertices; ++v) {
3258       if (out.pointmarkerlist[v]) {
3259         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3260       }
3261     }
3262     if (interpolate) {
3263       for (e = 0; e < out.numberofedges; e++) {
3264         if (out.edgemarkerlist[e]) {
3265           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3266           const PetscInt *edges;
3267           PetscInt        numEdges;
3268 
3269           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3270           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3271           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3272           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3273         }
3274       }
3275     }
3276     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3277   }
3278 #if 0 /* Do not currently support holes */
3279   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3280 #endif
3281   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3282   PetscFunctionReturn(0);
3283 }
3284 
3285 #undef __FUNCT__
3286 #define __FUNCT__ "DMPlexRefine_Triangle"
3287 PetscErrorCode DMPlexRefine_Triangle(DM dm, double *maxVolumes, DM *dmRefined)
3288 {
3289   MPI_Comm             comm;
3290   PetscInt             dim  = 2;
3291   struct triangulateio in;
3292   struct triangulateio out;
3293   PetscInt             vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3294   PetscMPIInt          rank;
3295   PetscErrorCode       ierr;
3296 
3297   PetscFunctionBegin;
3298   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3299   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3300   ierr = InitInput_Triangle(&in);CHKERRQ(ierr);
3301   ierr = InitOutput_Triangle(&out);CHKERRQ(ierr);
3302   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3303   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3304   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3305 
3306   in.numberofpoints = vEnd - vStart;
3307   if (in.numberofpoints > 0) {
3308     PetscSection coordSection;
3309     Vec          coordinates;
3310     PetscScalar *array;
3311 
3312     ierr = PetscMalloc(in.numberofpoints*dim * sizeof(double), &in.pointlist);CHKERRQ(ierr);
3313     ierr = PetscMalloc(in.numberofpoints * sizeof(int), &in.pointmarkerlist);CHKERRQ(ierr);
3314     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3315     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3316     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3317     for (v = vStart; v < vEnd; ++v) {
3318       const PetscInt idx = v - vStart;
3319       PetscInt       off, d;
3320 
3321       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3322       for (d = 0; d < dim; ++d) {
3323         in.pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3324       }
3325       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3326     }
3327     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3328   }
3329   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3330 
3331   in.numberofcorners   = 3;
3332   in.numberoftriangles = cEnd - cStart;
3333 
3334   in.trianglearealist  = (double*) maxVolumes;
3335   if (in.numberoftriangles > 0) {
3336     ierr = PetscMalloc(in.numberoftriangles*in.numberofcorners * sizeof(int), &in.trianglelist);CHKERRQ(ierr);
3337     for (c = cStart; c < cEnd; ++c) {
3338       const PetscInt idx      = c - cStart;
3339       PetscInt      *closure = NULL;
3340       PetscInt       closureSize;
3341 
3342       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3343       if ((closureSize != 4) && (closureSize != 7)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a triangle, %D vertices in closure", closureSize);
3344       for (v = 0; v < 3; ++v) {
3345         in.trianglelist[idx*in.numberofcorners + v] = closure[(v+closureSize-3)*2] - vStart;
3346       }
3347       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3348     }
3349   }
3350   /* TODO: Segment markers are missing on input */
3351 #if 0 /* Do not currently support holes */
3352   PetscReal *holeCoords;
3353   PetscInt   h, d;
3354 
3355   ierr = DMPlexGetHoles(boundary, &in.numberofholes, &holeCords);CHKERRQ(ierr);
3356   if (in.numberofholes > 0) {
3357     ierr = PetscMalloc(in.numberofholes*dim * sizeof(double), &in.holelist);CHKERRQ(ierr);
3358     for (h = 0; h < in.numberofholes; ++h) {
3359       for (d = 0; d < dim; ++d) {
3360         in.holelist[h*dim+d] = holeCoords[h*dim+d];
3361       }
3362     }
3363   }
3364 #endif
3365   if (!rank) {
3366     char args[32];
3367 
3368     /* Take away 'Q' for verbose output */
3369     ierr = PetscStrcpy(args, "pqezQra");CHKERRQ(ierr);
3370     triangulate(args, &in, &out, NULL);
3371   }
3372   ierr = PetscFree(in.pointlist);CHKERRQ(ierr);
3373   ierr = PetscFree(in.pointmarkerlist);CHKERRQ(ierr);
3374   ierr = PetscFree(in.segmentlist);CHKERRQ(ierr);
3375   ierr = PetscFree(in.segmentmarkerlist);CHKERRQ(ierr);
3376   ierr = PetscFree(in.trianglelist);CHKERRQ(ierr);
3377 
3378   {
3379     const PetscInt numCorners  = 3;
3380     const PetscInt numCells    = out.numberoftriangles;
3381     const PetscInt numVertices = out.numberofpoints;
3382     const int     *cells      = out.trianglelist;
3383     const double  *meshCoords = out.pointlist;
3384     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3385 
3386     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3387     /* Set labels */
3388     for (v = 0; v < numVertices; ++v) {
3389       if (out.pointmarkerlist[v]) {
3390         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3391       }
3392     }
3393     if (interpolate) {
3394       PetscInt e;
3395 
3396       for (e = 0; e < out.numberofedges; e++) {
3397         if (out.edgemarkerlist[e]) {
3398           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3399           const PetscInt *edges;
3400           PetscInt        numEdges;
3401 
3402           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3403           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3404           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3405           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3406         }
3407       }
3408     }
3409     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3410   }
3411 #if 0 /* Do not currently support holes */
3412   ierr = DMPlexCopyHoles(*dm, boundary);CHKERRQ(ierr);
3413 #endif
3414   ierr = FiniOutput_Triangle(&out);CHKERRQ(ierr);
3415   PetscFunctionReturn(0);
3416 }
3417 #endif
3418 
3419 #if defined(PETSC_HAVE_TETGEN)
3420 #include <tetgen.h>
3421 #undef __FUNCT__
3422 #define __FUNCT__ "DMPlexGenerate_Tetgen"
3423 PetscErrorCode DMPlexGenerate_Tetgen(DM boundary, PetscBool interpolate, DM *dm)
3424 {
3425   MPI_Comm       comm;
3426   const PetscInt dim  = 3;
3427   ::tetgenio     in;
3428   ::tetgenio     out;
3429   PetscInt       vStart, vEnd, v, fStart, fEnd, f;
3430   PetscMPIInt    rank;
3431   PetscErrorCode ierr;
3432 
3433   PetscFunctionBegin;
3434   ierr              = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3435   ierr              = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3436   ierr              = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3437   in.numberofpoints = vEnd - vStart;
3438   if (in.numberofpoints > 0) {
3439     PetscSection coordSection;
3440     Vec          coordinates;
3441     PetscScalar *array;
3442 
3443     in.pointlist       = new double[in.numberofpoints*dim];
3444     in.pointmarkerlist = new int[in.numberofpoints];
3445 
3446     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3447     ierr = DMPlexGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3448     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3449     for (v = vStart; v < vEnd; ++v) {
3450       const PetscInt idx = v - vStart;
3451       PetscInt       off, d;
3452 
3453       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3454       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3455       ierr = DMPlexGetLabelValue(boundary, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3456     }
3457     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3458   }
3459   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3460 
3461   in.numberoffacets = fEnd - fStart;
3462   if (in.numberoffacets > 0) {
3463     in.facetlist       = new tetgenio::facet[in.numberoffacets];
3464     in.facetmarkerlist = new int[in.numberoffacets];
3465     for (f = fStart; f < fEnd; ++f) {
3466       const PetscInt idx     = f - fStart;
3467       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v;
3468 
3469       in.facetlist[idx].numberofpolygons = 1;
3470       in.facetlist[idx].polygonlist      = new tetgenio::polygon[in.facetlist[idx].numberofpolygons];
3471       in.facetlist[idx].numberofholes    = 0;
3472       in.facetlist[idx].holelist         = NULL;
3473 
3474       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3475       for (p = 0; p < numPoints*2; p += 2) {
3476         const PetscInt point = points[p];
3477         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3478       }
3479 
3480       tetgenio::polygon *poly = in.facetlist[idx].polygonlist;
3481       poly->numberofvertices = numVertices;
3482       poly->vertexlist       = new int[poly->numberofvertices];
3483       for (v = 0; v < numVertices; ++v) {
3484         const PetscInt vIdx = points[v] - vStart;
3485         poly->vertexlist[v] = vIdx;
3486       }
3487       ierr = DMPlexGetLabelValue(boundary, "marker", f, &in.facetmarkerlist[idx]);CHKERRQ(ierr);
3488       ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3489     }
3490   }
3491   if (!rank) {
3492     char args[32];
3493 
3494     /* Take away 'Q' for verbose output */
3495     ierr = PetscStrcpy(args, "pqezQ");CHKERRQ(ierr);
3496     ::tetrahedralize(args, &in, &out);
3497   }
3498   {
3499     const PetscInt numCorners  = 4;
3500     const PetscInt numCells    = out.numberoftetrahedra;
3501     const PetscInt numVertices = out.numberofpoints;
3502     const double   *meshCoords = out.pointlist;
3503     int            *cells      = out.tetrahedronlist;
3504 
3505     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3506     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3507     /* Set labels */
3508     for (v = 0; v < numVertices; ++v) {
3509       if (out.pointmarkerlist[v]) {
3510         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3511       }
3512     }
3513     if (interpolate) {
3514       PetscInt e;
3515 
3516       for (e = 0; e < out.numberofedges; e++) {
3517         if (out.edgemarkerlist[e]) {
3518           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3519           const PetscInt *edges;
3520           PetscInt        numEdges;
3521 
3522           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3523           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3524           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3525           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3526         }
3527       }
3528       for (f = 0; f < out.numberoftrifaces; f++) {
3529         if (out.trifacemarkerlist[f]) {
3530           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3531           const PetscInt *faces;
3532           PetscInt        numFaces;
3533 
3534           ierr = DMPlexGetJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3535           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3536           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3537           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3538         }
3539       }
3540     }
3541     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3542   }
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 #undef __FUNCT__
3547 #define __FUNCT__ "DMPlexRefine_Tetgen"
3548 PetscErrorCode DMPlexRefine_Tetgen(DM dm, double *maxVolumes, DM *dmRefined)
3549 {
3550   MPI_Comm       comm;
3551   const PetscInt dim  = 3;
3552   ::tetgenio     in;
3553   ::tetgenio     out;
3554   PetscInt       vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3555   PetscMPIInt    rank;
3556   PetscErrorCode ierr;
3557 
3558   PetscFunctionBegin;
3559   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3560   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3561   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3562   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3563   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3564 
3565   in.numberofpoints = vEnd - vStart;
3566   if (in.numberofpoints > 0) {
3567     PetscSection coordSection;
3568     Vec          coordinates;
3569     PetscScalar *array;
3570 
3571     in.pointlist       = new double[in.numberofpoints*dim];
3572     in.pointmarkerlist = new int[in.numberofpoints];
3573 
3574     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3575     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3576     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3577     for (v = vStart; v < vEnd; ++v) {
3578       const PetscInt idx = v - vStart;
3579       PetscInt       off, d;
3580 
3581       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3582       for (d = 0; d < dim; ++d) in.pointlist[idx*dim + d] = array[off+d];
3583       ierr = DMPlexGetLabelValue(dm, "marker", v, &in.pointmarkerlist[idx]);CHKERRQ(ierr);
3584     }
3585     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3586   }
3587   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3588 
3589   in.numberofcorners       = 4;
3590   in.numberoftetrahedra    = cEnd - cStart;
3591   in.tetrahedronvolumelist = (double*) maxVolumes;
3592   if (in.numberoftetrahedra > 0) {
3593     in.tetrahedronlist = new int[in.numberoftetrahedra*in.numberofcorners];
3594     for (c = cStart; c < cEnd; ++c) {
3595       const PetscInt idx      = c - cStart;
3596       PetscInt      *closure = NULL;
3597       PetscInt       closureSize;
3598 
3599       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3600       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3601       for (v = 0; v < 4; ++v) {
3602         in.tetrahedronlist[idx*in.numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3603       }
3604       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3605     }
3606   }
3607   /* TODO: Put in boundary faces with markers */
3608   if (!rank) {
3609     char args[32];
3610 
3611     /* Take away 'Q' for verbose output */
3612     /*ierr = PetscStrcpy(args, "qezQra");CHKERRQ(ierr); */
3613     ierr = PetscStrcpy(args, "qezraVVVV");CHKERRQ(ierr);
3614     ::tetrahedralize(args, &in, &out);
3615   }
3616   in.tetrahedronvolumelist = NULL;
3617 
3618   {
3619     const PetscInt numCorners  = 4;
3620     const PetscInt numCells    = out.numberoftetrahedra;
3621     const PetscInt numVertices = out.numberofpoints;
3622     const double   *meshCoords = out.pointlist;
3623     int            *cells      = out.tetrahedronlist;
3624 
3625     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3626 
3627     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3628     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3629     /* Set labels */
3630     for (v = 0; v < numVertices; ++v) {
3631       if (out.pointmarkerlist[v]) {
3632         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out.pointmarkerlist[v]);CHKERRQ(ierr);
3633       }
3634     }
3635     if (interpolate) {
3636       PetscInt e, f;
3637 
3638       for (e = 0; e < out.numberofedges; e++) {
3639         if (out.edgemarkerlist[e]) {
3640           const PetscInt  vertices[2] = {out.edgelist[e*2+0]+numCells, out.edgelist[e*2+1]+numCells};
3641           const PetscInt *edges;
3642           PetscInt        numEdges;
3643 
3644           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3645           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3646           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out.edgemarkerlist[e]);CHKERRQ(ierr);
3647           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3648         }
3649       }
3650       for (f = 0; f < out.numberoftrifaces; f++) {
3651         if (out.trifacemarkerlist[f]) {
3652           const PetscInt  vertices[3] = {out.trifacelist[f*3+0]+numCells, out.trifacelist[f*3+1]+numCells, out.trifacelist[f*3+2]+numCells};
3653           const PetscInt *faces;
3654           PetscInt        numFaces;
3655 
3656           ierr = DMPlexGetJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3657           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3658           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out.trifacemarkerlist[f]);CHKERRQ(ierr);
3659           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3660         }
3661       }
3662     }
3663     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3664   }
3665   PetscFunctionReturn(0);
3666 }
3667 #endif
3668 
3669 #if defined(PETSC_HAVE_CTETGEN)
3670 #include "ctetgen.h"
3671 
3672 #undef __FUNCT__
3673 #define __FUNCT__ "DMPlexGenerate_CTetgen"
3674 PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
3675 {
3676   MPI_Comm       comm;
3677   const PetscInt dim  = 3;
3678   PLC           *in, *out;
3679   PetscInt       verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
3680   PetscMPIInt    rank;
3681   PetscErrorCode ierr;
3682 
3683   PetscFunctionBegin;
3684   ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
3685   ierr = PetscOptionsGetInt(((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3686   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3687   ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
3688   ierr = PLCCreate(&in);CHKERRQ(ierr);
3689   ierr = PLCCreate(&out);CHKERRQ(ierr);
3690 
3691   in->numberofpoints = vEnd - vStart;
3692   if (in->numberofpoints > 0) {
3693     PetscSection coordSection;
3694     Vec          coordinates;
3695     PetscScalar *array;
3696 
3697     ierr = PetscMalloc(in->numberofpoints*dim * sizeof(PetscReal), &in->pointlist);CHKERRQ(ierr);
3698     ierr = PetscMalloc(in->numberofpoints     * sizeof(int),       &in->pointmarkerlist);CHKERRQ(ierr);
3699     ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
3700     ierr = DMPlexGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
3701     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3702     for (v = vStart; v < vEnd; ++v) {
3703       const PetscInt idx = v - vStart;
3704       PetscInt       off, d, m;
3705 
3706       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3707       for (d = 0; d < dim; ++d) {
3708         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3709       }
3710       ierr = DMPlexGetLabelValue(boundary, "marker", v, &m);CHKERRQ(ierr);
3711 
3712       in->pointmarkerlist[idx] = (int) m;
3713     }
3714     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3715   }
3716   ierr  = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
3717 
3718   in->numberoffacets = fEnd - fStart;
3719   if (in->numberoffacets > 0) {
3720     ierr = PetscMalloc(in->numberoffacets * sizeof(facet), &in->facetlist);CHKERRQ(ierr);
3721     ierr = PetscMalloc(in->numberoffacets * sizeof(int),   &in->facetmarkerlist);CHKERRQ(ierr);
3722     for (f = fStart; f < fEnd; ++f) {
3723       const PetscInt idx     = f - fStart;
3724       PetscInt      *points = NULL, numPoints, p, numVertices = 0, v, m;
3725       polygon       *poly;
3726 
3727       in->facetlist[idx].numberofpolygons = 1;
3728 
3729       ierr = PetscMalloc(in->facetlist[idx].numberofpolygons * sizeof(polygon), &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
3730 
3731       in->facetlist[idx].numberofholes    = 0;
3732       in->facetlist[idx].holelist         = NULL;
3733 
3734       ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3735       for (p = 0; p < numPoints*2; p += 2) {
3736         const PetscInt point = points[p];
3737         if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
3738       }
3739 
3740       poly                   = in->facetlist[idx].polygonlist;
3741       poly->numberofvertices = numVertices;
3742       ierr                   = PetscMalloc(poly->numberofvertices * sizeof(int), &poly->vertexlist);CHKERRQ(ierr);
3743       for (v = 0; v < numVertices; ++v) {
3744         const PetscInt vIdx = points[v] - vStart;
3745         poly->vertexlist[v] = vIdx;
3746       }
3747       ierr                     = DMPlexGetLabelValue(boundary, "marker", f, &m);CHKERRQ(ierr);
3748       in->facetmarkerlist[idx] = (int) m;
3749       ierr                     = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
3750     }
3751   }
3752   if (!rank) {
3753     TetGenOpts t;
3754 
3755     ierr        = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3756     t.in        = boundary; /* Should go away */
3757     t.plc       = 1;
3758     t.quality   = 1;
3759     t.edgesout  = 1;
3760     t.zeroindex = 1;
3761     t.quiet     = 1;
3762     t.verbose   = verbose;
3763     ierr        = TetGenCheckOpts(&t);CHKERRQ(ierr);
3764     ierr        = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3765   }
3766   {
3767     const PetscInt numCorners  = 4;
3768     const PetscInt numCells    = out->numberoftetrahedra;
3769     const PetscInt numVertices = out->numberofpoints;
3770     const double   *meshCoords = out->pointlist;
3771     int            *cells      = out->tetrahedronlist;
3772 
3773     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3774     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
3775     /* Set labels */
3776     for (v = 0; v < numVertices; ++v) {
3777       if (out->pointmarkerlist[v]) {
3778         ierr = DMPlexSetLabelValue(*dm, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3779       }
3780     }
3781     if (interpolate) {
3782       PetscInt e;
3783 
3784       for (e = 0; e < out->numberofedges; e++) {
3785         if (out->edgemarkerlist[e]) {
3786           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3787           const PetscInt *edges;
3788           PetscInt        numEdges;
3789 
3790           ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3791           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3792           ierr = DMPlexSetLabelValue(*dm, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3793           ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3794         }
3795       }
3796       for (f = 0; f < out->numberoftrifaces; f++) {
3797         if (out->trifacemarkerlist[f]) {
3798           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3799           const PetscInt *faces;
3800           PetscInt        numFaces;
3801 
3802           ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3803           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3804           ierr = DMPlexSetLabelValue(*dm, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3805           ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3806         }
3807       }
3808     }
3809     ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
3810   }
3811 
3812   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3813   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3814   PetscFunctionReturn(0);
3815 }
3816 
3817 #undef __FUNCT__
3818 #define __FUNCT__ "DMPlexRefine_CTetgen"
3819 PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
3820 {
3821   MPI_Comm       comm;
3822   const PetscInt dim  = 3;
3823   PLC           *in, *out;
3824   PetscInt       verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
3825   PetscMPIInt    rank;
3826   PetscErrorCode ierr;
3827 
3828   PetscFunctionBegin;
3829   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3830   ierr = PetscOptionsGetInt(((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
3831   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
3832   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3833   ierr = MPI_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
3834   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3835   ierr = PLCCreate(&in);CHKERRQ(ierr);
3836   ierr = PLCCreate(&out);CHKERRQ(ierr);
3837 
3838   in->numberofpoints = vEnd - vStart;
3839   if (in->numberofpoints > 0) {
3840     PetscSection coordSection;
3841     Vec          coordinates;
3842     PetscScalar *array;
3843 
3844     ierr = PetscMalloc(in->numberofpoints*dim * sizeof(PetscReal), &in->pointlist);CHKERRQ(ierr);
3845     ierr = PetscMalloc(in->numberofpoints     * sizeof(int),       &in->pointmarkerlist);CHKERRQ(ierr);
3846     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3847     ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3848     ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
3849     for (v = vStart; v < vEnd; ++v) {
3850       const PetscInt idx = v - vStart;
3851       PetscInt       off, d, m;
3852 
3853       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
3854       for (d = 0; d < dim; ++d) {
3855         in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
3856       }
3857       ierr = DMPlexGetLabelValue(dm, "marker", v, &m);CHKERRQ(ierr);
3858 
3859       in->pointmarkerlist[idx] = (int) m;
3860     }
3861     ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
3862   }
3863   ierr  = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
3864 
3865   in->numberofcorners       = 4;
3866   in->numberoftetrahedra    = cEnd - cStart;
3867   in->tetrahedronvolumelist = maxVolumes;
3868   if (in->numberoftetrahedra > 0) {
3869     ierr = PetscMalloc(in->numberoftetrahedra*in->numberofcorners * sizeof(int), &in->tetrahedronlist);CHKERRQ(ierr);
3870     for (c = cStart; c < cEnd; ++c) {
3871       const PetscInt idx      = c - cStart;
3872       PetscInt      *closure = NULL;
3873       PetscInt       closureSize;
3874 
3875       ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3876       if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
3877       for (v = 0; v < 4; ++v) {
3878         in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
3879       }
3880       ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3881     }
3882   }
3883   if (!rank) {
3884     TetGenOpts t;
3885 
3886     ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
3887 
3888     t.in        = dm; /* Should go away */
3889     t.refine    = 1;
3890     t.varvolume = 1;
3891     t.quality   = 1;
3892     t.edgesout  = 1;
3893     t.zeroindex = 1;
3894     t.quiet     = 1;
3895     t.verbose   = verbose; /* Change this */
3896 
3897     ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
3898     ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
3899   }
3900   {
3901     const PetscInt numCorners  = 4;
3902     const PetscInt numCells    = out->numberoftetrahedra;
3903     const PetscInt numVertices = out->numberofpoints;
3904     const double   *meshCoords = out->pointlist;
3905     int            *cells      = out->tetrahedronlist;
3906     PetscBool      interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
3907 
3908     ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
3909     ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
3910     /* Set labels */
3911     for (v = 0; v < numVertices; ++v) {
3912       if (out->pointmarkerlist[v]) {
3913         ierr = DMPlexSetLabelValue(*dmRefined, "marker", v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);
3914       }
3915     }
3916     if (interpolate) {
3917       PetscInt e, f;
3918 
3919       for (e = 0; e < out->numberofedges; e++) {
3920         if (out->edgemarkerlist[e]) {
3921           const PetscInt  vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
3922           const PetscInt *edges;
3923           PetscInt        numEdges;
3924 
3925           ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3926           if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
3927           ierr = DMPlexSetLabelValue(*dmRefined, "marker", edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);
3928           ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
3929         }
3930       }
3931       for (f = 0; f < out->numberoftrifaces; f++) {
3932         if (out->trifacemarkerlist[f]) {
3933           const PetscInt  vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
3934           const PetscInt *faces;
3935           PetscInt        numFaces;
3936 
3937           ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3938           if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
3939           ierr = DMPlexSetLabelValue(*dmRefined, "marker", faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);
3940           ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
3941         }
3942       }
3943     }
3944     ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
3945   }
3946   ierr = PLCDestroy(&in);CHKERRQ(ierr);
3947   ierr = PLCDestroy(&out);CHKERRQ(ierr);
3948   PetscFunctionReturn(0);
3949 }
3950 #endif
3951 
3952 #undef __FUNCT__
3953 #define __FUNCT__ "DMPlexGenerate"
3954 /*@C
3955   DMPlexGenerate - Generates a mesh.
3956 
3957   Not Collective
3958 
3959   Input Parameters:
3960 + boundary - The DMPlex boundary object
3961 . name - The mesh generation package name
3962 - interpolate - Flag to create intermediate mesh elements
3963 
3964   Output Parameter:
3965 . mesh - The DMPlex object
3966 
3967   Level: intermediate
3968 
3969 .keywords: mesh, elements
3970 .seealso: DMPlexCreate(), DMRefine()
3971 @*/
3972 PetscErrorCode DMPlexGenerate(DM boundary, const char name[], PetscBool interpolate, DM *mesh)
3973 {
3974   PetscInt       dim;
3975   char           genname[1024];
3976   PetscBool      isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
3977   PetscErrorCode ierr;
3978 
3979   PetscFunctionBegin;
3980   PetscValidHeaderSpecific(boundary, DM_CLASSID, 1);
3981   PetscValidLogicalCollectiveBool(boundary, interpolate, 2);
3982   ierr = DMPlexGetDimension(boundary, &dim);CHKERRQ(ierr);
3983   ierr = PetscOptionsGetString(((PetscObject) boundary)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
3984   if (flg) name = genname;
3985   if (name) {
3986     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
3987     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
3988     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
3989   }
3990   switch (dim) {
3991   case 1:
3992     if (!name || isTriangle) {
3993 #if defined(PETSC_HAVE_TRIANGLE)
3994       ierr = DMPlexGenerate_Triangle(boundary, interpolate, mesh);CHKERRQ(ierr);
3995 #else
3996       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation needs external package support.\nPlease reconfigure with --download-triangle.");
3997 #endif
3998     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
3999     break;
4000   case 2:
4001     if (!name || isCTetgen) {
4002 #if defined(PETSC_HAVE_CTETGEN)
4003       ierr = DMPlexGenerate_CTetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4004 #else
4005       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
4006 #endif
4007     } else if (isTetgen) {
4008 #if defined(PETSC_HAVE_TETGEN)
4009       ierr = DMPlexGenerate_Tetgen(boundary, interpolate, mesh);CHKERRQ(ierr);
4010 #else
4011       SETERRQ(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
4012 #endif
4013     } else SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
4014     break;
4015   default:
4016     SETERRQ1(PetscObjectComm((PetscObject)boundary), PETSC_ERR_SUP, "Mesh generation for a dimension %d boundary is not supported.", dim);
4017   }
4018   PetscFunctionReturn(0);
4019 }
4020 
4021 typedef PetscInt CellRefiner;
4022 
4023 #undef __FUNCT__
4024 #define __FUNCT__ "GetDepthStart_Private"
4025 PETSC_STATIC_INLINE PetscErrorCode GetDepthStart_Private(PetscInt depth, PetscInt depthSize[], PetscInt *cStart, PetscInt *fStart, PetscInt *eStart, PetscInt *vStart)
4026 {
4027   PetscFunctionBegin;
4028   if (cStart) *cStart = 0;
4029   if (vStart) *vStart = depthSize[depth];
4030   if (fStart) *fStart = depthSize[depth] + depthSize[0];
4031   if (eStart) *eStart = depthSize[depth] + depthSize[0] + depthSize[depth-1];
4032   PetscFunctionReturn(0);
4033 }
4034 
4035 #undef __FUNCT__
4036 #define __FUNCT__ "GetDepthEnd_Private"
4037 PETSC_STATIC_INLINE PetscErrorCode GetDepthEnd_Private(PetscInt depth, PetscInt depthSize[], PetscInt *cEnd, PetscInt *fEnd, PetscInt *eEnd, PetscInt *vEnd)
4038 {
4039   PetscFunctionBegin;
4040   if (cEnd) *cEnd = depthSize[depth];
4041   if (vEnd) *vEnd = depthSize[depth] + depthSize[0];
4042   if (fEnd) *fEnd = depthSize[depth] + depthSize[0] + depthSize[depth-1];
4043   if (eEnd) *eEnd = depthSize[depth] + depthSize[0] + depthSize[depth-1] + depthSize[1];
4044   PetscFunctionReturn(0);
4045 }
4046 
4047 #undef __FUNCT__
4048 #define __FUNCT__ "CellRefinerGetSizes"
4049 PetscErrorCode CellRefinerGetSizes(CellRefiner refiner, DM dm, PetscInt depthSize[])
4050 {
4051   PetscInt       cStart, cEnd, cMax, vStart, vEnd, vMax, fStart, fEnd, fMax, eStart, eEnd, eMax;
4052   PetscErrorCode ierr;
4053 
4054   PetscFunctionBegin;
4055   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4056   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
4057   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4058   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
4059   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
4060   switch (refiner) {
4061   case 1:
4062     /* Simplicial 2D */
4063     depthSize[0] = vEnd - vStart + fEnd - fStart;         /* Add a vertex on every face */
4064     depthSize[1] = 2*(fEnd - fStart) + 3*(cEnd - cStart); /* Every face is split into 2 faces and 3 faces are added for each cell */
4065     depthSize[2] = 4*(cEnd - cStart);                     /* Every cell split into 4 cells */
4066     break;
4067   case 3:
4068     /* Hybrid 2D */
4069     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
4070     cMax = PetscMin(cEnd, cMax);
4071     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
4072     fMax         = PetscMin(fEnd, fMax);
4073     depthSize[0] = vEnd - vStart + fMax - fStart;                                         /* Add a vertex on every face, but not hybrid faces */
4074     depthSize[1] = 2*(fMax - fStart) + 3*(cMax - cStart) + (fEnd - fMax) + (cEnd - cMax); /* Every interior face is split into 2 faces, 3 faces are added for each interior cell, and one in each hybrid cell */
4075     depthSize[2] = 4*(cMax - cStart) + 2*(cEnd - cMax);                                   /* Interior cells split into 4 cells, Hybrid cells split into 2 cells */
4076     break;
4077   case 2:
4078     /* Hex 2D */
4079     depthSize[0] = vEnd - vStart + cEnd - cStart + fEnd - fStart; /* Add a vertex on every face and cell */
4080     depthSize[1] = 2*(fEnd - fStart) + 4*(cEnd - cStart);         /* Every face is split into 2 faces and 4 faces are added for each cell */
4081     depthSize[2] = 4*(cEnd - cStart);                             /* Every cell split into 4 cells */
4082     break;
4083   default:
4084     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
4085   }
4086   PetscFunctionReturn(0);
4087 }
4088 
4089 #undef __FUNCT__
4090 #define __FUNCT__ "CellRefinerSetConeSizes"
4091 PetscErrorCode CellRefinerSetConeSizes(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
4092 {
4093   PetscInt       depth, cStart, cStartNew, cEnd, cMax, c, vStart, vStartNew, vEnd, vMax, v, fStart, fStartNew, fEnd, fMax, f, eStart, eStartNew, eEnd, eMax, r;
4094   PetscErrorCode ierr;
4095 
4096   PetscFunctionBegin;
4097   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4098   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4099   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
4100   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4101   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
4102   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
4103   ierr = GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);CHKERRQ(ierr);
4104   switch (refiner) {
4105   case 1:
4106     /* Simplicial 2D */
4107     /* All cells have 3 faces */
4108     for (c = cStart; c < cEnd; ++c) {
4109       for (r = 0; r < 4; ++r) {
4110         const PetscInt newp = (c - cStart)*4 + r;
4111 
4112         ierr = DMPlexSetConeSize(rdm, newp, 3);CHKERRQ(ierr);
4113       }
4114     }
4115     /* Split faces have 2 vertices and the same cells as the parent */
4116     for (f = fStart; f < fEnd; ++f) {
4117       for (r = 0; r < 2; ++r) {
4118         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
4119         PetscInt       size;
4120 
4121         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4122         ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4123         ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4124       }
4125     }
4126     /* Interior faces have 2 vertices and 2 cells */
4127     for (c = cStart; c < cEnd; ++c) {
4128       for (r = 0; r < 3; ++r) {
4129         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*3 + r;
4130 
4131         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4132         ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4133       }
4134     }
4135     /* Old vertices have identical supports */
4136     for (v = vStart; v < vEnd; ++v) {
4137       const PetscInt newp = vStartNew + (v - vStart);
4138       PetscInt       size;
4139 
4140       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4141       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4142     }
4143     /* Face vertices have 2 + cells*2 supports */
4144     for (f = fStart; f < fEnd; ++f) {
4145       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
4146       PetscInt       size;
4147 
4148       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4149       ierr = DMPlexSetSupportSize(rdm, newp, 2 + size*2);CHKERRQ(ierr);
4150     }
4151     break;
4152   case 2:
4153     /* Hex 2D */
4154     /* All cells have 4 faces */
4155     for (c = cStart; c < cEnd; ++c) {
4156       for (r = 0; r < 4; ++r) {
4157         const PetscInt newp = (c - cStart)*4 + r;
4158 
4159         ierr = DMPlexSetConeSize(rdm, newp, 4);CHKERRQ(ierr);
4160       }
4161     }
4162     /* Split faces have 2 vertices and the same cells as the parent */
4163     for (f = fStart; f < fEnd; ++f) {
4164       for (r = 0; r < 2; ++r) {
4165         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
4166         PetscInt       size;
4167 
4168         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4169         ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4170         ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4171       }
4172     }
4173     /* Interior faces have 2 vertices and 2 cells */
4174     for (c = cStart; c < cEnd; ++c) {
4175       for (r = 0; r < 4; ++r) {
4176         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
4177 
4178         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4179         ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4180       }
4181     }
4182     /* Old vertices have identical supports */
4183     for (v = vStart; v < vEnd; ++v) {
4184       const PetscInt newp = vStartNew + (v - vStart);
4185       PetscInt       size;
4186 
4187       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4188       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4189     }
4190     /* Face vertices have 2 + cells supports */
4191     for (f = fStart; f < fEnd; ++f) {
4192       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
4193       PetscInt       size;
4194 
4195       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4196       ierr = DMPlexSetSupportSize(rdm, newp, 2 + size);CHKERRQ(ierr);
4197     }
4198     /* Cell vertices have 4 supports */
4199     for (c = cStart; c < cEnd; ++c) {
4200       const PetscInt newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
4201 
4202       ierr = DMPlexSetSupportSize(rdm, newp, 4);CHKERRQ(ierr);
4203     }
4204     break;
4205   case 3:
4206     /* Hybrid 2D */
4207     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
4208     cMax = PetscMin(cEnd, cMax);
4209     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
4210     fMax = PetscMin(fEnd, fMax);
4211     ierr = DMPlexSetHybridBounds(rdm, cStartNew + (cMax - cStart)*4, fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
4212     /* Interior cells have 3 faces */
4213     for (c = cStart; c < cMax; ++c) {
4214       for (r = 0; r < 4; ++r) {
4215         const PetscInt newp = cStartNew + (c - cStart)*4 + r;
4216 
4217         ierr = DMPlexSetConeSize(rdm, newp, 3);CHKERRQ(ierr);
4218       }
4219     }
4220     /* Hybrid cells have 4 faces */
4221     for (c = cMax; c < cEnd; ++c) {
4222       for (r = 0; r < 2; ++r) {
4223         const PetscInt newp = cStartNew + (cMax - cStart)*4 + (c - cMax)*2 + r;
4224 
4225         ierr = DMPlexSetConeSize(rdm, newp, 4);CHKERRQ(ierr);
4226       }
4227     }
4228     /* Interior split faces have 2 vertices and the same cells as the parent */
4229     for (f = fStart; f < fMax; ++f) {
4230       for (r = 0; r < 2; ++r) {
4231         const PetscInt newp = fStartNew + (f - fStart)*2 + r;
4232         PetscInt       size;
4233 
4234         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4235         ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4236         ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4237       }
4238     }
4239     /* Interior cell faces have 2 vertices and 2 cells */
4240     for (c = cStart; c < cMax; ++c) {
4241       for (r = 0; r < 3; ++r) {
4242         const PetscInt newp = fStartNew + (fMax - fStart)*2 + (c - cStart)*3 + r;
4243 
4244         ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4245         ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4246       }
4247     }
4248     /* Hybrid faces have 2 vertices and the same cells */
4249     for (f = fMax; f < fEnd; ++f) {
4250       const PetscInt newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (f - fMax);
4251       PetscInt       size;
4252 
4253       ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4254       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4255       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4256     }
4257     /* Hybrid cell faces have 2 vertices and 2 cells */
4258     for (c = cMax; c < cEnd; ++c) {
4259       const PetscInt newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (c - cMax);
4260 
4261       ierr = DMPlexSetConeSize(rdm, newp, 2);CHKERRQ(ierr);
4262       ierr = DMPlexSetSupportSize(rdm, newp, 2);CHKERRQ(ierr);
4263     }
4264     /* Old vertices have identical supports */
4265     for (v = vStart; v < vEnd; ++v) {
4266       const PetscInt newp = vStartNew + (v - vStart);
4267       PetscInt       size;
4268 
4269       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4270       ierr = DMPlexSetSupportSize(rdm, newp, size);CHKERRQ(ierr);
4271     }
4272     /* Face vertices have 2 + (2 interior, 1 hybrid) supports */
4273     for (f = fStart; f < fMax; ++f) {
4274       const PetscInt newp = vStartNew + (vEnd - vStart) + (f - fStart);
4275       const PetscInt *support;
4276       PetscInt       size, newSize = 2, s;
4277 
4278       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4279       ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4280       for (s = 0; s < size; ++s) {
4281         if (support[s] >= cMax) newSize += 1;
4282         else newSize += 2;
4283       }
4284       ierr = DMPlexSetSupportSize(rdm, newp, newSize);CHKERRQ(ierr);
4285     }
4286     break;
4287   default:
4288     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
4289   }
4290   PetscFunctionReturn(0);
4291 }
4292 
4293 #undef __FUNCT__
4294 #define __FUNCT__ "CellRefinerSetCones"
4295 PetscErrorCode CellRefinerSetCones(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
4296 {
4297   PetscInt       depth, cStart, cEnd, cMax, cStartNew, cEndNew, c, vStart, vEnd, vMax, vStartNew, vEndNew, v, fStart, fEnd, fMax, fStartNew, fEndNew, f, eStart, eEnd, eMax, eStartNew, eEndNew, r, p;
4298   PetscInt       maxSupportSize, *supportRef;
4299   PetscErrorCode ierr;
4300 
4301   PetscFunctionBegin;
4302   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
4303   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
4304   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
4305   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
4306   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
4307   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
4308   ierr = GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);CHKERRQ(ierr);
4309   ierr = GetDepthEnd_Private(depth, depthSize, &cEndNew, &fEndNew, &eEndNew, &vEndNew);CHKERRQ(ierr);
4310   switch (refiner) {
4311   case 1:
4312     /* Simplicial 2D */
4313     /*
4314      2
4315      |\
4316      | \
4317      |  \
4318      |   \
4319      | C  \
4320      |     \
4321      |      \
4322      2---1---1
4323      |\  D  / \
4324      | 2   0   \
4325      |A \ /  B  \
4326      0---0-------1
4327      */
4328     /* All cells have 3 faces */
4329     for (c = cStart; c < cEnd; ++c) {
4330       const PetscInt  newp = cStartNew + (c - cStart)*4;
4331       const PetscInt *cone, *ornt;
4332       PetscInt        coneNew[3], orntNew[3];
4333 
4334       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4335       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4336       /* A triangle */
4337       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4338       orntNew[0] = ornt[0];
4339       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 2;
4340       orntNew[1] = -2;
4341       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
4342       orntNew[2] = ornt[2];
4343       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4344       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4345 #if 1
4346       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4347       for (p = 0; p < 3; ++p) {
4348         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4349       }
4350 #endif
4351       /* B triangle */
4352       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4353       orntNew[0] = ornt[0];
4354       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4355       orntNew[1] = ornt[1];
4356       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 0;
4357       orntNew[2] = -2;
4358       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4359       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4360 #if 1
4361       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4362       for (p = 0; p < 3; ++p) {
4363         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4364       }
4365 #endif
4366       /* C triangle */
4367       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 1;
4368       orntNew[0] = -2;
4369       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4370       orntNew[1] = ornt[1];
4371       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
4372       orntNew[2] = ornt[2];
4373       ierr       = DMPlexSetCone(rdm, newp+2, coneNew);CHKERRQ(ierr);
4374       ierr       = DMPlexSetConeOrientation(rdm, newp+2, orntNew);CHKERRQ(ierr);
4375 #if 1
4376       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
4377       for (p = 0; p < 3; ++p) {
4378         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4379       }
4380 #endif
4381       /* D triangle */
4382       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 0;
4383       orntNew[0] = 0;
4384       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 1;
4385       orntNew[1] = 0;
4386       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*3 + 2;
4387       orntNew[2] = 0;
4388       ierr       = DMPlexSetCone(rdm, newp+3, coneNew);CHKERRQ(ierr);
4389       ierr       = DMPlexSetConeOrientation(rdm, newp+3, orntNew);CHKERRQ(ierr);
4390 #if 1
4391       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
4392       for (p = 0; p < 3; ++p) {
4393         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4394       }
4395 #endif
4396     }
4397     /* Split faces have 2 vertices and the same cells as the parent */
4398     ierr = DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);CHKERRQ(ierr);
4399     ierr = PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);CHKERRQ(ierr);
4400     for (f = fStart; f < fEnd; ++f) {
4401       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);
4402 
4403       for (r = 0; r < 2; ++r) {
4404         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
4405         const PetscInt *cone, *support;
4406         PetscInt        coneNew[2], coneSize, c, supportSize, s;
4407 
4408         ierr             = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4409         coneNew[0]       = vStartNew + (cone[0] - vStart);
4410         coneNew[1]       = vStartNew + (cone[1] - vStart);
4411         coneNew[(r+1)%2] = newv;
4412         ierr             = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4413 #if 1
4414         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4415         for (p = 0; p < 2; ++p) {
4416           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4417         }
4418 #endif
4419         ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
4420         ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4421         for (s = 0; s < supportSize; ++s) {
4422           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
4423           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4424           for (c = 0; c < coneSize; ++c) {
4425             if (cone[c] == f) break;
4426           }
4427           supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%3;
4428         }
4429         ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4430 #if 1
4431         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4432         for (p = 0; p < supportSize; ++p) {
4433           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
4434         }
4435 #endif
4436       }
4437     }
4438     /* Interior faces have 2 vertices and 2 cells */
4439     for (c = cStart; c < cEnd; ++c) {
4440       const PetscInt *cone;
4441 
4442       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4443       for (r = 0; r < 3; ++r) {
4444         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*3 + r;
4445         PetscInt       coneNew[2];
4446         PetscInt       supportNew[2];
4447 
4448         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r]       - fStart);
4449         coneNew[1] = vStartNew + (vEnd - vStart) + (cone[(r+1)%3] - fStart);
4450         ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4451 #if 1
4452         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4453         for (p = 0; p < 2; ++p) {
4454           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4455         }
4456 #endif
4457         supportNew[0] = (c - cStart)*4 + (r+1)%3;
4458         supportNew[1] = (c - cStart)*4 + 3;
4459         ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4460 #if 1
4461         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4462         for (p = 0; p < 2; ++p) {
4463           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4464         }
4465 #endif
4466       }
4467     }
4468     /* Old vertices have identical supports */
4469     for (v = vStart; v < vEnd; ++v) {
4470       const PetscInt  newp = vStartNew + (v - vStart);
4471       const PetscInt *support, *cone;
4472       PetscInt        size, s;
4473 
4474       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4475       ierr = DMPlexGetSupport(dm, v, &support);CHKERRQ(ierr);
4476       for (s = 0; s < size; ++s) {
4477         PetscInt r = 0;
4478 
4479         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4480         if (cone[1] == v) r = 1;
4481         supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
4482       }
4483       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4484 #if 1
4485       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4486       for (p = 0; p < size; ++p) {
4487         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4488       }
4489 #endif
4490     }
4491     /* Face vertices have 2 + cells*2 supports */
4492     for (f = fStart; f < fEnd; ++f) {
4493       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
4494       const PetscInt *cone, *support;
4495       PetscInt        size, s;
4496 
4497       ierr          = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4498       ierr          = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4499       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
4500       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
4501       for (s = 0; s < size; ++s) {
4502         PetscInt r = 0;
4503 
4504         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4505         if      (cone[1] == f) r = 1;
4506         else if (cone[2] == f) r = 2;
4507         supportRef[2+s*2+0] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*3 + (r+2)%3;
4508         supportRef[2+s*2+1] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*3 + r;
4509       }
4510       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4511 #if 1
4512       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4513       for (p = 0; p < 2+size*2; ++p) {
4514         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4515       }
4516 #endif
4517     }
4518     ierr = PetscFree(supportRef);CHKERRQ(ierr);
4519     break;
4520   case 2:
4521     /* Hex 2D */
4522     /*
4523      3---------2---------2
4524      |         |         |
4525      |    D    2    C    |
4526      |         |         |
4527      3----3----0----1----1
4528      |         |         |
4529      |    A    0    B    |
4530      |         |         |
4531      0---------0---------1
4532      */
4533     /* All cells have 4 faces */
4534     for (c = cStart; c < cEnd; ++c) {
4535       const PetscInt  newp = (c - cStart)*4;
4536       const PetscInt *cone, *ornt;
4537       PetscInt        coneNew[4], orntNew[4];
4538 
4539       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4540       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4541       /* A quad */
4542       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4543       orntNew[0] = ornt[0];
4544       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 0;
4545       orntNew[1] = 0;
4546       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 3;
4547       orntNew[2] = -2;
4548       coneNew[3] = fStartNew + (cone[3] - fStart)*2 + (ornt[3] < 0 ? 0 : 1);
4549       orntNew[3] = ornt[3];
4550       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4551       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4552 #if 1
4553       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4554       for (p = 0; p < 4; ++p) {
4555         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4556       }
4557 #endif
4558       /* B quad */
4559       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4560       orntNew[0] = ornt[0];
4561       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4562       orntNew[1] = ornt[1];
4563       coneNew[2] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 1;
4564       orntNew[2] = 0;
4565       coneNew[3] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 0;
4566       orntNew[3] = -2;
4567       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4568       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4569 #if 1
4570       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4571       for (p = 0; p < 4; ++p) {
4572         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4573       }
4574 #endif
4575       /* C quad */
4576       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 1;
4577       orntNew[0] = -2;
4578       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4579       orntNew[1] = ornt[1];
4580       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
4581       orntNew[2] = ornt[2];
4582       coneNew[3] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 2;
4583       orntNew[3] = 0;
4584       ierr       = DMPlexSetCone(rdm, newp+2, coneNew);CHKERRQ(ierr);
4585       ierr       = DMPlexSetConeOrientation(rdm, newp+2, orntNew);CHKERRQ(ierr);
4586 #if 1
4587       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
4588       for (p = 0; p < 4; ++p) {
4589         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4590       }
4591 #endif
4592       /* D quad */
4593       coneNew[0] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 3;
4594       orntNew[0] = 0;
4595       coneNew[1] = fStartNew + (fEnd    - fStart)*2 + (c - cStart)*4 + 2;
4596       orntNew[1] = -2;
4597       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
4598       orntNew[2] = ornt[2];
4599       coneNew[3] = fStartNew + (cone[3] - fStart)*2 + (ornt[3] < 0 ? 1 : 0);
4600       orntNew[3] = ornt[3];
4601       ierr       = DMPlexSetCone(rdm, newp+3, coneNew);CHKERRQ(ierr);
4602       ierr       = DMPlexSetConeOrientation(rdm, newp+3, orntNew);CHKERRQ(ierr);
4603 #if 1
4604       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
4605       for (p = 0; p < 4; ++p) {
4606         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4607       }
4608 #endif
4609     }
4610     /* Split faces have 2 vertices and the same cells as the parent */
4611     ierr = DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);CHKERRQ(ierr);
4612     ierr = PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);CHKERRQ(ierr);
4613     for (f = fStart; f < fEnd; ++f) {
4614       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);
4615 
4616       for (r = 0; r < 2; ++r) {
4617         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
4618         const PetscInt *cone, *support;
4619         PetscInt        coneNew[2], coneSize, c, supportSize, s;
4620 
4621         ierr             = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4622         coneNew[0]       = vStartNew + (cone[0] - vStart);
4623         coneNew[1]       = vStartNew + (cone[1] - vStart);
4624         coneNew[(r+1)%2] = newv;
4625         ierr             = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4626 #if 1
4627         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4628         for (p = 0; p < 2; ++p) {
4629           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4630         }
4631 #endif
4632         ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
4633         ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4634         for (s = 0; s < supportSize; ++s) {
4635           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
4636           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4637           for (c = 0; c < coneSize; ++c) {
4638             if (cone[c] == f) break;
4639           }
4640           supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%4;
4641         }
4642         ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4643 #if 1
4644         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4645         for (p = 0; p < supportSize; ++p) {
4646           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
4647         }
4648 #endif
4649       }
4650     }
4651     /* Interior faces have 2 vertices and 2 cells */
4652     for (c = cStart; c < cEnd; ++c) {
4653       const PetscInt *cone;
4654       PetscInt        coneNew[2], supportNew[2];
4655 
4656       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4657       for (r = 0; r < 4; ++r) {
4658         const PetscInt newp = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
4659 
4660         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r] - fStart);
4661         coneNew[1] = vStartNew + (vEnd - vStart) + (fEnd    - fStart) + (c - cStart);
4662         ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4663 #if 1
4664         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4665         for (p = 0; p < 2; ++p) {
4666           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4667         }
4668 #endif
4669         supportNew[0] = (c - cStart)*4 + r;
4670         supportNew[1] = (c - cStart)*4 + (r+1)%4;
4671         ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4672 #if 1
4673         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4674         for (p = 0; p < 2; ++p) {
4675           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4676         }
4677 #endif
4678       }
4679     }
4680     /* Old vertices have identical supports */
4681     for (v = vStart; v < vEnd; ++v) {
4682       const PetscInt  newp = vStartNew + (v - vStart);
4683       const PetscInt *support, *cone;
4684       PetscInt        size, s;
4685 
4686       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
4687       ierr = DMPlexGetSupport(dm, v, &support);CHKERRQ(ierr);
4688       for (s = 0; s < size; ++s) {
4689         PetscInt r = 0;
4690 
4691         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4692         if (cone[1] == v) r = 1;
4693         supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
4694       }
4695       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4696 #if 1
4697       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4698       for (p = 0; p < size; ++p) {
4699         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4700       }
4701 #endif
4702     }
4703     /* Face vertices have 2 + cells supports */
4704     for (f = fStart; f < fEnd; ++f) {
4705       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
4706       const PetscInt *cone, *support;
4707       PetscInt        size, s;
4708 
4709       ierr          = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4710       ierr          = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4711       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
4712       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
4713       for (s = 0; s < size; ++s) {
4714         PetscInt r = 0;
4715 
4716         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4717         if      (cone[1] == f) r = 1;
4718         else if (cone[2] == f) r = 2;
4719         else if (cone[3] == f) r = 3;
4720         supportRef[2+s] = fStartNew + (fEnd - fStart)*2 + (support[s] - cStart)*4 + r;
4721       }
4722       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4723 #if 1
4724       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
4725       for (p = 0; p < 2+size; ++p) {
4726         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
4727       }
4728 #endif
4729     }
4730     /* Cell vertices have 4 supports */
4731     for (c = cStart; c < cEnd; ++c) {
4732       const PetscInt newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
4733       PetscInt       supportNew[4];
4734 
4735       for (r = 0; r < 4; ++r) {
4736         supportNew[r] = fStartNew + (fEnd - fStart)*2 + (c - cStart)*4 + r;
4737       }
4738       ierr = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4739     }
4740     break;
4741   case 3:
4742     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
4743     cMax = PetscMin(cEnd, cMax);
4744     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
4745     fMax = PetscMin(fEnd, fMax);
4746     /* Interior cells have 3 faces */
4747     for (c = cStart; c < cMax; ++c) {
4748       const PetscInt  newp = cStartNew + (c - cStart)*4;
4749       const PetscInt *cone, *ornt;
4750       PetscInt        coneNew[3], orntNew[3];
4751 
4752       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4753       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4754       /* A triangle */
4755       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4756       orntNew[0] = ornt[0];
4757       coneNew[1] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 2;
4758       orntNew[1] = -2;
4759       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 0 : 1);
4760       orntNew[2] = ornt[2];
4761       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4762       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4763 #if 1
4764       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4765       for (p = 0; p < 3; ++p) {
4766         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4767       }
4768 #endif
4769       /* B triangle */
4770       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4771       orntNew[0] = ornt[0];
4772       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4773       orntNew[1] = ornt[1];
4774       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 0;
4775       orntNew[2] = -2;
4776       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4777       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4778 #if 1
4779       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4780       for (p = 0; p < 3; ++p) {
4781         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4782       }
4783 #endif
4784       /* C triangle */
4785       coneNew[0] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 1;
4786       orntNew[0] = -2;
4787       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4788       orntNew[1] = ornt[1];
4789       coneNew[2] = fStartNew + (cone[2] - fStart)*2 + (ornt[2] < 0 ? 1 : 0);
4790       orntNew[2] = ornt[2];
4791       ierr       = DMPlexSetCone(rdm, newp+2, coneNew);CHKERRQ(ierr);
4792       ierr       = DMPlexSetConeOrientation(rdm, newp+2, orntNew);CHKERRQ(ierr);
4793 #if 1
4794       if ((newp+2 < cStartNew) || (newp+2 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+2, cStartNew, cEndNew);
4795       for (p = 0; p < 3; ++p) {
4796         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4797       }
4798 #endif
4799       /* D triangle */
4800       coneNew[0] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 0;
4801       orntNew[0] = 0;
4802       coneNew[1] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 1;
4803       orntNew[1] = 0;
4804       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (c - cStart)*3 + 2;
4805       orntNew[2] = 0;
4806       ierr       = DMPlexSetCone(rdm, newp+3, coneNew);CHKERRQ(ierr);
4807       ierr       = DMPlexSetConeOrientation(rdm, newp+3, orntNew);CHKERRQ(ierr);
4808 #if 1
4809       if ((newp+3 < cStartNew) || (newp+3 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+3, cStartNew, cEndNew);
4810       for (p = 0; p < 3; ++p) {
4811         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4812       }
4813 #endif
4814     }
4815     /*
4816      2----3----3
4817      |         |
4818      |    B    |
4819      |         |
4820      0----4--- 1
4821      |         |
4822      |    A    |
4823      |         |
4824      0----2----1
4825      */
4826     /* Hybrid cells have 4 faces */
4827     for (c = cMax; c < cEnd; ++c) {
4828       const PetscInt  newp = cStartNew + (cMax - cStart)*4 + (c - cMax)*2;
4829       const PetscInt *cone, *ornt;
4830       PetscInt        coneNew[4], orntNew[4];
4831 
4832       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4833       ierr = DMPlexGetConeOrientation(dm, c, &ornt);CHKERRQ(ierr);
4834       /* A quad */
4835       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 1 : 0);
4836       orntNew[0] = ornt[0];
4837       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 1 : 0);
4838       orntNew[1] = ornt[1];
4839       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (cone[2] - fMax);
4840       orntNew[2] = 0;
4841       coneNew[3] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (fEnd    - fMax) + (c - cMax);
4842       orntNew[3] = 0;
4843       ierr       = DMPlexSetCone(rdm, newp+0, coneNew);CHKERRQ(ierr);
4844       ierr       = DMPlexSetConeOrientation(rdm, newp+0, orntNew);CHKERRQ(ierr);
4845 #if 1
4846       if ((newp+0 < cStartNew) || (newp+0 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+0, cStartNew, cEndNew);
4847       for (p = 0; p < 4; ++p) {
4848         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4849       }
4850 #endif
4851       /* B quad */
4852       coneNew[0] = fStartNew + (cone[0] - fStart)*2 + (ornt[0] < 0 ? 0 : 1);
4853       orntNew[0] = ornt[0];
4854       coneNew[1] = fStartNew + (cone[1] - fStart)*2 + (ornt[1] < 0 ? 0 : 1);
4855       orntNew[1] = ornt[1];
4856       coneNew[2] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (fEnd    - fMax) + (c - cMax);
4857       orntNew[2] = 0;
4858       coneNew[3] = fStartNew + (fMax    - fStart)*2 + (cMax - cStart)*3 + (cone[3] - fMax);
4859       orntNew[3] = 0;
4860       ierr       = DMPlexSetCone(rdm, newp+1, coneNew);CHKERRQ(ierr);
4861       ierr       = DMPlexSetConeOrientation(rdm, newp+1, orntNew);CHKERRQ(ierr);
4862 #if 1
4863       if ((newp+1 < cStartNew) || (newp+1 >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", newp+1, cStartNew, cEndNew);
4864       for (p = 0; p < 4; ++p) {
4865         if ((coneNew[p] < fStartNew) || (coneNew[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", coneNew[p], fStartNew, fEndNew);
4866       }
4867 #endif
4868     }
4869     /* Interior split faces have 2 vertices and the same cells as the parent */
4870     ierr = DMPlexGetMaxSizes(dm, NULL, &maxSupportSize);CHKERRQ(ierr);
4871     ierr = PetscMalloc((2 + maxSupportSize*2) * sizeof(PetscInt), &supportRef);CHKERRQ(ierr);
4872     for (f = fStart; f < fMax; ++f) {
4873       const PetscInt newv = vStartNew + (vEnd - vStart) + (f - fStart);
4874 
4875       for (r = 0; r < 2; ++r) {
4876         const PetscInt  newp = fStartNew + (f - fStart)*2 + r;
4877         const PetscInt *cone, *support;
4878         PetscInt        coneNew[2], coneSize, c, supportSize, s;
4879 
4880         ierr             = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4881         coneNew[0]       = vStartNew + (cone[0] - vStart);
4882         coneNew[1]       = vStartNew + (cone[1] - vStart);
4883         coneNew[(r+1)%2] = newv;
4884         ierr             = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4885 #if 1
4886         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4887         for (p = 0; p < 2; ++p) {
4888           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4889         }
4890 #endif
4891         ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
4892         ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4893         for (s = 0; s < supportSize; ++s) {
4894           if (support[s] >= cMax) {
4895             supportRef[s] = cStartNew + (cMax - cStart)*4 + (support[s] - cMax)*2 + r;
4896           } else {
4897             ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
4898             ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4899             for (c = 0; c < coneSize; ++c) {
4900               if (cone[c] == f) break;
4901             }
4902             supportRef[s] = cStartNew + (support[s] - cStart)*4 + (c+r)%3;
4903           }
4904         }
4905         ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
4906 #if 1
4907         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4908         for (p = 0; p < supportSize; ++p) {
4909           if ((supportRef[p] < cStartNew) || (supportRef[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportRef[p], cStartNew, cEndNew);
4910         }
4911 #endif
4912       }
4913     }
4914     /* Interior cell faces have 2 vertices and 2 cells */
4915     for (c = cStart; c < cMax; ++c) {
4916       const PetscInt *cone;
4917 
4918       ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4919       for (r = 0; r < 3; ++r) {
4920         const PetscInt newp = fStartNew + (fMax - fStart)*2 + (c - cStart)*3 + r;
4921         PetscInt       coneNew[2];
4922         PetscInt       supportNew[2];
4923 
4924         coneNew[0] = vStartNew + (vEnd - vStart) + (cone[r]       - fStart);
4925         coneNew[1] = vStartNew + (vEnd - vStart) + (cone[(r+1)%3] - fStart);
4926         ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4927 #if 1
4928         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4929         for (p = 0; p < 2; ++p) {
4930           if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4931         }
4932 #endif
4933         supportNew[0] = (c - cStart)*4 + (r+1)%3;
4934         supportNew[1] = (c - cStart)*4 + 3;
4935         ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4936 #if 1
4937         if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4938         for (p = 0; p < 2; ++p) {
4939           if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4940         }
4941 #endif
4942       }
4943     }
4944     /* Interior hybrid faces have 2 vertices and the same cells */
4945     for (f = fMax; f < fEnd; ++f) {
4946       const PetscInt  newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (f - fMax);
4947       const PetscInt *cone;
4948       const PetscInt *support;
4949       PetscInt        coneNew[2];
4950       PetscInt        supportNew[2];
4951       PetscInt        size, s, r;
4952 
4953       ierr       = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
4954       coneNew[0] = vStartNew + (cone[0] - vStart);
4955       coneNew[1] = vStartNew + (cone[1] - vStart);
4956       ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4957 #if 1
4958       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4959       for (p = 0; p < 2; ++p) {
4960         if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4961       }
4962 #endif
4963       ierr = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
4964       ierr = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
4965       for (s = 0; s < size; ++s) {
4966         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
4967         for (r = 0; r < 2; ++r) {
4968           if (cone[r+2] == f) break;
4969         }
4970         supportNew[s] = (cMax - cStart)*4 + (support[s] - cMax)*2 + r;
4971       }
4972       ierr = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
4973 #if 1
4974       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4975       for (p = 0; p < size; ++p) {
4976         if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
4977       }
4978 #endif
4979     }
4980     /* Cell hybrid faces have 2 vertices and 2 cells */
4981     for (c = cMax; c < cEnd; ++c) {
4982       const PetscInt  newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (c - cMax);
4983       const PetscInt *cone;
4984       PetscInt        coneNew[2];
4985       PetscInt        supportNew[2];
4986 
4987       ierr       = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
4988       coneNew[0] = vStartNew + (vEnd - vStart) + (cone[0] - fStart);
4989       coneNew[1] = vStartNew + (vEnd - vStart) + (cone[1] - fStart);
4990       ierr       = DMPlexSetCone(rdm, newp, coneNew);CHKERRQ(ierr);
4991 #if 1
4992       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
4993       for (p = 0; p < 2; ++p) {
4994         if ((coneNew[p] < vStartNew) || (coneNew[p] >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", coneNew[p], vStartNew, vEndNew);
4995       }
4996 #endif
4997       supportNew[0] = (cMax - cStart)*4 + (c - cMax)*2 + 0;
4998       supportNew[1] = (cMax - cStart)*4 + (c - cMax)*2 + 1;
4999       ierr          = DMPlexSetSupport(rdm, newp, supportNew);CHKERRQ(ierr);
5000 #if 1
5001       if ((newp < fStartNew) || (newp >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", newp, fStartNew, fEndNew);
5002       for (p = 0; p < 2; ++p) {
5003         if ((supportNew[p] < cStartNew) || (supportNew[p] >= cEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a cell [%d, %d)", supportNew[p], cStartNew, cEndNew);
5004       }
5005 #endif
5006     }
5007     /* Old vertices have identical supports */
5008     for (v = vStart; v < vEnd; ++v) {
5009       const PetscInt  newp = vStartNew + (v - vStart);
5010       const PetscInt *support, *cone;
5011       PetscInt        size, s;
5012 
5013       ierr = DMPlexGetSupportSize(dm, v, &size);CHKERRQ(ierr);
5014       ierr = DMPlexGetSupport(dm, v, &support);CHKERRQ(ierr);
5015       for (s = 0; s < size; ++s) {
5016         if (support[s] >= fMax) {
5017           supportRef[s] = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (support[s] - fMax);
5018         } else {
5019           PetscInt r = 0;
5020 
5021           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
5022           if (cone[1] == v) r = 1;
5023           supportRef[s] = fStartNew + (support[s] - fStart)*2 + r;
5024         }
5025       }
5026       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
5027 #if 1
5028       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
5029       for (p = 0; p < size; ++p) {
5030         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
5031       }
5032 #endif
5033     }
5034     /* Face vertices have 2 + (2 interior, 1 hybrid) supports */
5035     for (f = fStart; f < fMax; ++f) {
5036       const PetscInt  newp = vStartNew + (vEnd - vStart) + (f - fStart);
5037       const PetscInt *cone, *support;
5038       PetscInt        size, newSize = 2, s;
5039 
5040       ierr          = DMPlexGetSupportSize(dm, f, &size);CHKERRQ(ierr);
5041       ierr          = DMPlexGetSupport(dm, f, &support);CHKERRQ(ierr);
5042       supportRef[0] = fStartNew + (f - fStart)*2 + 0;
5043       supportRef[1] = fStartNew + (f - fStart)*2 + 1;
5044       for (s = 0; s < size; ++s) {
5045         PetscInt r = 0;
5046 
5047         ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
5048         if (support[s] >= cMax) {
5049           supportRef[newSize+0] = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (fEnd - fMax) + (support[s] - cMax);
5050 
5051           newSize += 1;
5052         } else {
5053           if      (cone[1] == f) r = 1;
5054           else if (cone[2] == f) r = 2;
5055           supportRef[newSize+0] = fStartNew + (fMax - fStart)*2 + (support[s] - cStart)*3 + (r+2)%3;
5056           supportRef[newSize+1] = fStartNew + (fMax - fStart)*2 + (support[s] - cStart)*3 + r;
5057 
5058           newSize += 2;
5059         }
5060       }
5061       ierr = DMPlexSetSupport(rdm, newp, supportRef);CHKERRQ(ierr);
5062 #if 1
5063       if ((newp < vStartNew) || (newp >= vEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a vertex [%d, %d)", newp, vStartNew, vEndNew);
5064       for (p = 0; p < newSize; ++p) {
5065         if ((supportRef[p] < fStartNew) || (supportRef[p] >= fEndNew)) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d is not a face [%d, %d)", supportRef[p], fStartNew, fEndNew);
5066       }
5067 #endif
5068     }
5069     ierr = PetscFree(supportRef);CHKERRQ(ierr);
5070     break;
5071   default:
5072     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5073   }
5074   PetscFunctionReturn(0);
5075 }
5076 
5077 #undef __FUNCT__
5078 #define __FUNCT__ "CellRefinerSetCoordinates"
5079 PetscErrorCode CellRefinerSetCoordinates(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
5080 {
5081   PetscSection   coordSection, coordSectionNew;
5082   Vec            coordinates, coordinatesNew;
5083   PetscScalar   *coords, *coordsNew;
5084   PetscInt       dim, depth, coordSizeNew, cStart, cEnd, c, vStart, vStartNew, vEnd, v, fStart, fEnd, fMax, f;
5085   PetscErrorCode ierr;
5086 
5087   PetscFunctionBegin;
5088   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5089   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5090   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5091   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5092   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
5093   ierr = DMPlexGetHybridBounds(dm, NULL, &fMax, NULL, NULL);CHKERRQ(ierr);
5094   ierr = GetDepthStart_Private(depth, depthSize, NULL, NULL, NULL, &vStartNew);CHKERRQ(ierr);
5095   ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
5096   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &coordSectionNew);CHKERRQ(ierr);
5097   ierr = PetscSectionSetNumFields(coordSectionNew, 1);CHKERRQ(ierr);
5098   ierr = PetscSectionSetFieldComponents(coordSectionNew, 0, dim);CHKERRQ(ierr);
5099   ierr = PetscSectionSetChart(coordSectionNew, vStartNew, vStartNew+depthSize[0]);CHKERRQ(ierr);
5100   if (fMax < 0) fMax = fEnd;
5101   switch (refiner) {
5102   case 1:
5103   case 2:
5104   case 3:
5105     /* Simplicial and Hex 2D */
5106     /* All vertices have the dim coordinates */
5107     for (v = vStartNew; v < vStartNew+depthSize[0]; ++v) {
5108       ierr = PetscSectionSetDof(coordSectionNew, v, dim);CHKERRQ(ierr);
5109       ierr = PetscSectionSetFieldDof(coordSectionNew, v, 0, dim);CHKERRQ(ierr);
5110     }
5111     ierr = PetscSectionSetUp(coordSectionNew);CHKERRQ(ierr);
5112     ierr = DMPlexSetCoordinateSection(rdm, coordSectionNew);CHKERRQ(ierr);
5113     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
5114     ierr = PetscSectionGetStorageSize(coordSectionNew, &coordSizeNew);CHKERRQ(ierr);
5115     ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinatesNew);CHKERRQ(ierr);
5116     ierr = PetscObjectSetName((PetscObject) coordinatesNew, "coordinates");CHKERRQ(ierr);
5117     ierr = VecSetSizes(coordinatesNew, coordSizeNew, PETSC_DETERMINE);CHKERRQ(ierr);
5118     ierr = VecSetFromOptions(coordinatesNew);CHKERRQ(ierr);
5119     ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
5120     ierr = VecGetArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
5121     /* Old vertices have the same coordinates */
5122     for (v = vStart; v < vEnd; ++v) {
5123       const PetscInt newv = vStartNew + (v - vStart);
5124       PetscInt       off, offnew, d;
5125 
5126       ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
5127       ierr = PetscSectionGetOffset(coordSectionNew, newv, &offnew);CHKERRQ(ierr);
5128       for (d = 0; d < dim; ++d) {
5129         coordsNew[offnew+d] = coords[off+d];
5130       }
5131     }
5132     /* Face vertices have the average of endpoint coordinates */
5133     for (f = fStart; f < fMax; ++f) {
5134       const PetscInt  newv = vStartNew + (vEnd - vStart) + (f - fStart);
5135       const PetscInt *cone;
5136       PetscInt        coneSize, offA, offB, offnew, d;
5137 
5138       ierr = DMPlexGetConeSize(dm, f, &coneSize);CHKERRQ(ierr);
5139       if (coneSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Face %d cone should have two vertices, not %d", f, coneSize);
5140       ierr = DMPlexGetCone(dm, f, &cone);CHKERRQ(ierr);
5141       ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
5142       ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
5143       ierr = PetscSectionGetOffset(coordSectionNew, newv, &offnew);CHKERRQ(ierr);
5144       for (d = 0; d < dim; ++d) {
5145         coordsNew[offnew+d] = 0.5*(coords[offA+d] + coords[offB+d]);
5146       }
5147     }
5148     /* Just Hex 2D */
5149     if (refiner == 2) {
5150       /* Cell vertices have the average of corner coordinates */
5151       for (c = cStart; c < cEnd; ++c) {
5152         const PetscInt newv = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (c - cStart);
5153         PetscInt      *cone = NULL;
5154         PetscInt       closureSize, coneSize = 0, offA, offB, offC, offD, offnew, p, d;
5155 
5156         ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &cone);CHKERRQ(ierr);
5157         for (p = 0; p < closureSize*2; p += 2) {
5158           const PetscInt point = cone[p];
5159           if ((point >= vStart) && (point < vEnd)) cone[coneSize++] = point;
5160         }
5161         if (coneSize != 4) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Quad %d cone should have four vertices, not %d", c, coneSize);
5162         ierr = PetscSectionGetOffset(coordSection, cone[0], &offA);CHKERRQ(ierr);
5163         ierr = PetscSectionGetOffset(coordSection, cone[1], &offB);CHKERRQ(ierr);
5164         ierr = PetscSectionGetOffset(coordSection, cone[2], &offC);CHKERRQ(ierr);
5165         ierr = PetscSectionGetOffset(coordSection, cone[3], &offD);CHKERRQ(ierr);
5166         ierr = PetscSectionGetOffset(coordSectionNew, newv, &offnew);CHKERRQ(ierr);
5167         for (d = 0; d < dim; ++d) {
5168           coordsNew[offnew+d] = 0.25*(coords[offA+d] + coords[offB+d] + coords[offC+d] + coords[offD+d]);
5169         }
5170         ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &cone);CHKERRQ(ierr);
5171       }
5172     }
5173     ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
5174     ierr = VecRestoreArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
5175     ierr = DMSetCoordinatesLocal(rdm, coordinatesNew);CHKERRQ(ierr);
5176     ierr = VecDestroy(&coordinatesNew);CHKERRQ(ierr);
5177     ierr = PetscSectionDestroy(&coordSectionNew);CHKERRQ(ierr);
5178     break;
5179   default:
5180     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5181   }
5182   PetscFunctionReturn(0);
5183 }
5184 
5185 #undef __FUNCT__
5186 #define __FUNCT__ "DMPlexCreateProcessSF"
5187 PetscErrorCode DMPlexCreateProcessSF(DM dm, PetscSF sfPoint, IS *processRanks, PetscSF *sfProcess)
5188 {
5189   PetscInt           numRoots, numLeaves, l;
5190   const PetscInt    *localPoints;
5191   const PetscSFNode *remotePoints;
5192   PetscInt          *localPointsNew;
5193   PetscSFNode       *remotePointsNew;
5194   PetscInt          *ranks, *ranksNew;
5195   PetscErrorCode     ierr;
5196 
5197   PetscFunctionBegin;
5198   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
5199   ierr = PetscMalloc(numLeaves * sizeof(PetscInt), &ranks);CHKERRQ(ierr);
5200   for (l = 0; l < numLeaves; ++l) {
5201     ranks[l] = remotePoints[l].rank;
5202   }
5203   ierr = PetscSortRemoveDupsInt(&numLeaves, ranks);CHKERRQ(ierr);
5204   ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &ranksNew);CHKERRQ(ierr);
5205   ierr = PetscMalloc(numLeaves * sizeof(PetscInt),    &localPointsNew);CHKERRQ(ierr);
5206   ierr = PetscMalloc(numLeaves * sizeof(PetscSFNode), &remotePointsNew);CHKERRQ(ierr);
5207   for (l = 0; l < numLeaves; ++l) {
5208     ranksNew[l]              = ranks[l];
5209     localPointsNew[l]        = l;
5210     remotePointsNew[l].index = 0;
5211     remotePointsNew[l].rank  = ranksNew[l];
5212   }
5213   ierr = PetscFree(ranks);CHKERRQ(ierr);
5214   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), numLeaves, ranksNew, PETSC_OWN_POINTER, processRanks);CHKERRQ(ierr);
5215   ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfProcess);CHKERRQ(ierr);
5216   ierr = PetscSFSetFromOptions(*sfProcess);CHKERRQ(ierr);
5217   ierr = PetscSFSetGraph(*sfProcess, 1, numLeaves, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
5218   PetscFunctionReturn(0);
5219 }
5220 
5221 #undef __FUNCT__
5222 #define __FUNCT__ "CellRefinerCreateSF"
5223 PetscErrorCode CellRefinerCreateSF(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
5224 {
5225   PetscSF            sf, sfNew, sfProcess;
5226   IS                 processRanks;
5227   MPI_Datatype       depthType;
5228   PetscInt           numRoots, numLeaves, numLeavesNew = 0, l, m;
5229   const PetscInt    *localPoints, *neighbors;
5230   const PetscSFNode *remotePoints;
5231   PetscInt          *localPointsNew;
5232   PetscSFNode       *remotePointsNew;
5233   PetscInt          *depthSizeOld, *rdepthSize, *rdepthSizeOld, *rdepthMaxOld, *rvStart, *rvStartNew, *reStart, *reStartNew, *rfStart, *rfStartNew, *rcStart, *rcStartNew;
5234   PetscInt           depth, numNeighbors, pStartNew, pEndNew, cStart, cStartNew, cEnd, cMax, vStart, vStartNew, vEnd, vMax, fStart, fStartNew, fEnd, fMax, eStart, eStartNew, eEnd, eMax, r, n;
5235   PetscErrorCode     ierr;
5236 
5237   PetscFunctionBegin;
5238   ierr = DMPlexGetChart(rdm, &pStartNew, &pEndNew);CHKERRQ(ierr);
5239   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5240   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5241   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
5242   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5243   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
5244   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
5245   ierr = GetDepthStart_Private(depth, depthSize, &cStartNew, &fStartNew, &eStartNew, &vStartNew);CHKERRQ(ierr);
5246   switch (refiner) {
5247   case 3:
5248     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
5249     cMax = PetscMin(cEnd, cMax);
5250     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
5251     fMax = PetscMin(fEnd, fMax);
5252   }
5253   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
5254   ierr = DMGetPointSF(rdm, &sfNew);CHKERRQ(ierr);
5255   /* Caculate size of new SF */
5256   ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
5257   if (numRoots < 0) PetscFunctionReturn(0);
5258   for (l = 0; l < numLeaves; ++l) {
5259     const PetscInt p = localPoints[l];
5260 
5261     switch (refiner) {
5262     case 1:
5263       /* Simplicial 2D */
5264       if ((p >= vStart) && (p < vEnd)) {
5265         /* Old vertices stay the same */
5266         ++numLeavesNew;
5267       } else if ((p >= fStart) && (p < fEnd)) {
5268         /* Old faces add new faces and vertex */
5269         numLeavesNew += 1 + 2;
5270       } else if ((p >= cStart) && (p < cEnd)) {
5271         /* Old cells add new cells and interior faces */
5272         numLeavesNew += 4 + 3;
5273       }
5274       break;
5275     case 2:
5276       /* Hex 2D */
5277       if ((p >= vStart) && (p < vEnd)) {
5278         /* Old vertices stay the same */
5279         ++numLeavesNew;
5280       } else if ((p >= fStart) && (p < fEnd)) {
5281         /* Old faces add new faces and vertex */
5282         numLeavesNew += 1 + 2;
5283       } else if ((p >= cStart) && (p < cEnd)) {
5284         /* Old cells add new cells and interior faces */
5285         numLeavesNew += 4 + 4;
5286       }
5287       break;
5288     default:
5289       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5290     }
5291   }
5292   /* Communicate depthSizes for each remote rank */
5293   ierr = DMPlexCreateProcessSF(dm, sf, &processRanks, &sfProcess);CHKERRQ(ierr);
5294   ierr = ISGetLocalSize(processRanks, &numNeighbors);CHKERRQ(ierr);
5295   ierr = PetscMalloc5((depth+1)*numNeighbors,PetscInt,&rdepthSize,numNeighbors,PetscInt,&rvStartNew,numNeighbors,PetscInt,&reStartNew,numNeighbors,PetscInt,&rfStartNew,numNeighbors,PetscInt,&rcStartNew);CHKERRQ(ierr);
5296   ierr = PetscMalloc7(depth+1,PetscInt,&depthSizeOld,(depth+1)*numNeighbors,PetscInt,&rdepthSizeOld,(depth+1)*numNeighbors,PetscInt,&rdepthMaxOld,numNeighbors,PetscInt,&rvStart,numNeighbors,PetscInt,&reStart,numNeighbors,PetscInt,&rfStart,numNeighbors,PetscInt,&rcStart);CHKERRQ(ierr);
5297   ierr = MPI_Type_contiguous(depth+1, MPIU_INT, &depthType);CHKERRQ(ierr);
5298   ierr = MPI_Type_commit(&depthType);CHKERRQ(ierr);
5299   ierr = PetscSFBcastBegin(sfProcess, depthType, depthSize, rdepthSize);CHKERRQ(ierr);
5300   ierr = PetscSFBcastEnd(sfProcess, depthType, depthSize, rdepthSize);CHKERRQ(ierr);
5301   for (n = 0; n < numNeighbors; ++n) {
5302     ierr = GetDepthStart_Private(depth, &rdepthSize[n*(depth+1)], &rcStartNew[n], &rfStartNew[n], &reStartNew[n], &rvStartNew[n]);CHKERRQ(ierr);
5303   }
5304   depthSizeOld[depth]   = cMax;
5305   depthSizeOld[0]       = vMax;
5306   depthSizeOld[depth-1] = fMax;
5307   depthSizeOld[1]       = eMax;
5308 
5309   ierr = PetscSFBcastBegin(sfProcess, depthType, depthSizeOld, rdepthMaxOld);CHKERRQ(ierr);
5310   ierr = PetscSFBcastEnd(sfProcess, depthType, depthSizeOld, rdepthMaxOld);CHKERRQ(ierr);
5311 
5312   depthSizeOld[depth]   = cEnd - cStart;
5313   depthSizeOld[0]       = vEnd - vStart;
5314   depthSizeOld[depth-1] = fEnd - fStart;
5315   depthSizeOld[1]       = eEnd - eStart;
5316 
5317   ierr = PetscSFBcastBegin(sfProcess, depthType, depthSizeOld, rdepthSizeOld);CHKERRQ(ierr);
5318   ierr = PetscSFBcastEnd(sfProcess, depthType, depthSizeOld, rdepthSizeOld);CHKERRQ(ierr);
5319   for (n = 0; n < numNeighbors; ++n) {
5320     ierr = GetDepthStart_Private(depth, &rdepthSizeOld[n*(depth+1)], &rcStart[n], &rfStart[n], &reStart[n], &rvStart[n]);CHKERRQ(ierr);
5321   }
5322   ierr = MPI_Type_free(&depthType);CHKERRQ(ierr);
5323   ierr = PetscSFDestroy(&sfProcess);CHKERRQ(ierr);
5324   /* Calculate new point SF */
5325   ierr = PetscMalloc(numLeavesNew * sizeof(PetscInt),    &localPointsNew);CHKERRQ(ierr);
5326   ierr = PetscMalloc(numLeavesNew * sizeof(PetscSFNode), &remotePointsNew);CHKERRQ(ierr);
5327   ierr = ISGetIndices(processRanks, &neighbors);CHKERRQ(ierr);
5328   for (l = 0, m = 0; l < numLeaves; ++l) {
5329     PetscInt    p     = localPoints[l];
5330     PetscInt    rp    = remotePoints[l].index, n;
5331     PetscMPIInt rrank = remotePoints[l].rank;
5332 
5333     ierr = PetscFindInt(rrank, numNeighbors, neighbors, &n);CHKERRQ(ierr);
5334     if (n < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not locate remote rank %d", rrank);
5335     switch (refiner) {
5336     case 1:
5337       /* Simplicial 2D */
5338       if ((p >= vStart) && (p < vEnd)) {
5339         /* Old vertices stay the same */
5340         localPointsNew[m]        = vStartNew     + (p  - vStart);
5341         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
5342         remotePointsNew[m].rank  = rrank;
5343         ++m;
5344       } else if ((p >= fStart) && (p < fEnd)) {
5345         /* Old faces add new faces and vertex */
5346         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
5347         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
5348         remotePointsNew[m].rank  = rrank;
5349         ++m;
5350         for (r = 0; r < 2; ++r, ++m) {
5351           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
5352           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
5353           remotePointsNew[m].rank  = rrank;
5354         }
5355       } else if ((p >= cStart) && (p < cEnd)) {
5356         /* Old cells add new cells and interior faces */
5357         for (r = 0; r < 4; ++r, ++m) {
5358           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5359           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5360           remotePointsNew[m].rank  = rrank;
5361         }
5362         for (r = 0; r < 3; ++r, ++m) {
5363           localPointsNew[m]        = fStartNew     + (fEnd - fStart)*2                    + (p  - cStart)*3     + r;
5364           remotePointsNew[m].index = rfStartNew[n] + rdepthSizeOld[n*(depth+1)+depth-1]*2 + (rp - rcStart[n])*3 + r;
5365           remotePointsNew[m].rank  = rrank;
5366         }
5367       }
5368       break;
5369     case 2:
5370       /* Hex 2D */
5371       if ((p >= vStart) && (p < vEnd)) {
5372         /* Old vertices stay the same */
5373         localPointsNew[m]        = vStartNew     + (p  - vStart);
5374         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
5375         remotePointsNew[m].rank  = rrank;
5376         ++m;
5377       } else if ((p >= fStart) && (p < fEnd)) {
5378         /* Old faces add new faces and vertex */
5379         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
5380         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
5381         remotePointsNew[m].rank  = rrank;
5382         ++m;
5383         for (r = 0; r < 2; ++r, ++m) {
5384           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
5385           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
5386           remotePointsNew[m].rank  = rrank;
5387         }
5388       } else if ((p >= cStart) && (p < cEnd)) {
5389         /* Old cells add new cells and interior faces */
5390         for (r = 0; r < 4; ++r, ++m) {
5391           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5392           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5393           remotePointsNew[m].rank  = rrank;
5394         }
5395         for (r = 0; r < 4; ++r, ++m) {
5396           localPointsNew[m]        = fStartNew     + (fEnd - fStart)*2                    + (p  - cStart)*4     + r;
5397           remotePointsNew[m].index = rfStartNew[n] + rdepthSizeOld[n*(depth+1)+depth-1]*2 + (rp - rcStart[n])*4 + r;
5398           remotePointsNew[m].rank  = rrank;
5399         }
5400       }
5401       break;
5402     case 3:
5403       /* Hybrid simplicial 2D */
5404       if ((p >= vStart) && (p < vEnd)) {
5405         /* Old vertices stay the same */
5406         localPointsNew[m]        = vStartNew     + (p  - vStart);
5407         remotePointsNew[m].index = rvStartNew[n] + (rp - rvStart[n]);
5408         remotePointsNew[m].rank  = rrank;
5409         ++m;
5410       } else if ((p >= fStart) && (p < fMax)) {
5411         /* Old interior faces add new faces and vertex */
5412         localPointsNew[m]        = vStartNew     + (vEnd - vStart)              + (p  - fStart);
5413         remotePointsNew[m].index = rvStartNew[n] + rdepthSizeOld[n*(depth+1)+0] + (rp - rfStart[n]);
5414         remotePointsNew[m].rank  = rrank;
5415         ++m;
5416         for (r = 0; r < 2; ++r, ++m) {
5417           localPointsNew[m]        = fStartNew     + (p  - fStart)*2     + r;
5418           remotePointsNew[m].index = rfStartNew[n] + (rp - rfStart[n])*2 + r;
5419           remotePointsNew[m].rank  = rrank;
5420         }
5421       } else if ((p >= fMax) && (p < fEnd)) {
5422         /* Old hybrid faces stay the same */
5423         localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (p  - fMax);
5424         remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rp - rdepthMaxOld[n*(depth+1)+depth-1]);
5425         remotePointsNew[m].rank  = rrank;
5426         ++m;
5427       } else if ((p >= cStart) && (p < cMax)) {
5428         /* Old interior cells add new cells and interior faces */
5429         for (r = 0; r < 4; ++r, ++m) {
5430           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5431           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5432           remotePointsNew[m].rank  = rrank;
5433         }
5434         for (r = 0; r < 3; ++r, ++m) {
5435           localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (p  - cStart)*3     + r;
5436           remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rp - rcStart[n])*3 + r;
5437           remotePointsNew[m].rank  = rrank;
5438         }
5439       } else if ((p >= cStart) && (p < cMax)) {
5440         /* Old hybrid cells add new cells and hybrid face */
5441         for (r = 0; r < 2; ++r, ++m) {
5442           localPointsNew[m]        = cStartNew     + (p  - cStart)*4     + r;
5443           remotePointsNew[m].index = rcStartNew[n] + (rp - rcStart[n])*4 + r;
5444           remotePointsNew[m].rank  = rrank;
5445         }
5446         localPointsNew[m]        = fStartNew     + (fMax                              - fStart)*2     + (cMax                            - cStart)*3     + (p  - cMax);
5447         remotePointsNew[m].index = rfStartNew[n] + (rdepthMaxOld[n*(depth+1)+depth-1] - rfStart[n])*2 + (rdepthMaxOld[n*(depth+1)+depth] - rcStart[n])*3 + (rp - rdepthMaxOld[n*(depth+1)+depth]);
5448         remotePointsNew[m].rank  = rrank;
5449         ++m;
5450       }
5451       break;
5452     default:
5453       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5454     }
5455   }
5456   ierr = ISRestoreIndices(processRanks, &neighbors);CHKERRQ(ierr);
5457   ierr = ISDestroy(&processRanks);CHKERRQ(ierr);
5458   ierr = PetscSFSetGraph(sfNew, pEndNew-pStartNew, numLeavesNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
5459   ierr = PetscFree5(rdepthSize,rvStartNew,reStartNew,rfStartNew,rcStartNew);CHKERRQ(ierr);
5460   ierr = PetscFree6(depthSizeOld,rdepthSizeOld,rvStart,reStart,rfStart,rcStart);CHKERRQ(ierr);
5461   PetscFunctionReturn(0);
5462 }
5463 
5464 #undef __FUNCT__
5465 #define __FUNCT__ "CellRefinerCreateLabels"
5466 PetscErrorCode CellRefinerCreateLabels(CellRefiner refiner, DM dm, PetscInt depthSize[], DM rdm)
5467 {
5468   PetscInt       numLabels, l;
5469   PetscInt       newp, cStart, cStartNew, cEnd, cMax, vStart, vStartNew, vEnd, vMax, fStart, fStartNew, fEnd, fMax, eStart, eEnd, eMax, r;
5470   PetscErrorCode ierr;
5471 
5472   PetscFunctionBegin;
5473   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
5474   ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd);CHKERRQ(ierr);
5475   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5476   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
5477 
5478   cStartNew = 0;
5479   vStartNew = depthSize[2];
5480   fStartNew = depthSize[2] + depthSize[0];
5481 
5482   ierr = DMPlexGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
5483   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
5484   switch (refiner) {
5485   case 3:
5486     if (cMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No cell maximum specified in hybrid mesh");
5487     cMax = PetscMin(cEnd, cMax);
5488     if (fMax < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No face maximum specified in hybrid mesh");
5489     fMax = PetscMin(fEnd, fMax);
5490   }
5491   for (l = 0; l < numLabels; ++l) {
5492     DMLabel         label, labelNew;
5493     const char     *lname;
5494     PetscBool       isDepth;
5495     IS              valueIS;
5496     const PetscInt *values;
5497     PetscInt        numValues, val;
5498 
5499     ierr = DMPlexGetLabelName(dm, l, &lname);CHKERRQ(ierr);
5500     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
5501     if (isDepth) continue;
5502     ierr = DMPlexCreateLabel(rdm, lname);CHKERRQ(ierr);
5503     ierr = DMPlexGetLabel(dm, lname, &label);CHKERRQ(ierr);
5504     ierr = DMPlexGetLabel(rdm, lname, &labelNew);CHKERRQ(ierr);
5505     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
5506     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
5507     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
5508     for (val = 0; val < numValues; ++val) {
5509       IS              pointIS;
5510       const PetscInt *points;
5511       PetscInt        numPoints, n;
5512 
5513       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
5514       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
5515       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
5516       for (n = 0; n < numPoints; ++n) {
5517         const PetscInt p = points[n];
5518         switch (refiner) {
5519         case 1:
5520           /* Simplicial 2D */
5521           if ((p >= vStart) && (p < vEnd)) {
5522             /* Old vertices stay the same */
5523             newp = vStartNew + (p - vStart);
5524             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5525           } else if ((p >= fStart) && (p < fEnd)) {
5526             /* Old faces add new faces and vertex */
5527             newp = vStartNew + (vEnd - vStart) + (p - fStart);
5528             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5529             for (r = 0; r < 2; ++r) {
5530               newp = fStartNew + (p - fStart)*2 + r;
5531               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5532             }
5533           } else if ((p >= cStart) && (p < cEnd)) {
5534             /* Old cells add new cells and interior faces */
5535             for (r = 0; r < 4; ++r) {
5536               newp = cStartNew + (p - cStart)*4 + r;
5537               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5538             }
5539             for (r = 0; r < 3; ++r) {
5540               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*3 + r;
5541               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5542             }
5543           }
5544           break;
5545         case 2:
5546           /* Hex 2D */
5547           if ((p >= vStart) && (p < vEnd)) {
5548             /* Old vertices stay the same */
5549             newp = vStartNew + (p - vStart);
5550             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5551           } else if ((p >= fStart) && (p < fEnd)) {
5552             /* Old faces add new faces and vertex */
5553             newp = vStartNew + (vEnd - vStart) + (p - fStart);
5554             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5555             for (r = 0; r < 2; ++r) {
5556               newp = fStartNew + (p - fStart)*2 + r;
5557               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5558             }
5559           } else if ((p >= cStart) && (p < cEnd)) {
5560             /* Old cells add new cells and interior faces and vertex */
5561             for (r = 0; r < 4; ++r) {
5562               newp = cStartNew + (p - cStart)*4 + r;
5563               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5564             }
5565             for (r = 0; r < 4; ++r) {
5566               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*4 + r;
5567               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5568             }
5569             newp = vStartNew + (vEnd - vStart) + (fEnd - fStart) + (p - cStart);
5570             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5571           }
5572           break;
5573         case 3:
5574           /* Hybrid simplicial 2D */
5575           if ((p >= vStart) && (p < vEnd)) {
5576             /* Old vertices stay the same */
5577             newp = vStartNew + (p - vStart);
5578             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5579           } else if ((p >= fStart) && (p < fMax)) {
5580             /* Old interior faces add new faces and vertex */
5581             newp = vStartNew + (vEnd - vStart) + (p - fStart);
5582             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5583             for (r = 0; r < 2; ++r) {
5584               newp = fStartNew + (p - fStart)*2 + r;
5585               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5586             }
5587           } else if ((p >= fMax) && (p < fEnd)) {
5588             /* Old hybrid faces stay the same */
5589             newp = fStartNew + (fMax - fStart)*2 + (p - fMax);
5590             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5591           } else if ((p >= cStart) && (p < cMax)) {
5592             /* Old interior cells add new cells and interior faces */
5593             for (r = 0; r < 4; ++r) {
5594               newp = cStartNew + (p - cStart)*4 + r;
5595               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5596             }
5597             for (r = 0; r < 3; ++r) {
5598               newp = fStartNew + (fEnd - fStart)*2 + (p - cStart)*3 + r;
5599               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5600             }
5601           } else if ((p >= cMax) && (p < cEnd)) {
5602             /* Old hybrid cells add new cells and hybrid face */
5603             for (r = 0; r < 2; ++r) {
5604               newp = cStartNew + (cMax - cStart)*4 + (p - cMax)*2 + r;
5605               ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5606             }
5607             newp = fStartNew + (fMax - fStart)*2 + (cMax - cStart)*3 + (p - cMax);
5608             ierr = DMLabelSetValue(labelNew, newp, values[val]);CHKERRQ(ierr);
5609           }
5610           break;
5611         default:
5612           SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown cell refiner %d", refiner);
5613         }
5614       }
5615       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
5616       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
5617     }
5618     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
5619     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
5620     if (0) {
5621       ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD, PETSC_TRUE);CHKERRQ(ierr);
5622       ierr = DMLabelView(labelNew, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
5623       ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
5624     }
5625   }
5626   PetscFunctionReturn(0);
5627 }
5628 
5629 #undef __FUNCT__
5630 #define __FUNCT__ "DMPlexRefine_Uniform"
5631 /* This will only work for interpolated meshes */
5632 PetscErrorCode DMPlexRefine_Uniform(DM dm, CellRefiner cellRefiner, DM *dmRefined)
5633 {
5634   DM             rdm;
5635   PetscInt      *depthSize;
5636   PetscInt       dim, depth = 0, d, pStart = 0, pEnd = 0;
5637   PetscErrorCode ierr;
5638 
5639   PetscFunctionBegin;
5640   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &rdm);CHKERRQ(ierr);
5641   ierr = DMSetType(rdm, DMPLEX);CHKERRQ(ierr);
5642   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5643   ierr = DMPlexSetDimension(rdm, dim);CHKERRQ(ierr);
5644   /* Calculate number of new points of each depth */
5645   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
5646   ierr = PetscMalloc((depth+1) * sizeof(PetscInt), &depthSize);CHKERRQ(ierr);
5647   ierr = PetscMemzero(depthSize, (depth+1) * sizeof(PetscInt));CHKERRQ(ierr);
5648   ierr = CellRefinerGetSizes(cellRefiner, dm, depthSize);CHKERRQ(ierr);
5649   /* Step 1: Set chart */
5650   for (d = 0; d <= depth; ++d) pEnd += depthSize[d];
5651   ierr = DMPlexSetChart(rdm, pStart, pEnd);CHKERRQ(ierr);
5652   /* Step 2: Set cone/support sizes */
5653   ierr = CellRefinerSetConeSizes(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5654   /* Step 3: Setup refined DM */
5655   ierr = DMSetUp(rdm);CHKERRQ(ierr);
5656   /* Step 4: Set cones and supports */
5657   ierr = CellRefinerSetCones(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5658   /* Step 5: Stratify */
5659   ierr = DMPlexStratify(rdm);CHKERRQ(ierr);
5660   /* Step 6: Set coordinates for vertices */
5661   ierr = CellRefinerSetCoordinates(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5662   /* Step 7: Create pointSF */
5663   ierr = CellRefinerCreateSF(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5664   /* Step 8: Create labels */
5665   ierr = CellRefinerCreateLabels(cellRefiner, dm, depthSize, rdm);CHKERRQ(ierr);
5666   ierr = PetscFree(depthSize);CHKERRQ(ierr);
5667 
5668   *dmRefined = rdm;
5669   PetscFunctionReturn(0);
5670 }
5671 
5672 #undef __FUNCT__
5673 #define __FUNCT__ "DMPlexSetRefinementUniform"
5674 PetscErrorCode DMPlexSetRefinementUniform(DM dm, PetscBool refinementUniform)
5675 {
5676   DM_Plex *mesh = (DM_Plex*) dm->data;
5677 
5678   PetscFunctionBegin;
5679   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5680   mesh->refinementUniform = refinementUniform;
5681   PetscFunctionReturn(0);
5682 }
5683 
5684 #undef __FUNCT__
5685 #define __FUNCT__ "DMPlexGetRefinementUniform"
5686 PetscErrorCode DMPlexGetRefinementUniform(DM dm, PetscBool *refinementUniform)
5687 {
5688   DM_Plex *mesh = (DM_Plex*) dm->data;
5689 
5690   PetscFunctionBegin;
5691   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5692   PetscValidPointer(refinementUniform,  2);
5693   *refinementUniform = mesh->refinementUniform;
5694   PetscFunctionReturn(0);
5695 }
5696 
5697 #undef __FUNCT__
5698 #define __FUNCT__ "DMPlexSetRefinementLimit"
5699 PetscErrorCode DMPlexSetRefinementLimit(DM dm, PetscReal refinementLimit)
5700 {
5701   DM_Plex *mesh = (DM_Plex*) dm->data;
5702 
5703   PetscFunctionBegin;
5704   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5705   mesh->refinementLimit = refinementLimit;
5706   PetscFunctionReturn(0);
5707 }
5708 
5709 #undef __FUNCT__
5710 #define __FUNCT__ "DMPlexGetRefinementLimit"
5711 PetscErrorCode DMPlexGetRefinementLimit(DM dm, PetscReal *refinementLimit)
5712 {
5713   DM_Plex *mesh = (DM_Plex*) dm->data;
5714 
5715   PetscFunctionBegin;
5716   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5717   PetscValidPointer(refinementLimit,  2);
5718   /* if (mesh->refinementLimit < 0) = getMaxVolume()/2.0; */
5719   *refinementLimit = mesh->refinementLimit;
5720   PetscFunctionReturn(0);
5721 }
5722 
5723 #undef __FUNCT__
5724 #define __FUNCT__ "DMPlexGetCellRefiner_Private"
5725 PetscErrorCode DMPlexGetCellRefiner_Private(DM dm, CellRefiner *cellRefiner)
5726 {
5727   PetscInt       dim, cStart, coneSize, cMax;
5728   PetscErrorCode ierr;
5729 
5730   PetscFunctionBegin;
5731   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5732   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
5733   ierr = DMPlexGetConeSize(dm, cStart, &coneSize);CHKERRQ(ierr);
5734   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
5735   switch (dim) {
5736   case 2:
5737     switch (coneSize) {
5738     case 3:
5739       if (cMax >= 0) *cellRefiner = 3; /* Hybrid */
5740       else *cellRefiner = 1; /* Triangular */
5741       break;
5742     case 4:
5743       if (cMax >= 0) *cellRefiner = 4; /* Hybrid */
5744       else *cellRefiner = 2; /* Quadrilateral */
5745       break;
5746     default:
5747       SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown coneSize %d in dimension %d for cell refiner", coneSize, dim);
5748     }
5749     break;
5750   default:
5751     SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown dimension %d for cell refiner", dim);
5752   }
5753   PetscFunctionReturn(0);
5754 }
5755 
5756 #undef __FUNCT__
5757 #define __FUNCT__ "DMRefine_Plex"
5758 PetscErrorCode DMRefine_Plex(DM dm, MPI_Comm comm, DM *dmRefined)
5759 {
5760   PetscReal      refinementLimit;
5761   PetscInt       dim, cStart, cEnd;
5762   char           genname[1024], *name = NULL;
5763   PetscBool      isUniform, isTriangle = PETSC_FALSE, isTetgen = PETSC_FALSE, isCTetgen = PETSC_FALSE, flg;
5764   PetscErrorCode ierr;
5765 
5766   PetscFunctionBegin;
5767   ierr = DMPlexGetRefinementUniform(dm, &isUniform);CHKERRQ(ierr);
5768   if (isUniform) {
5769     CellRefiner cellRefiner;
5770 
5771     ierr = DMPlexGetCellRefiner_Private(dm, &cellRefiner);CHKERRQ(ierr);
5772     ierr = DMPlexRefine_Uniform(dm, cellRefiner, dmRefined);CHKERRQ(ierr);
5773     PetscFunctionReturn(0);
5774   }
5775   ierr = DMPlexGetRefinementLimit(dm, &refinementLimit);CHKERRQ(ierr);
5776   if (refinementLimit == 0.0) PetscFunctionReturn(0);
5777   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
5778   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
5779   ierr = PetscOptionsGetString(((PetscObject) dm)->prefix, "-dm_plex_generator", genname, 1024, &flg);CHKERRQ(ierr);
5780   if (flg) name = genname;
5781   if (name) {
5782     ierr = PetscStrcmp(name, "triangle", &isTriangle);CHKERRQ(ierr);
5783     ierr = PetscStrcmp(name, "tetgen",   &isTetgen);CHKERRQ(ierr);
5784     ierr = PetscStrcmp(name, "ctetgen",  &isCTetgen);CHKERRQ(ierr);
5785   }
5786   switch (dim) {
5787   case 2:
5788     if (!name || isTriangle) {
5789 #if defined(PETSC_HAVE_TRIANGLE)
5790       double  *maxVolumes;
5791       PetscInt c;
5792 
5793       ierr = PetscMalloc((cEnd - cStart) * sizeof(double), &maxVolumes);CHKERRQ(ierr);
5794       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
5795       ierr = DMPlexRefine_Triangle(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
5796       ierr = PetscFree(maxVolumes);CHKERRQ(ierr);
5797 #else
5798       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement needs external package support.\nPlease reconfigure with --download-triangle.");
5799 #endif
5800     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 2D mesh generation package %s", name);
5801     break;
5802   case 3:
5803     if (!name || isCTetgen) {
5804 #if defined(PETSC_HAVE_CTETGEN)
5805       PetscReal *maxVolumes;
5806       PetscInt   c;
5807 
5808       ierr = PetscMalloc((cEnd - cStart) * sizeof(PetscReal), &maxVolumes);CHKERRQ(ierr);
5809       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
5810       ierr = DMPlexRefine_CTetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
5811 #else
5812       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CTetgen needs external package support.\nPlease reconfigure with --download-ctetgen.");
5813 #endif
5814     } else if (isTetgen) {
5815 #if defined(PETSC_HAVE_TETGEN)
5816       double  *maxVolumes;
5817       PetscInt c;
5818 
5819       ierr = PetscMalloc((cEnd - cStart) * sizeof(double), &maxVolumes);CHKERRQ(ierr);
5820       for (c = 0; c < cEnd-cStart; ++c) maxVolumes[c] = refinementLimit;
5821       ierr = DMPlexRefine_Tetgen(dm, maxVolumes, dmRefined);CHKERRQ(ierr);
5822 #else
5823       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Tetgen needs external package support.\nPlease reconfigure with --with-c-language=cxx --download-tetgen.");
5824 #endif
5825     } else SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unknown 3D mesh generation package %s", name);
5826     break;
5827   default:
5828     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh refinement in dimension %d is not supported.", dim);
5829   }
5830   PetscFunctionReturn(0);
5831 }
5832 
5833 #undef __FUNCT__
5834 #define __FUNCT__ "DMPlexGetDepthLabel"
5835 /*@
5836   DMPlexGetDepthLabel - Get the DMLabel recording the depth of each point
5837 
5838   Not Collective
5839 
5840   Input Parameter:
5841 . dm    - The DMPlex object
5842 
5843   Output Parameter:
5844 . depthLabel - The DMLabel recording point depth
5845 
5846   Level: developer
5847 
5848 .keywords: mesh, points
5849 .seealso: DMPlexGetDepth(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
5850 @*/
5851 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
5852 {
5853   DM_Plex       *mesh = (DM_Plex*) dm->data;
5854   PetscErrorCode ierr;
5855 
5856   PetscFunctionBegin;
5857   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5858   PetscValidPointer(depthLabel, 2);
5859   if (!mesh->depthLabel) {
5860     ierr = DMPlexGetLabel(dm, "depth", &mesh->depthLabel);CHKERRQ(ierr);
5861   }
5862   *depthLabel = mesh->depthLabel;
5863   PetscFunctionReturn(0);
5864 }
5865 
5866 #undef __FUNCT__
5867 #define __FUNCT__ "DMPlexGetDepth"
5868 /*@
5869   DMPlexGetDepth - Get the depth of the DAG representing this mesh
5870 
5871   Not Collective
5872 
5873   Input Parameter:
5874 . dm    - The DMPlex object
5875 
5876   Output Parameter:
5877 . depth - The number of strata (breadth first levels) in the DAG
5878 
5879   Level: developer
5880 
5881 .keywords: mesh, points
5882 .seealso: DMPlexGetDepthLabel(), DMPlexGetHeightStratum(), DMPlexGetDepthStratum()
5883 @*/
5884 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
5885 {
5886   DMLabel        label;
5887   PetscInt       d = 0;
5888   PetscErrorCode ierr;
5889 
5890   PetscFunctionBegin;
5891   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5892   PetscValidPointer(depth, 2);
5893   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5894   if (label) {ierr = DMLabelGetNumValues(label, &d);CHKERRQ(ierr);}
5895   *depth = d-1;
5896   PetscFunctionReturn(0);
5897 }
5898 
5899 #undef __FUNCT__
5900 #define __FUNCT__ "DMPlexGetDepthStratum"
5901 /*@
5902   DMPlexGetDepthStratum - Get the bounds [start, end) for all points at a certain depth.
5903 
5904   Not Collective
5905 
5906   Input Parameters:
5907 + dm           - The DMPlex object
5908 - stratumValue - The requested depth
5909 
5910   Output Parameters:
5911 + start - The first point at this depth
5912 - end   - One beyond the last point at this depth
5913 
5914   Level: developer
5915 
5916 .keywords: mesh, points
5917 .seealso: DMPlexGetHeightStratum(), DMPlexGetDepth()
5918 @*/
5919 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
5920 {
5921   DMLabel        label;
5922   PetscInt       depth;
5923   PetscErrorCode ierr;
5924 
5925   PetscFunctionBegin;
5926   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5927   if (stratumValue < 0) {
5928     ierr = DMPlexGetChart(dm, start, end);CHKERRQ(ierr);
5929     PetscFunctionReturn(0);
5930   } else {
5931     PetscInt pStart, pEnd;
5932 
5933     if (start) *start = 0;
5934     if (end)   *end   = 0;
5935     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5936     if (pStart == pEnd) PetscFunctionReturn(0);
5937   }
5938   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5939   if (!label) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
5940   /* Strata are sorted and contiguous -- In addition, depth/height is either full or 1-level */
5941   depth = stratumValue;
5942   if ((depth < 0) || (depth >= label->numStrata)) {
5943     if (start) *start = 0;
5944     if (end)   *end   = 0;
5945   } else {
5946     if (start) *start = label->points[label->stratumOffsets[depth]];
5947     if (end)   *end   = label->points[label->stratumOffsets[depth]+label->stratumSizes[depth]-1]+1;
5948   }
5949   PetscFunctionReturn(0);
5950 }
5951 
5952 #undef __FUNCT__
5953 #define __FUNCT__ "DMPlexGetHeightStratum"
5954 /*@
5955   DMPlexGetHeightStratum - Get the bounds [start, end) for all points at a certain height.
5956 
5957   Not Collective
5958 
5959   Input Parameters:
5960 + dm           - The DMPlex object
5961 - stratumValue - The requested height
5962 
5963   Output Parameters:
5964 + start - The first point at this height
5965 - end   - One beyond the last point at this height
5966 
5967   Level: developer
5968 
5969 .keywords: mesh, points
5970 .seealso: DMPlexGetDepthStratum(), DMPlexGetDepth()
5971 @*/
5972 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt stratumValue, PetscInt *start, PetscInt *end)
5973 {
5974   DMLabel        label;
5975   PetscInt       depth;
5976   PetscErrorCode ierr;
5977 
5978   PetscFunctionBegin;
5979   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5980   if (stratumValue < 0) {
5981     ierr = DMPlexGetChart(dm, start, end);CHKERRQ(ierr);
5982   } else {
5983     PetscInt pStart, pEnd;
5984 
5985     if (start) *start = 0;
5986     if (end)   *end   = 0;
5987     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
5988     if (pStart == pEnd) PetscFunctionReturn(0);
5989   }
5990   ierr = DMPlexGetDepthLabel(dm, &label);CHKERRQ(ierr);
5991   if (!label) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");CHKERRQ(ierr);
5992   /* Strata are sorted and contiguous -- In addition, depth/height is either full or 1-level */
5993   depth = label->stratumValues[label->numStrata-1] - stratumValue;
5994   if ((depth < 0) || (depth >= label->numStrata)) {
5995     if (start) *start = 0;
5996     if (end)   *end   = 0;
5997   } else {
5998     if (start) *start = label->points[label->stratumOffsets[depth]];
5999     if (end)   *end   = label->points[label->stratumOffsets[depth]+label->stratumSizes[depth]-1]+1;
6000   }
6001   PetscFunctionReturn(0);
6002 }
6003 
6004 #undef __FUNCT__
6005 #define __FUNCT__ "DMPlexCreateSectionInitial"
6006 /* Set the number of dof on each point and separate by fields */
6007 PetscErrorCode DMPlexCreateSectionInitial(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscSection *section)
6008 {
6009   PetscInt      *numDofTot;
6010   PetscInt       pStart = 0, pEnd = 0;
6011   PetscInt       p, d, f;
6012   PetscErrorCode ierr;
6013 
6014   PetscFunctionBegin;
6015   ierr = PetscMalloc((dim+1) * sizeof(PetscInt), &numDofTot);CHKERRQ(ierr);
6016   for (d = 0; d <= dim; ++d) {
6017     numDofTot[d] = 0;
6018     for (f = 0; f < numFields; ++f) numDofTot[d] += numDof[f*(dim+1)+d];
6019   }
6020   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), section);CHKERRQ(ierr);
6021   if (numFields > 0) {
6022     ierr = PetscSectionSetNumFields(*section, numFields);CHKERRQ(ierr);
6023     if (numComp) {
6024       for (f = 0; f < numFields; ++f) {
6025         ierr = PetscSectionSetFieldComponents(*section, f, numComp[f]);CHKERRQ(ierr);
6026       }
6027     }
6028   }
6029   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
6030   ierr = PetscSectionSetChart(*section, pStart, pEnd);CHKERRQ(ierr);
6031   for (d = 0; d <= dim; ++d) {
6032     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
6033     for (p = pStart; p < pEnd; ++p) {
6034       for (f = 0; f < numFields; ++f) {
6035         ierr = PetscSectionSetFieldDof(*section, p, f, numDof[f*(dim+1)+d]);CHKERRQ(ierr);
6036       }
6037       ierr = PetscSectionSetDof(*section, p, numDofTot[d]);CHKERRQ(ierr);
6038     }
6039   }
6040   ierr = PetscFree(numDofTot);CHKERRQ(ierr);
6041   PetscFunctionReturn(0);
6042 }
6043 
6044 #undef __FUNCT__
6045 #define __FUNCT__ "DMPlexCreateSectionBCDof"
6046 /* Set the number of dof on each point and separate by fields
6047    If constDof is PETSC_DETERMINE, constrain every dof on the point
6048 */
6049 PetscErrorCode DMPlexCreateSectionBCDof(DM dm, PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscInt constDof, PetscSection section)
6050 {
6051   PetscInt       numFields;
6052   PetscInt       bc;
6053   PetscErrorCode ierr;
6054 
6055   PetscFunctionBegin;
6056   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6057   for (bc = 0; bc < numBC; ++bc) {
6058     PetscInt        field = 0;
6059     const PetscInt *idx;
6060     PetscInt        n, i;
6061 
6062     if (numFields) field = bcField[bc];
6063     ierr = ISGetLocalSize(bcPoints[bc], &n);CHKERRQ(ierr);
6064     ierr = ISGetIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
6065     for (i = 0; i < n; ++i) {
6066       const PetscInt p        = idx[i];
6067       PetscInt       numConst = constDof;
6068 
6069       /* Constrain every dof on the point */
6070       if (numConst < 0) {
6071         if (numFields) {
6072           ierr = PetscSectionGetFieldDof(section, p, field, &numConst);CHKERRQ(ierr);
6073         } else {
6074           ierr = PetscSectionGetDof(section, p, &numConst);CHKERRQ(ierr);
6075         }
6076       }
6077       if (numFields) {
6078         ierr = PetscSectionAddFieldConstraintDof(section, p, field, numConst);CHKERRQ(ierr);
6079       }
6080       ierr = PetscSectionAddConstraintDof(section, p, numConst);CHKERRQ(ierr);
6081     }
6082     ierr = ISRestoreIndices(bcPoints[bc], &idx);CHKERRQ(ierr);
6083   }
6084   PetscFunctionReturn(0);
6085 }
6086 
6087 #undef __FUNCT__
6088 #define __FUNCT__ "DMPlexCreateSectionBCIndicesAll"
6089 /* Set the constrained indices on each point and separate by fields */
6090 PetscErrorCode DMPlexCreateSectionBCIndicesAll(DM dm, PetscSection section)
6091 {
6092   PetscInt      *maxConstraints;
6093   PetscInt       numFields, f, pStart = 0, pEnd = 0, p;
6094   PetscErrorCode ierr;
6095 
6096   PetscFunctionBegin;
6097   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6098   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6099   ierr = PetscMalloc((numFields+1) * sizeof(PetscInt), &maxConstraints);CHKERRQ(ierr);
6100   for (f = 0; f <= numFields; ++f) maxConstraints[f] = 0;
6101   for (p = pStart; p < pEnd; ++p) {
6102     PetscInt cdof;
6103 
6104     if (numFields) {
6105       for (f = 0; f < numFields; ++f) {
6106         ierr              = PetscSectionGetFieldConstraintDof(section, p, f, &cdof);CHKERRQ(ierr);
6107         maxConstraints[f] = PetscMax(maxConstraints[f], cdof);
6108       }
6109     } else {
6110       ierr              = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
6111       maxConstraints[0] = PetscMax(maxConstraints[0], cdof);
6112     }
6113   }
6114   for (f = 0; f < numFields; ++f) {
6115     maxConstraints[numFields] += maxConstraints[f];
6116   }
6117   if (maxConstraints[numFields]) {
6118     PetscInt *indices;
6119 
6120     ierr = PetscMalloc(maxConstraints[numFields] * sizeof(PetscInt), &indices);CHKERRQ(ierr);
6121     for (p = pStart; p < pEnd; ++p) {
6122       PetscInt cdof, d;
6123 
6124       ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
6125       if (cdof) {
6126         if (cdof > maxConstraints[numFields]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_LIB, "Likely memory corruption, point %D cDof %D > maxConstraints %D", p, cdof, maxConstraints[numFields]);
6127         if (numFields) {
6128           PetscInt numConst = 0, foff = 0;
6129 
6130           for (f = 0; f < numFields; ++f) {
6131             PetscInt cfdof, fdof;
6132 
6133             ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
6134             ierr = PetscSectionGetFieldConstraintDof(section, p, f, &cfdof);CHKERRQ(ierr);
6135             /* Change constraint numbering from absolute local dof number to field relative local dof number */
6136             for (d = 0; d < cfdof; ++d) indices[numConst+d] = d;
6137             ierr = PetscSectionSetFieldConstraintIndices(section, p, f, &indices[numConst]);CHKERRQ(ierr);
6138             for (d = 0; d < cfdof; ++d) indices[numConst+d] += foff;
6139             numConst += cfdof;
6140             foff     += fdof;
6141           }
6142           if (cdof != numConst) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
6143         } else {
6144           for (d = 0; d < cdof; ++d) indices[d] = d;
6145         }
6146         ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
6147       }
6148     }
6149     ierr = PetscFree(indices);CHKERRQ(ierr);
6150   }
6151   ierr = PetscFree(maxConstraints);CHKERRQ(ierr);
6152   PetscFunctionReturn(0);
6153 }
6154 
6155 #undef __FUNCT__
6156 #define __FUNCT__ "DMPlexCreateSectionBCIndicesField"
6157 /* Set the constrained field indices on each point */
6158 PetscErrorCode DMPlexCreateSectionBCIndicesField(DM dm, PetscInt field, IS bcPoints, IS constraintIndices, PetscSection section)
6159 {
6160   const PetscInt *points, *indices;
6161   PetscInt        numFields, maxDof, numPoints, p, numConstraints;
6162   PetscErrorCode  ierr;
6163 
6164   PetscFunctionBegin;
6165   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6166   if ((field < 0) || (field >= numFields)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Section field %d should be in [%d, %d)", field, 0, numFields);
6167 
6168   ierr = ISGetLocalSize(bcPoints, &numPoints);CHKERRQ(ierr);
6169   ierr = ISGetIndices(bcPoints, &points);CHKERRQ(ierr);
6170   if (!constraintIndices) {
6171     PetscInt *idx, i;
6172 
6173     ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6174     ierr = PetscMalloc(maxDof * sizeof(PetscInt), &idx);CHKERRQ(ierr);
6175     for (i = 0; i < maxDof; ++i) idx[i] = i;
6176     for (p = 0; p < numPoints; ++p) {
6177       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, idx);CHKERRQ(ierr);
6178     }
6179     ierr = PetscFree(idx);CHKERRQ(ierr);
6180   } else {
6181     ierr = ISGetLocalSize(constraintIndices, &numConstraints);CHKERRQ(ierr);
6182     ierr = ISGetIndices(constraintIndices, &indices);CHKERRQ(ierr);
6183     for (p = 0; p < numPoints; ++p) {
6184       PetscInt fcdof;
6185 
6186       ierr = PetscSectionGetFieldConstraintDof(section, points[p], field, &fcdof);CHKERRQ(ierr);
6187       if (fcdof != numConstraints) SETERRQ4(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section point %d field %d has %d constraints, but yo ugave %d indices", p, field, fcdof, numConstraints);
6188       ierr = PetscSectionSetFieldConstraintIndices(section, points[p], field, indices);CHKERRQ(ierr);
6189     }
6190     ierr = ISRestoreIndices(constraintIndices, &indices);CHKERRQ(ierr);
6191   }
6192   ierr = ISRestoreIndices(bcPoints, &points);CHKERRQ(ierr);
6193   PetscFunctionReturn(0);
6194 }
6195 
6196 #undef __FUNCT__
6197 #define __FUNCT__ "DMPlexCreateSectionBCIndices"
6198 /* Set the constrained indices on each point and separate by fields */
6199 PetscErrorCode DMPlexCreateSectionBCIndices(DM dm, PetscSection section)
6200 {
6201   PetscInt      *indices;
6202   PetscInt       numFields, maxDof, f, pStart = 0, pEnd = 0, p;
6203   PetscErrorCode ierr;
6204 
6205   PetscFunctionBegin;
6206   ierr = PetscSectionGetMaxDof(section, &maxDof);CHKERRQ(ierr);
6207   ierr = PetscMalloc(maxDof * sizeof(PetscInt), &indices);CHKERRQ(ierr);
6208   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6209   if (!numFields) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "This function only works after users have set field constraint indices.");
6210   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6211   for (p = pStart; p < pEnd; ++p) {
6212     PetscInt cdof, d;
6213 
6214     ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
6215     if (cdof) {
6216       PetscInt numConst = 0, foff = 0;
6217 
6218       for (f = 0; f < numFields; ++f) {
6219         const PetscInt *fcind;
6220         PetscInt        fdof, fcdof;
6221 
6222         ierr = PetscSectionGetFieldDof(section, p, f, &fdof);CHKERRQ(ierr);
6223         ierr = PetscSectionGetFieldConstraintDof(section, p, f, &fcdof);CHKERRQ(ierr);
6224         if (fcdof) {ierr = PetscSectionGetFieldConstraintIndices(section, p, f, &fcind);CHKERRQ(ierr);}
6225         /* Change constraint numbering from field relative local dof number to absolute local dof number */
6226         for (d = 0; d < fcdof; ++d) indices[numConst+d] = fcind[d]+foff;
6227         foff     += fdof;
6228         numConst += fcdof;
6229       }
6230       if (cdof != numConst) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_LIB, "Total number of field constraints %D should be %D", numConst, cdof);
6231       ierr = PetscSectionSetConstraintIndices(section, p, indices);CHKERRQ(ierr);
6232     }
6233   }
6234   ierr = PetscFree(indices);CHKERRQ(ierr);
6235   PetscFunctionReturn(0);
6236 }
6237 
6238 #undef __FUNCT__
6239 #define __FUNCT__ "DMPlexCreateSection"
6240 /*@C
6241   DMPlexCreateSection - Create a PetscSection based upon the dof layout specification provided.
6242 
6243   Not Collective
6244 
6245   Input Parameters:
6246 + dm        - The DMPlex object
6247 . dim       - The spatial dimension of the problem
6248 . numFields - The number of fields in the problem
6249 . numComp   - An array of size numFields that holds the number of components for each field
6250 . numDof    - An array of size numFields*(dim+1) which holds the number of dof for each field on a mesh piece of dimension d
6251 . numBC     - The number of boundary conditions
6252 . bcField   - An array of size numBC giving the field number for each boundry condition
6253 - bcPoints  - An array of size numBC giving an IS holding the sieve points to which each boundary condition applies
6254 
6255   Output Parameter:
6256 . section - The PetscSection object
6257 
6258   Notes: numDof[f*(dim+1)+d] gives the number of dof for field f on sieve points of dimension d. For instance, numDof[1] is the
6259   nubmer of dof for field 0 on each edge.
6260 
6261   Level: developer
6262 
6263   Fortran Notes:
6264   A Fortran 90 version is available as DMPlexCreateSectionF90()
6265 
6266 .keywords: mesh, elements
6267 .seealso: DMPlexCreate(), PetscSectionCreate()
6268 @*/
6269 PetscErrorCode DMPlexCreateSection(DM dm, PetscInt dim, PetscInt numFields,const PetscInt numComp[],const PetscInt numDof[], PetscInt numBC,const PetscInt bcField[],const IS bcPoints[], PetscSection *section)
6270 {
6271   PetscErrorCode ierr;
6272 
6273   PetscFunctionBegin;
6274   ierr = DMPlexCreateSectionInitial(dm, dim, numFields, numComp, numDof, section);CHKERRQ(ierr);
6275   ierr = DMPlexCreateSectionBCDof(dm, numBC, bcField, bcPoints, PETSC_DETERMINE, *section);CHKERRQ(ierr);
6276   ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
6277   if (numBC) {ierr = DMPlexCreateSectionBCIndicesAll(dm, *section);CHKERRQ(ierr);}
6278   {
6279     PetscBool view = PETSC_FALSE;
6280 
6281     ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-section_view", &view);CHKERRQ(ierr);
6282     if (view) {ierr = PetscSectionView(*section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
6283   }
6284   PetscFunctionReturn(0);
6285 }
6286 
6287 #undef __FUNCT__
6288 #define __FUNCT__ "DMCreateCoordinateDM_Plex"
6289 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
6290 {
6291   PetscSection   section;
6292   PetscErrorCode ierr;
6293 
6294   PetscFunctionBegin;
6295   ierr = DMClone(dm, cdm);CHKERRQ(ierr);
6296   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
6297   ierr = DMSetDefaultSection(*cdm, section);CHKERRQ(ierr);
6298   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
6299   PetscFunctionReturn(0);
6300 }
6301 
6302 #undef __FUNCT__
6303 #define __FUNCT__ "DMPlexGetCoordinateSection"
6304 /*@
6305   DMPlexGetCoordinateSection - Retrieve the layout of coordinate values over the mesh.
6306 
6307   Not Collective
6308 
6309   Input Parameter:
6310 . dm - The DMPlex object
6311 
6312   Output Parameter:
6313 . section - The PetscSection object
6314 
6315   Level: intermediate
6316 
6317 .keywords: mesh, coordinates
6318 .seealso: DMGetCoordinateDM(), DMPlexGetDefaultSection(), DMPlexSetDefaultSection()
6319 @*/
6320 PetscErrorCode DMPlexGetCoordinateSection(DM dm, PetscSection *section)
6321 {
6322   DM             cdm;
6323   PetscErrorCode ierr;
6324 
6325   PetscFunctionBegin;
6326   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6327   PetscValidPointer(section, 2);
6328   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
6329   ierr = DMGetDefaultSection(cdm, section);CHKERRQ(ierr);
6330   PetscFunctionReturn(0);
6331 }
6332 
6333 #undef __FUNCT__
6334 #define __FUNCT__ "DMPlexSetCoordinateSection"
6335 /*@
6336   DMPlexSetCoordinateSection - Set the layout of coordinate values over the mesh.
6337 
6338   Not Collective
6339 
6340   Input Parameters:
6341 + dm      - The DMPlex object
6342 - section - The PetscSection object
6343 
6344   Level: intermediate
6345 
6346 .keywords: mesh, coordinates
6347 .seealso: DMPlexGetCoordinateSection(), DMPlexGetDefaultSection(), DMPlexSetDefaultSection()
6348 @*/
6349 PetscErrorCode DMPlexSetCoordinateSection(DM dm, PetscSection section)
6350 {
6351   DM             cdm;
6352   PetscErrorCode ierr;
6353 
6354   PetscFunctionBegin;
6355   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
6356   PetscValidHeaderSpecific(section,PETSC_SECTION_CLASSID,2);
6357   ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
6358   ierr = DMSetDefaultSection(cdm, section);CHKERRQ(ierr);
6359   PetscFunctionReturn(0);
6360 }
6361 
6362 #undef __FUNCT__
6363 #define __FUNCT__ "DMPlexGetConeSection"
6364 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
6365 {
6366   DM_Plex *mesh = (DM_Plex*) dm->data;
6367 
6368   PetscFunctionBegin;
6369   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6370   if (section) *section = mesh->coneSection;
6371   PetscFunctionReturn(0);
6372 }
6373 
6374 #undef __FUNCT__
6375 #define __FUNCT__ "DMPlexGetCones"
6376 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
6377 {
6378   DM_Plex *mesh = (DM_Plex*) dm->data;
6379 
6380   PetscFunctionBegin;
6381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6382   if (cones) *cones = mesh->cones;
6383   PetscFunctionReturn(0);
6384 }
6385 
6386 #undef __FUNCT__
6387 #define __FUNCT__ "DMPlexGetConeOrientations"
6388 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
6389 {
6390   DM_Plex *mesh = (DM_Plex*) dm->data;
6391 
6392   PetscFunctionBegin;
6393   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6394   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
6395   PetscFunctionReturn(0);
6396 }
6397 
6398 /******************************** FEM Support **********************************/
6399 
6400 #undef __FUNCT__
6401 #define __FUNCT__ "DMPlexVecGetClosure"
6402 /*@C
6403   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
6404 
6405   Not collective
6406 
6407   Input Parameters:
6408 + dm - The DM
6409 . section - The section describing the layout in v, or NULL to use the default section
6410 . v - The local vector
6411 - point - The sieve point in the DM
6412 
6413   Output Parameters:
6414 + csize - The number of values in the closure, or NULL
6415 - values - The array of values, which is a borrowed array and should not be freed
6416 
6417   Fortran Notes:
6418   Since it returns an array, this routine is only available in Fortran 90, and you must
6419   include petsc.h90 in your code.
6420 
6421   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
6422 
6423   Level: intermediate
6424 
6425 .seealso DMPlexVecRestoreClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
6426 @*/
6427 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6428 {
6429   PetscScalar   *array, *vArray;
6430   PetscInt      *points = NULL;
6431   PetscInt       offsets[32];
6432   PetscInt       depth, numFields, size = 0, numPoints, pStart, pEnd, p, q, f;
6433   PetscErrorCode ierr;
6434 
6435   PetscFunctionBegin;
6436   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6437   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6438   if (!section) {
6439     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
6440   }
6441   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6442   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6443   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6444   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6445   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
6446   if (depth == 1 && numFields < 2) {
6447     const PetscInt *cone, *coneO;
6448 
6449     ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6450     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6451     ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6452     if (!*values) {
6453       if ((point >= pStart) && (point < pEnd)) {
6454         PetscInt dof;
6455         ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6456         size += dof;
6457       }
6458       for (p = 0; p < numPoints; ++p) {
6459         const PetscInt cp = cone[p];
6460         PetscInt       dof;
6461 
6462         if ((cp < pStart) || (cp >= pEnd)) continue;
6463         ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6464         size += dof;
6465       }
6466       ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
6467     } else {
6468       array = *values;
6469     }
6470     size = 0;
6471     ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
6472     if ((point >= pStart) && (point < pEnd)) {
6473       PetscInt     dof, off, d;
6474       PetscScalar *varr;
6475       ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6476       ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6477       varr = &vArray[off];
6478       for (d = 0; d < dof; ++d, ++offsets[0]) {
6479         array[offsets[0]] = varr[d];
6480       }
6481       size += dof;
6482     }
6483     for (p = 0; p < numPoints; ++p) {
6484       const PetscInt cp = cone[p];
6485       PetscInt       o  = coneO[p];
6486       PetscInt       dof, off, d;
6487       PetscScalar   *varr;
6488 
6489       if ((cp < pStart) || (cp >= pEnd)) continue;
6490       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6491       ierr = PetscSectionGetOffset(section, cp, &off);CHKERRQ(ierr);
6492       varr = &vArray[off];
6493       if (o >= 0) {
6494         for (d = 0; d < dof; ++d, ++offsets[0]) {
6495           array[offsets[0]] = varr[d];
6496         }
6497       } else {
6498         for (d = dof-1; d >= 0; --d, ++offsets[0]) {
6499           array[offsets[0]] = varr[d];
6500         }
6501       }
6502       size += dof;
6503     }
6504     ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
6505     if (!*values) {
6506       if (csize) *csize = size;
6507       *values = array;
6508     } else {
6509       if (size > *csize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %d < actual size %d", *csize, size);
6510       *csize = size;
6511     }
6512     PetscFunctionReturn(0);
6513   }
6514   ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6515   /* Compress out points not in the section */
6516   for (p = 0, q = 0; p < numPoints*2; p += 2) {
6517     if ((points[p] >= pStart) && (points[p] < pEnd)) {
6518       points[q*2]   = points[p];
6519       points[q*2+1] = points[p+1];
6520       ++q;
6521     }
6522   }
6523   numPoints = q;
6524   for (p = 0, size = 0; p < numPoints*2; p += 2) {
6525     PetscInt dof, fdof;
6526 
6527     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6528     for (f = 0; f < numFields; ++f) {
6529       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6530       offsets[f+1] += fdof;
6531     }
6532     size += dof;
6533   }
6534   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6535   if (numFields && offsets[numFields] != size) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], size);
6536   ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &array);CHKERRQ(ierr);
6537   ierr = VecGetArray(v, &vArray);CHKERRQ(ierr);
6538   for (p = 0; p < numPoints*2; p += 2) {
6539     PetscInt     o = points[p+1];
6540     PetscInt     dof, off, d;
6541     PetscScalar *varr;
6542 
6543     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6544     ierr = PetscSectionGetOffset(section, points[p], &off);CHKERRQ(ierr);
6545     varr = &vArray[off];
6546     if (numFields) {
6547       PetscInt fdof, foff, fcomp, f, c;
6548 
6549       for (f = 0, foff = 0; f < numFields; ++f) {
6550         ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6551         if (o >= 0) {
6552           for (d = 0; d < fdof; ++d, ++offsets[f]) {
6553             array[offsets[f]] = varr[foff+d];
6554           }
6555         } else {
6556           ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
6557           for (d = fdof/fcomp-1; d >= 0; --d) {
6558             for (c = 0; c < fcomp; ++c, ++offsets[f]) {
6559               array[offsets[f]] = varr[foff+d*fcomp+c];
6560             }
6561           }
6562         }
6563         foff += fdof;
6564       }
6565     } else {
6566       if (o >= 0) {
6567         for (d = 0; d < dof; ++d, ++offsets[0]) {
6568           array[offsets[0]] = varr[d];
6569         }
6570       } else {
6571         for (d = dof-1; d >= 0; --d, ++offsets[0]) {
6572           array[offsets[0]] = varr[d];
6573         }
6574       }
6575     }
6576   }
6577   ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6578   ierr = VecRestoreArray(v, &vArray);CHKERRQ(ierr);
6579   if (csize) *csize = size;
6580   *values = array;
6581   PetscFunctionReturn(0);
6582 }
6583 
6584 #undef __FUNCT__
6585 #define __FUNCT__ "DMPlexVecRestoreClosure"
6586 /*@C
6587   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point'
6588 
6589   Not collective
6590 
6591   Input Parameters:
6592 + dm - The DM
6593 . section - The section describing the layout in v, or NULL to use the default section
6594 . v - The local vector
6595 . point - The sieve point in the DM
6596 . csize - The number of values in the closure, or NULL
6597 - values - The array of values, which is a borrowed array and should not be freed
6598 
6599   Fortran Notes:
6600   Since it returns an array, this routine is only available in Fortran 90, and you must
6601   include petsc.h90 in your code.
6602 
6603   The csize argument is not present in the Fortran 90 binding since it is internal to the array.
6604 
6605   Level: intermediate
6606 
6607 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure(), DMPlexMatSetClosure()
6608 @*/
6609 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6610 {
6611   PetscInt       size = 0;
6612   PetscErrorCode ierr;
6613 
6614   PetscFunctionBegin;
6615   /* Should work without recalculating size */
6616   ierr = DMRestoreWorkArray(dm, size, PETSC_SCALAR, (void*) values);CHKERRQ(ierr);
6617   PetscFunctionReturn(0);
6618 }
6619 
6620 PETSC_STATIC_INLINE void add   (PetscScalar *x, PetscScalar y) {*x += y;}
6621 PETSC_STATIC_INLINE void insert(PetscScalar *x, PetscScalar y) {*x  = y;}
6622 
6623 #undef __FUNCT__
6624 #define __FUNCT__ "updatePoint_private"
6625 PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
6626 {
6627   PetscInt        cdof;   /* The number of constraints on this point */
6628   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6629   PetscScalar    *a;
6630   PetscInt        off, cind = 0, k;
6631   PetscErrorCode  ierr;
6632 
6633   PetscFunctionBegin;
6634   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6635   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6636   a    = &array[off];
6637   if (!cdof || setBC) {
6638     if (orientation >= 0) {
6639       for (k = 0; k < dof; ++k) {
6640         fuse(&a[k], values[k]);
6641       }
6642     } else {
6643       for (k = 0; k < dof; ++k) {
6644         fuse(&a[k], values[dof-k-1]);
6645       }
6646     }
6647   } else {
6648     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6649     if (orientation >= 0) {
6650       for (k = 0; k < dof; ++k) {
6651         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6652         fuse(&a[k], values[k]);
6653       }
6654     } else {
6655       for (k = 0; k < dof; ++k) {
6656         if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6657         fuse(&a[k], values[dof-k-1]);
6658       }
6659     }
6660   }
6661   PetscFunctionReturn(0);
6662 }
6663 
6664 #undef __FUNCT__
6665 #define __FUNCT__ "updatePointFields_private"
6666 PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, PetscInt foffs[], void (*fuse)(PetscScalar*, PetscScalar), PetscBool setBC, PetscInt orientation, const PetscScalar values[], PetscScalar array[])
6667 {
6668   PetscScalar   *a;
6669   PetscInt       numFields, off, foff, f;
6670   PetscErrorCode ierr;
6671 
6672   PetscFunctionBegin;
6673   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6674   ierr = PetscSectionGetOffset(section, point, &off);CHKERRQ(ierr);
6675   a    = &array[off];
6676   for (f = 0, foff = 0; f < numFields; ++f) {
6677     PetscInt        fdof, fcomp, fcdof;
6678     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6679     PetscInt        cind = 0, k, c;
6680 
6681     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
6682     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6683     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &fcdof);CHKERRQ(ierr);
6684     if (!fcdof || setBC) {
6685       if (orientation >= 0) {
6686         for (k = 0; k < fdof; ++k) {
6687           fuse(&a[foff+k], values[foffs[f]+k]);
6688         }
6689       } else {
6690         for (k = fdof/fcomp-1; k >= 0; --k) {
6691           for (c = 0; c < fcomp; ++c) {
6692             fuse(&a[foff+(fdof/fcomp-1-k)*fcomp+c], values[foffs[f]+k*fcomp+c]);
6693           }
6694         }
6695       }
6696     } else {
6697       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6698       if (orientation >= 0) {
6699         for (k = 0; k < fdof; ++k) {
6700           if ((cind < fcdof) && (k == fcdofs[cind])) {++cind; continue;}
6701           fuse(&a[foff+k], values[foffs[f]+k]);
6702         }
6703       } else {
6704         for (k = fdof/fcomp-1; k >= 0; --k) {
6705           for (c = 0; c < fcomp; ++c) {
6706             if ((cind < fcdof) && (k*fcomp+c == fcdofs[cind])) {++cind; continue;}
6707             fuse(&a[foff+(fdof/fcomp-1-k)*fcomp+c], values[foffs[f]+k*fcomp+c]);
6708           }
6709         }
6710       }
6711     }
6712     foff     += fdof;
6713     foffs[f] += fdof;
6714   }
6715   PetscFunctionReturn(0);
6716 }
6717 
6718 #undef __FUNCT__
6719 #define __FUNCT__ "DMPlexVecSetClosure"
6720 /*@C
6721   DMPlexVecSetClosure - Set an array of the values on the closure of 'point'
6722 
6723   Not collective
6724 
6725   Input Parameters:
6726 + dm - The DM
6727 . section - The section describing the layout in v, or NULL to use the default section
6728 . v - The local vector
6729 . point - The sieve point in the DM
6730 . values - The array of values
6731 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
6732 
6733   Fortran Notes:
6734   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
6735 
6736   Level: intermediate
6737 
6738 .seealso DMPlexVecGetClosure(), DMPlexMatSetClosure()
6739 @*/
6740 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6741 {
6742   PetscScalar   *array;
6743   PetscInt      *points = NULL;
6744   PetscInt       offsets[32];
6745   PetscInt       depth, numFields, numPoints, off, dof, pStart, pEnd, p, q, f;
6746   PetscErrorCode ierr;
6747 
6748   PetscFunctionBegin;
6749   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6750   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6751   if (!section) {
6752     ierr = DMGetDefaultSection(dm, &section);CHKERRQ(ierr);
6753   }
6754   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
6755   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6756   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
6757   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
6758   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
6759   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
6760     const PetscInt *cone, *coneO;
6761 
6762     ierr = DMPlexGetConeSize(dm, point, &numPoints);CHKERRQ(ierr);
6763     ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
6764     ierr = DMPlexGetConeOrientation(dm, point, &coneO);CHKERRQ(ierr);
6765     ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6766     for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6767       const PetscInt cp = !p ? point : cone[p-1];
6768       const PetscInt o  = !p ? 0     : coneO[p-1];
6769 
6770       if ((cp < pStart) || (cp >= pEnd)) {dof = 0; continue;}
6771       ierr = PetscSectionGetDof(section, cp, &dof);CHKERRQ(ierr);
6772       /* ADD_VALUES */
6773       {
6774         const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6775         PetscScalar    *a;
6776         PetscInt        cdof, coff, cind = 0, k;
6777 
6778         ierr = PetscSectionGetConstraintDof(section, cp, &cdof);CHKERRQ(ierr);
6779         ierr = PetscSectionGetOffset(section, cp, &coff);CHKERRQ(ierr);
6780         a    = &array[coff];
6781         if (!cdof) {
6782           if (o >= 0) {
6783             for (k = 0; k < dof; ++k) {
6784               a[k] += values[off+k];
6785             }
6786           } else {
6787             for (k = 0; k < dof; ++k) {
6788               a[k] += values[off+dof-k-1];
6789             }
6790           }
6791         } else {
6792           ierr = PetscSectionGetConstraintIndices(section, cp, &cdofs);CHKERRQ(ierr);
6793           if (o >= 0) {
6794             for (k = 0; k < dof; ++k) {
6795               if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6796               a[k] += values[off+k];
6797             }
6798           } else {
6799             for (k = 0; k < dof; ++k) {
6800               if ((cind < cdof) && (k == cdofs[cind])) {++cind; continue;}
6801               a[k] += values[off+dof-k-1];
6802             }
6803           }
6804         }
6805       }
6806     }
6807     ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6808     PetscFunctionReturn(0);
6809   }
6810   ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6811   /* Compress out points not in the section */
6812   for (p = 0, q = 0; p < numPoints*2; p += 2) {
6813     if ((points[p] >= pStart) && (points[p] < pEnd)) {
6814       points[q*2]   = points[p];
6815       points[q*2+1] = points[p+1];
6816       ++q;
6817     }
6818   }
6819   numPoints = q;
6820   for (p = 0; p < numPoints*2; p += 2) {
6821     PetscInt fdof;
6822 
6823     for (f = 0; f < numFields; ++f) {
6824       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
6825       offsets[f+1] += fdof;
6826     }
6827   }
6828   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
6829   ierr = VecGetArray(v, &array);CHKERRQ(ierr);
6830   if (numFields) {
6831     switch (mode) {
6832     case INSERT_VALUES:
6833       for (p = 0; p < numPoints*2; p += 2) {
6834         PetscInt o = points[p+1];
6835         updatePointFields_private(section, points[p], offsets, insert, PETSC_FALSE, o, values, array);
6836       } break;
6837     case INSERT_ALL_VALUES:
6838       for (p = 0; p < numPoints*2; p += 2) {
6839         PetscInt o = points[p+1];
6840         updatePointFields_private(section, points[p], offsets, insert, PETSC_TRUE,  o, values, array);
6841       } break;
6842     case ADD_VALUES:
6843       for (p = 0; p < numPoints*2; p += 2) {
6844         PetscInt o = points[p+1];
6845         updatePointFields_private(section, points[p], offsets, add,    PETSC_FALSE, o, values, array);
6846       } break;
6847     case ADD_ALL_VALUES:
6848       for (p = 0; p < numPoints*2; p += 2) {
6849         PetscInt o = points[p+1];
6850         updatePointFields_private(section, points[p], offsets, add,    PETSC_TRUE,  o, values, array);
6851       } break;
6852     default:
6853       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
6854     }
6855   } else {
6856     switch (mode) {
6857     case INSERT_VALUES:
6858       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6859         PetscInt o = points[p+1];
6860         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6861         updatePoint_private(section, points[p], dof, insert, PETSC_FALSE, o, &values[off], array);
6862       } break;
6863     case INSERT_ALL_VALUES:
6864       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6865         PetscInt o = points[p+1];
6866         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6867         updatePoint_private(section, points[p], dof, insert, PETSC_TRUE,  o, &values[off], array);
6868       } break;
6869     case ADD_VALUES:
6870       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6871         PetscInt o = points[p+1];
6872         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6873         updatePoint_private(section, points[p], dof, add,    PETSC_FALSE, o, &values[off], array);
6874       } break;
6875     case ADD_ALL_VALUES:
6876       for (p = 0, off = 0; p < numPoints*2; p += 2, off += dof) {
6877         PetscInt o = points[p+1];
6878         ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
6879         updatePoint_private(section, points[p], dof, add,    PETSC_TRUE,  o, &values[off], array);
6880       } break;
6881     default:
6882       SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %D", mode);
6883     }
6884   }
6885   ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
6886   ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
6887   PetscFunctionReturn(0);
6888 }
6889 
6890 #undef __FUNCT__
6891 #define __FUNCT__ "DMPlexPrintMatSetValues"
6892 PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numIndices, const PetscInt indices[], const PetscScalar values[])
6893 {
6894   PetscMPIInt    rank;
6895   PetscInt       i, j;
6896   PetscErrorCode ierr;
6897 
6898   PetscFunctionBegin;
6899   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr);
6900   ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat for sieve point %D\n", rank, point);CHKERRQ(ierr);
6901   for (i = 0; i < numIndices; i++) {
6902     ierr = PetscViewerASCIIPrintf(viewer, "[%D]mat indices[%D] = %D\n", rank, i, indices[i]);CHKERRQ(ierr);
6903   }
6904   for (i = 0; i < numIndices; i++) {
6905     ierr = PetscViewerASCIIPrintf(viewer, "[%D]", rank);CHKERRQ(ierr);
6906     for (j = 0; j < numIndices; j++) {
6907 #if defined(PETSC_USE_COMPLEX)
6908       ierr = PetscViewerASCIIPrintf(viewer, " (%G,%G)", PetscRealPart(values[i*numIndices+j]), PetscImaginaryPart(values[i*numIndices+j]));CHKERRQ(ierr);
6909 #else
6910       ierr = PetscViewerASCIIPrintf(viewer, " %G", values[i*numIndices+j]);CHKERRQ(ierr);
6911 #endif
6912     }
6913     ierr = PetscViewerASCIIPrintf(viewer, "\n");CHKERRQ(ierr);
6914   }
6915   PetscFunctionReturn(0);
6916 }
6917 
6918 #undef __FUNCT__
6919 #define __FUNCT__ "indicesPoint_private"
6920 /* . off - The global offset of this point */
6921 PetscErrorCode indicesPoint_private(PetscSection section, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, PetscInt orientation, PetscInt indices[])
6922 {
6923   PetscInt        dof;    /* The number of unknowns on this point */
6924   PetscInt        cdof;   /* The number of constraints on this point */
6925   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6926   PetscInt        cind = 0, k;
6927   PetscErrorCode  ierr;
6928 
6929   PetscFunctionBegin;
6930   ierr = PetscSectionGetDof(section, point, &dof);CHKERRQ(ierr);
6931   ierr = PetscSectionGetConstraintDof(section, point, &cdof);CHKERRQ(ierr);
6932   if (!cdof || setBC) {
6933     if (orientation >= 0) {
6934       for (k = 0; k < dof; ++k) indices[*loff+k] = off+k;
6935     } else {
6936       for (k = 0; k < dof; ++k) indices[*loff+dof-k-1] = off+k;
6937     }
6938   } else {
6939     ierr = PetscSectionGetConstraintIndices(section, point, &cdofs);CHKERRQ(ierr);
6940     if (orientation >= 0) {
6941       for (k = 0; k < dof; ++k) {
6942         if ((cind < cdof) && (k == cdofs[cind])) {
6943           /* Insert check for returning constrained indices */
6944           indices[*loff+k] = -(off+k+1);
6945           ++cind;
6946         } else {
6947           indices[*loff+k] = off+k-cind;
6948         }
6949       }
6950     } else {
6951       for (k = 0; k < dof; ++k) {
6952         if ((cind < cdof) && (k == cdofs[cind])) {
6953           /* Insert check for returning constrained indices */
6954           indices[*loff+dof-k-1] = -(off+k+1);
6955           ++cind;
6956         } else {
6957           indices[*loff+dof-k-1] = off+k-cind;
6958         }
6959       }
6960     }
6961   }
6962   *loff += dof;
6963   PetscFunctionReturn(0);
6964 }
6965 
6966 #undef __FUNCT__
6967 #define __FUNCT__ "indicesPointFields_private"
6968 /* . off - The global offset of this point */
6969 PetscErrorCode indicesPointFields_private(PetscSection section, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, PetscInt orientation, PetscInt indices[])
6970 {
6971   PetscInt       numFields, foff, f;
6972   PetscErrorCode ierr;
6973 
6974   PetscFunctionBegin;
6975   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
6976   for (f = 0, foff = 0; f < numFields; ++f) {
6977     PetscInt        fdof, fcomp, cfdof;
6978     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6979     PetscInt        cind = 0, k, c;
6980 
6981     ierr = PetscSectionGetFieldComponents(section, f, &fcomp);CHKERRQ(ierr);
6982     ierr = PetscSectionGetFieldDof(section, point, f, &fdof);CHKERRQ(ierr);
6983     ierr = PetscSectionGetFieldConstraintDof(section, point, f, &cfdof);CHKERRQ(ierr);
6984     if (!cfdof || setBC) {
6985       if (orientation >= 0) {
6986         for (k = 0; k < fdof; ++k) indices[foffs[f]+k] = off+foff+k;
6987       } else {
6988         for (k = fdof/fcomp-1; k >= 0; --k) {
6989           for (c = 0; c < fcomp; ++c) {
6990             indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c;
6991           }
6992         }
6993       }
6994     } else {
6995       ierr = PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs);CHKERRQ(ierr);
6996       if (orientation >= 0) {
6997         for (k = 0; k < fdof; ++k) {
6998           if ((cind < cfdof) && (k == fcdofs[cind])) {
6999             indices[foffs[f]+k] = -(off+foff+k+1);
7000             ++cind;
7001           } else {
7002             indices[foffs[f]+k] = off+foff+k-cind;
7003           }
7004         }
7005       } else {
7006         for (k = fdof/fcomp-1; k >= 0; --k) {
7007           for (c = 0; c < fcomp; ++c) {
7008             if ((cind < cfdof) && ((fdof/fcomp-1-k)*fcomp+c == fcdofs[cind])) {
7009               indices[foffs[f]+k*fcomp+c] = -(off+foff+(fdof/fcomp-1-k)*fcomp+c+1);
7010               ++cind;
7011             } else {
7012               indices[foffs[f]+k*fcomp+c] = off+foff+(fdof/fcomp-1-k)*fcomp+c-cind;
7013             }
7014           }
7015         }
7016       }
7017     }
7018     foff     += fdof - cfdof;
7019     foffs[f] += fdof;
7020   }
7021   PetscFunctionReturn(0);
7022 }
7023 
7024 #undef __FUNCT__
7025 #define __FUNCT__ "DMPlexMatSetClosure"
7026 /*@C
7027   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
7028 
7029   Not collective
7030 
7031   Input Parameters:
7032 + dm - The DM
7033 . section - The section describing the layout in v
7034 . globalSection - The section describing the layout in v
7035 . A - The matrix
7036 . point - The sieve point in the DM
7037 . values - The array of values
7038 - mode - The insert mode, where INSERT_ALL_VALUES and ADD_ALL_VALUES also overwrite boundary conditions
7039 
7040   Fortran Notes:
7041   This routine is only available in Fortran 90, and you must include petsc.h90 in your code.
7042 
7043   Level: intermediate
7044 
7045 .seealso DMPlexVecGetClosure(), DMPlexVecSetClosure()
7046 @*/
7047 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
7048 {
7049   DM_Plex       *mesh   = (DM_Plex*) dm->data;
7050   PetscInt      *points = NULL;
7051   PetscInt      *indices;
7052   PetscInt       offsets[32];
7053   PetscInt       numFields, numPoints, numIndices, dof, off, globalOff, pStart, pEnd, p, q, f;
7054   PetscErrorCode ierr;
7055 
7056   PetscFunctionBegin;
7057   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7058   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7059   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
7060   PetscValidHeaderSpecific(A, MAT_CLASSID, 4);
7061   ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
7062   if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
7063   ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
7064   ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
7065   /* Compress out points not in the section */
7066   ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
7067   for (p = 0, q = 0; p < numPoints*2; p += 2) {
7068     if ((points[p] >= pStart) && (points[p] < pEnd)) {
7069       points[q*2]   = points[p];
7070       points[q*2+1] = points[p+1];
7071       ++q;
7072     }
7073   }
7074   numPoints = q;
7075   for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
7076     PetscInt fdof;
7077 
7078     ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
7079     for (f = 0; f < numFields; ++f) {
7080       ierr          = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
7081       offsets[f+1] += fdof;
7082     }
7083     numIndices += dof;
7084   }
7085   for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
7086 
7087   if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
7088   ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
7089   if (numFields) {
7090     for (p = 0; p < numPoints*2; p += 2) {
7091       PetscInt o = points[p+1];
7092       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
7093       indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
7094     }
7095   } else {
7096     for (p = 0, off = 0; p < numPoints*2; p += 2) {
7097       PetscInt o = points[p+1];
7098       ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
7099       indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
7100     }
7101   }
7102   if (mesh->printSetValues) {ierr = DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, values);CHKERRQ(ierr);}
7103   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
7104   if (ierr) {
7105     PetscMPIInt    rank;
7106     PetscErrorCode ierr2;
7107 
7108     ierr2 = MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank);CHKERRQ(ierr2);
7109     ierr2 = (*PetscErrorPrintf)("[%D]ERROR in DMPlexMatSetClosure\n", rank);CHKERRQ(ierr2);
7110     ierr2 = DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, values);CHKERRQ(ierr2);
7111     ierr2 = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr2);
7112     CHKERRQ(ierr);
7113   }
7114   ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
7115   ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
7116   PetscFunctionReturn(0);
7117 }
7118 
7119 #undef __FUNCT__
7120 #define __FUNCT__ "DMPlexGetHybridBounds"
7121 PetscErrorCode DMPlexGetHybridBounds(DM dm, PetscInt *cMax, PetscInt *fMax, PetscInt *eMax, PetscInt *vMax)
7122 {
7123   DM_Plex       *mesh = (DM_Plex*) dm->data;
7124   PetscInt       dim;
7125   PetscErrorCode ierr;
7126 
7127   PetscFunctionBegin;
7128   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7129   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
7130   if (cMax) *cMax = mesh->hybridPointMax[dim];
7131   if (fMax) *fMax = mesh->hybridPointMax[dim-1];
7132   if (eMax) *eMax = mesh->hybridPointMax[1];
7133   if (vMax) *vMax = mesh->hybridPointMax[0];
7134   PetscFunctionReturn(0);
7135 }
7136 
7137 #undef __FUNCT__
7138 #define __FUNCT__ "DMPlexSetHybridBounds"
7139 PetscErrorCode DMPlexSetHybridBounds(DM dm, PetscInt cMax, PetscInt fMax, PetscInt eMax, PetscInt vMax)
7140 {
7141   DM_Plex       *mesh = (DM_Plex*) dm->data;
7142   PetscInt       dim;
7143   PetscErrorCode ierr;
7144 
7145   PetscFunctionBegin;
7146   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7147   ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
7148   if (cMax >= 0) mesh->hybridPointMax[dim]   = cMax;
7149   if (fMax >= 0) mesh->hybridPointMax[dim-1] = fMax;
7150   if (eMax >= 0) mesh->hybridPointMax[1]     = eMax;
7151   if (vMax >= 0) mesh->hybridPointMax[0]     = vMax;
7152   PetscFunctionReturn(0);
7153 }
7154 
7155 #undef __FUNCT__
7156 #define __FUNCT__ "DMPlexGetVTKCellHeight"
7157 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
7158 {
7159   DM_Plex *mesh = (DM_Plex*) dm->data;
7160 
7161   PetscFunctionBegin;
7162   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7163   PetscValidPointer(cellHeight, 2);
7164   *cellHeight = mesh->vtkCellHeight;
7165   PetscFunctionReturn(0);
7166 }
7167 
7168 #undef __FUNCT__
7169 #define __FUNCT__ "DMPlexSetVTKCellHeight"
7170 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
7171 {
7172   DM_Plex *mesh = (DM_Plex*) dm->data;
7173 
7174   PetscFunctionBegin;
7175   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7176   mesh->vtkCellHeight = cellHeight;
7177   PetscFunctionReturn(0);
7178 }
7179 
7180 #undef __FUNCT__
7181 #define __FUNCT__ "DMPlexCreateNumbering_Private"
7182 /* We can easily have a form that takes an IS instead */
7183 PetscErrorCode DMPlexCreateNumbering_Private(DM dm, PetscInt pStart, PetscInt pEnd, PetscSF sf, IS *numbering)
7184 {
7185   PetscSection   section, globalSection;
7186   PetscInt      *numbers, p;
7187   PetscErrorCode ierr;
7188 
7189   PetscFunctionBegin;
7190   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section);CHKERRQ(ierr);
7191   ierr = PetscSectionSetChart(section, pStart, pEnd);CHKERRQ(ierr);
7192   for (p = pStart; p < pEnd; ++p) {
7193     ierr = PetscSectionSetDof(section, p, 1);CHKERRQ(ierr);
7194   }
7195   ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
7196   ierr = PetscSectionCreateGlobalSection(section, sf, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
7197   ierr = PetscMalloc((pEnd - pStart) * sizeof(PetscInt), &numbers);CHKERRQ(ierr);
7198   for (p = pStart; p < pEnd; ++p) {
7199     ierr = PetscSectionGetOffset(globalSection, p, &numbers[p-pStart]);CHKERRQ(ierr);
7200   }
7201   ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering);CHKERRQ(ierr);
7202   ierr = PetscSectionDestroy(&section);CHKERRQ(ierr);
7203   ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
7204   PetscFunctionReturn(0);
7205 }
7206 
7207 #undef __FUNCT__
7208 #define __FUNCT__ "DMPlexGetCellNumbering"
7209 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
7210 {
7211   DM_Plex       *mesh = (DM_Plex*) dm->data;
7212   PetscInt       cellHeight, cStart, cEnd, cMax;
7213   PetscErrorCode ierr;
7214 
7215   PetscFunctionBegin;
7216   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7217   if (!mesh->globalCellNumbers) {
7218     ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
7219     ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
7220     ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
7221     if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
7222     ierr = DMPlexCreateNumbering_Private(dm, cStart, cEnd, dm->sf, &mesh->globalCellNumbers);CHKERRQ(ierr);
7223   }
7224   *globalCellNumbers = mesh->globalCellNumbers;
7225   PetscFunctionReturn(0);
7226 }
7227 
7228 #undef __FUNCT__
7229 #define __FUNCT__ "DMPlexGetVertexNumbering"
7230 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
7231 {
7232   DM_Plex       *mesh = (DM_Plex*) dm->data;
7233   PetscInt       vStart, vEnd, vMax;
7234   PetscErrorCode ierr;
7235 
7236   PetscFunctionBegin;
7237   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7238   if (!mesh->globalVertexNumbers) {
7239     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
7240     ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
7241     if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
7242     ierr = DMPlexCreateNumbering_Private(dm, vStart, vEnd, dm->sf, &mesh->globalVertexNumbers);CHKERRQ(ierr);
7243   }
7244   *globalVertexNumbers = mesh->globalVertexNumbers;
7245   PetscFunctionReturn(0);
7246 }
7247 
7248 
7249 #undef __FUNCT__
7250 #define __FUNCT__ "PetscSectionCreateGlobalSectionLabel"
7251 /*@C
7252   PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
7253   the local section and an SF describing the section point overlap.
7254 
7255   Input Parameters:
7256   + s - The PetscSection for the local field layout
7257   . sf - The SF describing parallel layout of the section points
7258   . includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
7259   . label - The label specifying the points
7260   - labelValue - The label stratum specifying the points
7261 
7262   Output Parameter:
7263   . gsection - The PetscSection for the global field layout
7264 
7265   Note: This gives negative sizes and offsets to points not owned by this process
7266 
7267   Level: developer
7268 
7269 .seealso: PetscSectionCreate()
7270 @*/
7271 PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
7272 {
7273   PetscInt      *neg = NULL, *tmpOff = NULL;
7274   PetscInt       pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
7275   PetscErrorCode ierr;
7276 
7277   PetscFunctionBegin;
7278   ierr = PetscSectionCreate(s->atlasLayout.comm, gsection);CHKERRQ(ierr);
7279   ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
7280   ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
7281   ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
7282   if (nroots >= 0) {
7283     if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
7284     ierr = PetscMalloc(nroots * sizeof(PetscInt), &neg);CHKERRQ(ierr);
7285     ierr = PetscMemzero(neg, nroots * sizeof(PetscInt));CHKERRQ(ierr);
7286     if (nroots > pEnd-pStart) {
7287       ierr = PetscMalloc(nroots * sizeof(PetscInt), &tmpOff);CHKERRQ(ierr);
7288       ierr = PetscMemzero(tmpOff, nroots * sizeof(PetscInt));CHKERRQ(ierr);
7289     } else {
7290       tmpOff = &(*gsection)->atlasDof[-pStart];
7291     }
7292   }
7293   /* Mark ghost points with negative dof */
7294   for (p = pStart; p < pEnd; ++p) {
7295     PetscInt value;
7296 
7297     ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
7298     if (value != labelValue) continue;
7299     ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
7300     ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
7301     ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
7302     if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
7303     if (neg) neg[p] = -(dof+1);
7304   }
7305   ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
7306   if (nroots >= 0) {
7307     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7308     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7309     if (nroots > pEnd-pStart) {
7310       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
7311     }
7312   }
7313   /* Calculate new sizes, get proccess offset, and calculate point offsets */
7314   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
7315     cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
7316     (*gsection)->atlasOff[p] = off;
7317     off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
7318   }
7319   ierr       = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, s->atlasLayout.comm);CHKERRQ(ierr);
7320   globalOff -= off;
7321   for (p = 0, off = 0; p < pEnd-pStart; ++p) {
7322     (*gsection)->atlasOff[p] += globalOff;
7323     if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
7324   }
7325   /* Put in negative offsets for ghost points */
7326   if (nroots >= 0) {
7327     ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7328     ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
7329     if (nroots > pEnd-pStart) {
7330       for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
7331     }
7332   }
7333   if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
7334   ierr = PetscFree(neg);CHKERRQ(ierr);
7335   PetscFunctionReturn(0);
7336 }
7337