1 static char help[] = "Partition a mesh in parallel, perhaps with overlap\n\n";
2
3 #include <petscdmplex.h>
4 #include <petscsf.h>
5
6 /* Sample usage:
7
8 Load a file in serial and distribute it on 24 processes:
9
10 make -f ./gmakefile test search="dm_impls_plex_tests-ex12_0" EXTRA_OPTIONS="-filename $PETSC_DIR/share/petsc/datafiles/meshes/squaremotor-30.exo -orig_dm_view -dm_view" NP=24
11
12 Load a file in serial, distribute it, and then redistribute it on 24 processes using two different partitioners:
13
14 make -f ./gmakefile test search="dm_impls_plex_tests-ex12_0" EXTRA_OPTIONS="-filename $PETSC_DIR/share/petsc/datafiles/meshes/squaremotor-30.exo -petscpartitioner_type simple -load_balance -lb_petscpartitioner_type parmetis -orig_dm_view -dm_view" NP=24
15
16 Load a file in serial, distribute it randomly, refine it in parallel, and then redistribute it on 24 processes using two different partitioners, and view to VTK:
17
18 make -f ./gmakefile test search="dm_impls_plex_tests-ex12_0" EXTRA_OPTIONS="-filename $PETSC_DIR/share/petsc/datafiles/meshes/squaremotor-30.exo -petscpartitioner_type shell -petscpartitioner_shell_random -dm_refine 1 -load_balance -lb_petscpartitioner_type parmetis -prelb_dm_view vtk:$PWD/prelb.vtk -dm_view vtk:$PWD/balance.vtk -dm_partition_view" NP=24
19
20 */
21
22 enum {
23 STAGE_LOAD,
24 STAGE_DISTRIBUTE,
25 STAGE_REFINE,
26 STAGE_REDISTRIBUTE
27 };
28
29 typedef struct {
30 /* Domain and mesh definition */
31 PetscInt overlap; /* The cell overlap to use during partitioning */
32 PetscBool testSection; /* Use a PetscSection to specify cell weights */
33 PetscBool testPartition; /* Use a fixed partitioning for testing */
34 PetscBool testRedundant; /* Use a redundant partitioning for testing */
35 PetscBool loadBalance; /* Load balance via a second distribute step */
36 PetscBool partitionBalance; /* Balance shared point partition */
37 PetscLogStage stages[4];
38 } AppCtx;
39
ProcessOptions(MPI_Comm comm,AppCtx * options)40 PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options)
41 {
42 PetscFunctionBegin;
43 options->overlap = 0;
44 options->testPartition = PETSC_FALSE;
45 options->testSection = PETSC_FALSE;
46 options->testRedundant = PETSC_FALSE;
47 options->loadBalance = PETSC_FALSE;
48 options->partitionBalance = PETSC_FALSE;
49
50 PetscOptionsBegin(comm, "", "Meshing Problem Options", "DMPLEX");
51 PetscCall(PetscOptionsBoundedInt("-overlap", "The cell overlap for partitioning", "ex12.c", options->overlap, &options->overlap, NULL, 0));
52 PetscCall(PetscOptionsBool("-test_partition", "Use a fixed partition for testing", "ex12.c", options->testPartition, &options->testPartition, NULL));
53 PetscCall(PetscOptionsBool("-test_section", "Use a PetscSection for cell weights", "ex12.c", options->testSection, &options->testSection, NULL));
54 PetscCall(PetscOptionsBool("-test_redundant", "Use a redundant partition for testing", "ex12.c", options->testRedundant, &options->testRedundant, NULL));
55 PetscCall(PetscOptionsBool("-load_balance", "Perform parallel load balancing in a second distribution step", "ex12.c", options->loadBalance, &options->loadBalance, NULL));
56 PetscCall(PetscOptionsBool("-partition_balance", "Balance the ownership of shared points", "ex12.c", options->partitionBalance, &options->partitionBalance, NULL));
57 PetscOptionsEnd();
58
59 PetscCall(PetscLogStageRegister("MeshLoad", &options->stages[STAGE_LOAD]));
60 PetscCall(PetscLogStageRegister("MeshDistribute", &options->stages[STAGE_DISTRIBUTE]));
61 PetscCall(PetscLogStageRegister("MeshRefine", &options->stages[STAGE_REFINE]));
62 PetscCall(PetscLogStageRegister("MeshRedistribute", &options->stages[STAGE_REDISTRIBUTE]));
63 PetscFunctionReturn(PETSC_SUCCESS);
64 }
65
CreateMesh(MPI_Comm comm,AppCtx * user,DM * dm)66 PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
67 {
68 DM pdm = NULL;
69 PetscInt triSizes_n2[2] = {4, 4};
70 PetscInt triPoints_n2[8] = {0, 1, 4, 6, 2, 3, 5, 7};
71 PetscInt triSizes_n3[3] = {3, 2, 3};
72 PetscInt triPoints_n3[8] = {3, 5, 6, 1, 7, 0, 2, 4};
73 PetscInt triSizes_n4[4] = {2, 2, 2, 2};
74 PetscInt triPoints_n4[8] = {0, 7, 1, 5, 2, 3, 4, 6};
75 PetscInt triSizes_n8[8] = {1, 1, 1, 1, 1, 1, 1, 1};
76 PetscInt triPoints_n8[8] = {0, 1, 2, 3, 4, 5, 6, 7};
77 PetscInt quadSizes[2] = {2, 2};
78 PetscInt quadPoints[4] = {2, 3, 0, 1};
79 PetscInt overlap = user->overlap >= 0 ? user->overlap : 0;
80 PetscInt dim;
81 PetscBool simplex;
82 PetscMPIInt rank, size;
83
84 PetscFunctionBegin;
85 PetscCallMPI(MPI_Comm_rank(comm, &rank));
86 PetscCallMPI(MPI_Comm_size(comm, &size));
87 PetscCall(PetscLogStagePush(user->stages[STAGE_LOAD]));
88 PetscCall(DMCreate(comm, dm));
89 PetscCall(DMSetType(*dm, DMPLEX));
90 PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
91 PetscCall(DMSetFromOptions(*dm));
92 PetscCall(DMGetDimension(*dm, &dim));
93 if (user->testSection) {
94 PetscInt numComp[] = {1};
95 PetscInt numDof[] = {0, 0, 0, 1};
96 PetscSection section;
97
98 PetscCall(DMSetNumFields(*dm, 1));
99 PetscCall(DMPlexCreateSection(*dm, NULL, numComp, numDof + 3 - dim, 0, NULL, NULL, NULL, NULL, §ion));
100 PetscCall(DMSetLocalSection(*dm, section));
101 PetscCall(PetscSectionViewFromOptions(section, NULL, "-cell_section_view"));
102 PetscCall(PetscSectionDestroy(§ion));
103 }
104 PetscCall(DMViewFromOptions(*dm, NULL, "-orig_dm_view"));
105 PetscCall(PetscLogStagePop());
106 PetscCall(DMPlexIsSimplex(*dm, &simplex));
107 PetscCall(PetscLogStagePush(user->stages[STAGE_DISTRIBUTE]));
108 if (!user->testRedundant) {
109 PetscPartitioner part;
110
111 PetscCall(DMPlexGetPartitioner(*dm, &part));
112 PetscCall(PetscPartitionerSetFromOptions(part));
113 PetscCall(PetscPartitionerViewFromOptions(part, NULL, "-view_partitioner_pre"));
114 PetscCall(DMPlexSetPartitionBalance(*dm, user->partitionBalance));
115 if (user->testPartition) {
116 const PetscInt *sizes = NULL;
117 const PetscInt *points = NULL;
118
119 if (rank == 0) {
120 if (dim == 2 && simplex && size == 2) {
121 sizes = triSizes_n2;
122 points = triPoints_n2;
123 } else if (dim == 2 && simplex && size == 3) {
124 sizes = triSizes_n3;
125 points = triPoints_n3;
126 } else if (dim == 2 && simplex && size == 4) {
127 sizes = triSizes_n4;
128 points = triPoints_n4;
129 } else if (dim == 2 && simplex && size == 8) {
130 sizes = triSizes_n8;
131 points = triPoints_n8;
132 } else if (dim == 2 && !simplex && size == 2) {
133 sizes = quadSizes;
134 points = quadPoints;
135 }
136 }
137 PetscCall(PetscPartitionerSetType(part, PETSCPARTITIONERSHELL));
138 PetscCall(PetscPartitionerShellSetPartition(part, size, sizes, points));
139 }
140 PetscCall(DMPlexDistribute(*dm, overlap, NULL, &pdm));
141 PetscCall(PetscPartitionerViewFromOptions(part, NULL, "-view_partitioner"));
142 } else {
143 PetscSF sf;
144
145 PetscCall(DMPlexGetRedundantDM(*dm, &sf, &pdm));
146 if (sf) {
147 DM test;
148
149 PetscCall(DMPlexCreate(comm, &test));
150 PetscCall(PetscObjectSetName((PetscObject)test, "Test SF-migrated Redundant Mesh"));
151 PetscCall(DMPlexMigrate(*dm, sf, test));
152 PetscCall(DMViewFromOptions(test, NULL, "-redundant_migrated_dm_view"));
153 PetscCall(DMDestroy(&test));
154 }
155 PetscCall(PetscSFDestroy(&sf));
156 }
157 if (pdm) {
158 PetscCall(DMDestroy(dm));
159 *dm = pdm;
160 }
161 PetscCall(PetscLogStagePop());
162 PetscCall(DMSetFromOptions(*dm));
163 if (user->loadBalance) {
164 PetscPartitioner part;
165
166 PetscCall(DMViewFromOptions(*dm, NULL, "-prelb_dm_view"));
167 PetscCall(DMPlexSetOptionsPrefix(*dm, "lb_"));
168 PetscCall(PetscLogStagePush(user->stages[STAGE_REDISTRIBUTE]));
169 PetscCall(DMPlexGetPartitioner(*dm, &part));
170 PetscCall(PetscPartitionerViewFromOptions(part, NULL, "-view_partitioner_pre"));
171 PetscCall(PetscObjectSetOptionsPrefix((PetscObject)part, "lb_"));
172 PetscCall(PetscPartitionerSetFromOptions(part));
173 if (user->testPartition) {
174 PetscInt reSizes_n2[2] = {2, 2};
175 PetscInt rePoints_n2[4] = {2, 3, 0, 1};
176 if (rank) {
177 rePoints_n2[0] = 1;
178 rePoints_n2[1] = 2, rePoints_n2[2] = 0, rePoints_n2[3] = 3;
179 }
180
181 PetscCall(PetscPartitionerSetType(part, PETSCPARTITIONERSHELL));
182 PetscCall(PetscPartitionerShellSetPartition(part, size, reSizes_n2, rePoints_n2));
183 }
184 PetscCall(DMPlexSetPartitionBalance(*dm, user->partitionBalance));
185 PetscCall(DMPlexDistribute(*dm, overlap, NULL, &pdm));
186 PetscCall(PetscPartitionerViewFromOptions(part, NULL, "-view_partitioner"));
187 if (pdm) {
188 PetscCall(DMDestroy(dm));
189 *dm = pdm;
190 }
191 PetscCall(PetscLogStagePop());
192 }
193 PetscCall(PetscLogStagePush(user->stages[STAGE_REFINE]));
194 PetscCall(DMViewFromOptions(*dm, NULL, "-dm_view"));
195 PetscCall(PetscLogStagePop());
196 PetscFunctionReturn(PETSC_SUCCESS);
197 }
198
main(int argc,char ** argv)199 int main(int argc, char **argv)
200 {
201 DM dm;
202 AppCtx user; /* user-defined work context */
203
204 PetscFunctionBeginUser;
205 PetscCall(PetscInitialize(&argc, &argv, NULL, help));
206 PetscCall(ProcessOptions(PETSC_COMM_WORLD, &user));
207 PetscCall(CreateMesh(PETSC_COMM_WORLD, &user, &dm));
208 PetscCall(DMDestroy(&dm));
209 PetscCall(PetscFinalize());
210 return 0;
211 }
212
213 /*TEST
214 # Parallel, no overlap tests 0-2
215 test:
216 suffix: 0
217 requires: triangle
218 args: -dm_coord_space 0 -dm_view ascii:mesh.tex:ascii_latex -petscpartitioner_type {{simple multistage}}
219 output_file: output/empty.out
220 test:
221 suffix: 1
222 requires: triangle
223 nsize: 3
224 args: -dm_coord_space 0 -test_partition -dm_view ascii::ascii_info_detail
225 test:
226 suffix: 1_ms_default
227 requires: triangle
228 nsize: 3
229 args: -dm_coord_space 0 -petscpartitioner_type multistage -petscpartitioner_multistage_levels_petscpartitioner_type simple
230 output_file: output/empty.out
231 test:
232 suffix: 1_ms_cellsection
233 requires: triangle
234 nsize: 3
235 args: -dm_coord_space 0 -test_section -dm_view ascii::ascii_info_detail -petscpartitioner_type multistage -petscpartitioner_multistage_levels_petscpartitioner_type simple -petscpartitioner_multistage_node_size 2 -view_partitioner ::ascii_info_detail
236 test:
237 suffix: 2
238 requires: triangle
239 nsize: 8
240 args: -dm_coord_space 0 -test_partition -dm_view ascii::ascii_info_detail
241 test:
242 suffix: 2_ms
243 requires: triangle
244 nsize: 8
245 args: -dm_coord_space 0 -dm_view ascii::ascii_info_detail \
246 -petscpartitioner_type multistage \
247 -petscpartitioner_multistage_strategy node -petscpartitioner_multistage_node_size {{1 2 3 4}separate output} -petscpartitioner_multistage_node_interleaved {{0 1}separate output} \
248 -petscpartitioner_multistage_levels_petscpartitioner_type simple -petscpartitioner_view ascii::ascii_info_detail
249 # Parallel, level-1 overlap tests 3-4
250 test:
251 suffix: 3
252 requires: triangle
253 nsize: 3
254 args: -dm_coord_space 0 -test_partition -overlap 1 -dm_view ascii::ascii_info_detail
255 test:
256 suffix: 4
257 requires: triangle
258 nsize: 8
259 args: -dm_coord_space 0 -test_partition -overlap 1 -dm_view ascii::ascii_info_detail
260 # Parallel, level-2 overlap test 5
261 test:
262 suffix: 5
263 requires: triangle
264 nsize: 8
265 args: -dm_coord_space 0 -test_partition -overlap 2 -dm_view ascii::ascii_info_detail
266 test:
267 suffix: 5_ms
268 requires: triangle
269 nsize: 8
270 args: -dm_coord_space 0 -overlap 2 -dm_view ascii::ascii_info_detail \
271 -petscpartitioner_type multistage \
272 -petscpartitioner_multistage_strategy msection -petscpartitioner_multistage_msection {{2 3}separate output} \
273 -petscpartitioner_multistage_levels_petscpartitioner_type simple -petscpartitioner_view ascii::ascii_info_detail
274 # Parallel load balancing, test 6-7
275 test:
276 suffix: 6
277 requires: triangle
278 nsize: 2
279 args: -dm_coord_space 0 -test_partition -overlap 1 -dm_view ascii::ascii_info_detail
280 test:
281 suffix: 7
282 requires: triangle
283 nsize: 2
284 args: -dm_coord_space 0 -test_partition -overlap 1 -load_balance -dm_view ascii::ascii_info_detail
285 # Parallel redundant copying, test 8
286 test:
287 suffix: 8
288 requires: triangle
289 nsize: 2
290 args: -dm_coord_space 0 -test_redundant -redundant_migrated_dm_view ascii::ascii_info_detail -dm_view ascii::ascii_info_detail
291 test:
292 suffix: lb_0
293 requires: parmetis
294 nsize: 4
295 args: -dm_coord_space 0 -dm_plex_simplex 0 -dm_plex_box_faces 4,4 -petscpartitioner_type shell -petscpartitioner_shell_random -lb_petscpartitioner_type parmetis -load_balance -lb_petscpartitioner_view -prelb_dm_view ::load_balance -dm_view ::load_balance
296 test:
297 suffix: lb_0_ms
298 requires: parmetis
299 nsize: 4
300 args: -dm_coord_space 0 -dm_plex_simplex 0 -dm_plex_box_faces 4,4 -petscpartitioner_type shell -petscpartitioner_shell_random -lb_petscpartitioner_type multistage -load_balance -lb_petscpartitioner_view ::ascii_info_detail -prelb_dm_view ::load_balance -dm_view ::load_balance -lb_petscpartitioner_multistage_levels_petscpartitioner_type parmetis -lb_petscpartitioner_multistage_node_size 2
301
302 # Same tests as above, but with balancing of the shared point partition
303 test:
304 suffix: 9
305 requires: triangle
306 args: -dm_coord_space 0 -dm_view ascii:mesh.tex:ascii_latex -partition_balance -petscpartitioner_type {{simple multistage}}
307 output_file: output/empty.out
308 test:
309 suffix: 10
310 requires: triangle
311 nsize: 3
312 args: -dm_coord_space 0 -test_partition -dm_view ascii::ascii_info_detail -partition_balance
313 test:
314 suffix: 11
315 requires: triangle
316 nsize: 8
317 args: -dm_coord_space 0 -test_partition -dm_view ascii::ascii_info_detail -partition_balance
318 test:
319 suffix: 11_ms
320 requires: triangle
321 nsize: 8
322 args: -dm_coord_space 0 -dm_view ascii::ascii_info_detail -partition_balance \
323 -petscpartitioner_type multistage \
324 -petscpartitioner_multistage_strategy node -petscpartitioner_multistage_node_size {{1 2 3 4}separate output} -petscpartitioner_multistage_node_interleaved {{0 1}separate output} \
325 -petscpartitioner_multistage_levels_petscpartitioner_type simple -petscpartitioner_view ascii::ascii_info_detail
326 # Parallel, level-1 overlap tests 3-4
327 test:
328 suffix: 12
329 requires: triangle
330 nsize: 3
331 args: -dm_coord_space 0 -test_partition -overlap 1 -dm_view ascii::ascii_info_detail -partition_balance
332 test:
333 suffix: 13
334 requires: triangle
335 nsize: 8
336 args: -dm_coord_space 0 -test_partition -overlap 1 -dm_view ascii::ascii_info_detail -partition_balance
337 # Parallel, level-2 overlap test 5
338 test:
339 suffix: 14
340 requires: triangle
341 nsize: 8
342 args: -dm_coord_space 0 -test_partition -overlap 2 -dm_view ascii::ascii_info_detail -partition_balance
343 # Parallel load balancing, test 6-7
344 test:
345 suffix: 15
346 requires: triangle
347 nsize: 2
348 args: -dm_coord_space 0 -test_partition -overlap 1 -dm_view ascii::ascii_info_detail -partition_balance
349 test:
350 suffix: 16
351 requires: triangle
352 nsize: 2
353 args: -dm_coord_space 0 -test_partition -overlap 1 -load_balance -dm_view ascii::ascii_info_detail -partition_balance
354 # Parallel redundant copying, test 8
355 test:
356 suffix: 17
357 requires: triangle
358 nsize: 2
359 args: -dm_coord_space 0 -test_redundant -dm_view ascii::ascii_info_detail -partition_balance
360 test:
361 suffix: lb_1
362 requires: parmetis
363 nsize: 4
364 args: -dm_coord_space 0 -dm_plex_simplex 0 -dm_plex_box_faces 4,4 -petscpartitioner_type shell -petscpartitioner_shell_random -lb_petscpartitioner_type parmetis -load_balance -lb_petscpartitioner_view -partition_balance -prelb_dm_view ::load_balance -dm_view ::load_balance
365 TEST*/
366