1 static char help[] = "This example demonstrates the use of DMNetwork with subnetworks for solving a coupled nonlinear \n\
2 electric power grid and water pipe problem.\n\
3 The available solver options are in the ex1options file \n\
4 and the data files are in the datafiles of subdirectories.\n\
5 Run this program: mpiexec -n <n> ./ex1 \n\\n";
6 /*
7 Example:
8 mpiexec -n 3 ./ex1 -petscpartitioner_type parmetis -dmnetwork_view draw -dmnetwork_view_distributed draw -dmnetwork_view_rank_range 0,1,2
9 mpiexec -n 3 ./ex1 -petscpartitioner_type simple -dmnetwork_view_distributed draw -dmnetwork_view_zoomin_vertices 0 -dmnetwork_view_zoomin_vertices_padding 2 -dmnetwork_view_rank_range 0
10 mpiexec -n <n> ./ex1 -monitorIteration -monitorColor -power_snes_max_it 0 -water_snes_max_it 0 -coupled_snes_max_it 10 -draw_pause 5.0
11 */
12
13 #include "power/power.h"
14 #include "water/water.h"
15
16 typedef struct {
17 UserCtx_Power appctx_power;
18 AppCtx_Water appctx_water;
19 PetscInt subsnes_id; /* snes solver id */
20 PetscInt it; /* iteration number */
21 Vec localXold; /* store previous solution, used by FormFunction_Dummy() */
22 PetscBool monitorColor;
23 } UserCtx;
24
25 /*
26 UserMonitor -- called at the end of every SNES iteration via option `-monitorIteration' or `-monitorColor'
27 */
UserMonitor(SNES snes,PetscInt its,PetscReal fnorm,void * appctx)28 PetscErrorCode UserMonitor(SNES snes, PetscInt its, PetscReal fnorm, void *appctx)
29 {
30 UserCtx *user = (UserCtx *)appctx;
31 PetscMPIInt rank;
32 MPI_Comm comm;
33 PetscInt it;
34
35 PetscFunctionBegin;
36 PetscCall(PetscObjectGetComm((PetscObject)snes, &comm));
37 PetscCallMPI(MPI_Comm_rank(comm, &rank));
38
39 PetscCall(SNESGetIterationNumber(snes, &it));
40 if (rank == 0) {
41 PetscCall(SNESGetIterationNumber(snes, &it));
42 if (user->subsnes_id == 0 || user->subsnes_id == 1) {
43 PetscCall(PetscPrintf(PETSC_COMM_SELF, " subsnes_id %" PetscInt_FMT ", it %" PetscInt_FMT ", fnorm %g\n", user->subsnes_id, it, (double)fnorm));
44 } else {
45 PetscCall(PetscPrintf(PETSC_COMM_SELF, " coupled_snes_it %" PetscInt_FMT ", total_snes_it %" PetscInt_FMT ", fnorm %g\n", it, user->it, (double)fnorm));
46 }
47 }
48
49 if (user->monitorColor) {
50 DM networkdm, dmcoords;
51 Vec F;
52 PetscInt v, vStart, vEnd, offset, gidx, rstart;
53 PetscReal *color;
54 PetscScalar *farr;
55 PetscBool ghost;
56
57 PetscCall(SNESGetDM(snes, &networkdm));
58 PetscCall(DMGetCoordinateDM(networkdm, &dmcoords));
59
60 PetscCall(SNESGetFunction(snes, &F, NULL, NULL));
61 PetscCall(VecGetOwnershipRange(F, &rstart, NULL));
62
63 PetscCall(VecGetArray(F, &farr));
64 PetscCall(DMNetworkGetVertexRange(dmcoords, &vStart, &vEnd));
65
66 PetscCall(PetscPrintf(MPI_COMM_WORLD, "\nColorPrint:\n"));
67 for (v = vStart; v < vEnd; v++) {
68 PetscCall(DMNetworkIsGhostVertex(networkdm, v, &ghost));
69 PetscCall(DMNetworkGetComponent(dmcoords, v, 0, NULL, (void **)&color, NULL));
70 PetscCall(DMNetworkGetGlobalVertexIndex(networkdm, v, &gidx));
71 if (!ghost) {
72 PetscCall(DMNetworkGetGlobalVecOffset(networkdm, v, 0, &offset));
73 *color = (PetscRealPart(farr[offset - rstart]));
74 }
75 PetscCall(PetscSynchronizedPrintf(MPI_COMM_WORLD, "[%d] v %" PetscInt_FMT ": color[%" PetscInt_FMT "] = %g\n", rank, gidx, offset - rstart, *color));
76 }
77 PetscCall(PetscSynchronizedFlush(MPI_COMM_WORLD, NULL));
78 PetscCall(VecRestoreArray(F, &farr));
79
80 PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_CSV));
81 PetscCall(DMView(networkdm, PETSC_VIEWER_DRAW_WORLD));
82 PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
83 }
84 PetscFunctionReturn(PETSC_SUCCESS);
85 }
86
FormJacobian_subPower(SNES snes,Vec X,Mat J,Mat Jpre,void * appctx)87 PetscErrorCode FormJacobian_subPower(SNES snes, Vec X, Mat J, Mat Jpre, void *appctx)
88 {
89 DM networkdm;
90 Vec localX;
91 PetscInt nv, ne, i, j, offset, nvar, row;
92 const PetscInt *vtx, *edges;
93 PetscBool ghostvtex;
94 PetscScalar one = 1.0;
95 PetscMPIInt rank;
96 MPI_Comm comm;
97
98 PetscFunctionBegin;
99 PetscCall(SNESGetDM(snes, &networkdm));
100 PetscCall(DMGetLocalVector(networkdm, &localX));
101
102 PetscCall(PetscObjectGetComm((PetscObject)networkdm, &comm));
103 PetscCallMPI(MPI_Comm_rank(comm, &rank));
104
105 PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, localX));
106 PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, localX));
107
108 PetscCall(MatZeroEntries(J));
109
110 /* Power subnetwork: copied from power/FormJacobian_Power() */
111 PetscCall(DMNetworkGetSubnetwork(networkdm, 0, &nv, &ne, &vtx, &edges));
112 PetscCall(FormJacobian_Power_private(networkdm, localX, J, nv, ne, vtx, edges, appctx));
113
114 /* Water subnetwork: Identity */
115 PetscCall(DMNetworkGetSubnetwork(networkdm, 1, &nv, &ne, &vtx, &edges));
116 for (i = 0; i < nv; i++) {
117 PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[i], &ghostvtex));
118 if (ghostvtex) continue;
119
120 PetscCall(DMNetworkGetGlobalVecOffset(networkdm, vtx[i], ALL_COMPONENTS, &offset));
121 PetscCall(DMNetworkGetComponent(networkdm, vtx[i], ALL_COMPONENTS, NULL, NULL, &nvar));
122 for (j = 0; j < nvar; j++) {
123 row = offset + j;
124 PetscCall(MatSetValues(J, 1, &row, 1, &row, &one, ADD_VALUES));
125 }
126 }
127 PetscCall(MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY));
128 PetscCall(MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY));
129
130 PetscCall(DMRestoreLocalVector(networkdm, &localX));
131 PetscFunctionReturn(PETSC_SUCCESS);
132 }
133
134 /* Dummy equation localF(X) = localX - localXold */
FormFunction_Dummy(DM networkdm,Vec localX,Vec localF,PetscInt nv,PetscInt ne,const PetscInt * vtx,const PetscInt * edges,void * appctx)135 PetscErrorCode FormFunction_Dummy(DM networkdm, Vec localX, Vec localF, PetscInt nv, PetscInt ne, const PetscInt *vtx, const PetscInt *edges, void *appctx)
136 {
137 const PetscScalar *xarr, *xoldarr;
138 PetscScalar *farr;
139 PetscInt i, j, offset, nvar;
140 PetscBool ghostvtex;
141 UserCtx *user = (UserCtx *)appctx;
142 Vec localXold = user->localXold;
143
144 PetscFunctionBegin;
145 PetscCall(VecGetArrayRead(localX, &xarr));
146 PetscCall(VecGetArrayRead(localXold, &xoldarr));
147 PetscCall(VecGetArray(localF, &farr));
148
149 for (i = 0; i < nv; i++) {
150 PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[i], &ghostvtex));
151 if (ghostvtex) continue;
152
153 PetscCall(DMNetworkGetLocalVecOffset(networkdm, vtx[i], ALL_COMPONENTS, &offset));
154 PetscCall(DMNetworkGetComponent(networkdm, vtx[i], ALL_COMPONENTS, NULL, NULL, &nvar));
155 for (j = 0; j < nvar; j++) farr[offset + j] = xarr[offset + j] - xoldarr[offset + j];
156 }
157
158 PetscCall(VecRestoreArrayRead(localX, &xarr));
159 PetscCall(VecRestoreArrayRead(localXold, &xoldarr));
160 PetscCall(VecRestoreArray(localF, &farr));
161 PetscFunctionReturn(PETSC_SUCCESS);
162 }
163
FormFunction(SNES snes,Vec X,Vec F,void * appctx)164 PetscErrorCode FormFunction(SNES snes, Vec X, Vec F, void *appctx)
165 {
166 DM networkdm;
167 Vec localX, localF;
168 PetscInt nv, ne, v;
169 const PetscInt *vtx, *edges;
170 PetscMPIInt rank;
171 MPI_Comm comm;
172 UserCtx *user = (UserCtx *)appctx;
173 UserCtx_Power appctx_power = (*user).appctx_power;
174 AppCtx_Water appctx_water = (*user).appctx_water;
175
176 PetscFunctionBegin;
177 PetscCall(SNESGetDM(snes, &networkdm));
178 PetscCall(PetscObjectGetComm((PetscObject)networkdm, &comm));
179 PetscCallMPI(MPI_Comm_rank(comm, &rank));
180
181 PetscCall(DMGetLocalVector(networkdm, &localX));
182 PetscCall(DMGetLocalVector(networkdm, &localF));
183 PetscCall(VecSet(F, 0.0));
184 PetscCall(VecSet(localF, 0.0));
185
186 PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, localX));
187 PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, localX));
188
189 /* Form Function for power subnetwork */
190 PetscCall(DMNetworkGetSubnetwork(networkdm, 0, &nv, &ne, &vtx, &edges));
191 if (user->subsnes_id == 1) { /* snes_water only */
192 PetscCall(FormFunction_Dummy(networkdm, localX, localF, nv, ne, vtx, edges, user));
193 } else {
194 PetscCall(FormFunction_Power(networkdm, localX, localF, nv, ne, vtx, edges, &appctx_power));
195 }
196
197 /* Form Function for water subnetwork */
198 PetscCall(DMNetworkGetSubnetwork(networkdm, 1, &nv, &ne, &vtx, &edges));
199 if (user->subsnes_id == 0) { /* snes_power only */
200 PetscCall(FormFunction_Dummy(networkdm, localX, localF, nv, ne, vtx, edges, user));
201 } else {
202 PetscCall(FormFunction_Water(networkdm, localX, localF, nv, ne, vtx, edges, NULL));
203 }
204
205 /* Illustrate how to access the coupling vertex of the subnetworks without doing anything to F yet */
206 PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
207 for (v = 0; v < nv; v++) {
208 PetscInt key, ncomp, nvar, nconnedges, k, e, keye, goffset[3];
209 void *component;
210 const PetscInt *connedges;
211
212 PetscCall(DMNetworkGetComponent(networkdm, vtx[v], ALL_COMPONENTS, NULL, NULL, &nvar));
213 PetscCall(DMNetworkGetNumComponents(networkdm, vtx[v], &ncomp));
214 /* printf(" [%d] coupling vertex[%" PetscInt_FMT "]: v %" PetscInt_FMT ", ncomp %" PetscInt_FMT "; nvar %" PetscInt_FMT "\n",rank,v,vtx[v], ncomp,nvar); */
215
216 for (k = 0; k < ncomp; k++) {
217 PetscCall(DMNetworkGetComponent(networkdm, vtx[v], k, &key, &component, &nvar));
218 PetscCall(DMNetworkGetGlobalVecOffset(networkdm, vtx[v], k, &goffset[k]));
219
220 /* Verify the coupling vertex is a powernet load vertex or a water vertex */
221 switch (k) {
222 case 0:
223 PetscCheck(key == appctx_power.compkey_bus && nvar == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "key %" PetscInt_FMT " not a power bus vertex or nvar %" PetscInt_FMT " != 2", key, nvar);
224 break;
225 case 1:
226 PetscCheck(key == appctx_power.compkey_load && nvar == 0 && goffset[1] == goffset[0] + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not a power load vertex");
227 break;
228 case 2:
229 PetscCheck(key == appctx_water.compkey_vtx && nvar == 1 && goffset[2] == goffset[1], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not a water vertex");
230 break;
231 default:
232 SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "k %" PetscInt_FMT " is wrong", k);
233 }
234 /* printf(" [%d] coupling vertex[%" PetscInt_FMT "]: key %" PetscInt_FMT "; nvar %" PetscInt_FMT ", goffset %" PetscInt_FMT "\n",rank,v,key,nvar,goffset[k]); */
235 }
236
237 /* Get its supporting edges */
238 PetscCall(DMNetworkGetSupportingEdges(networkdm, vtx[v], &nconnedges, &connedges));
239 /* printf("\n[%d] coupling vertex: nconnedges %" PetscInt_FMT "\n",rank,nconnedges); */
240 for (k = 0; k < nconnedges; k++) {
241 e = connedges[k];
242 PetscCall(DMNetworkGetNumComponents(networkdm, e, &ncomp));
243 /* printf("\n [%d] connected edge[%" PetscInt_FMT "]=%" PetscInt_FMT " has ncomp %" PetscInt_FMT "\n",rank,k,e,ncomp); */
244 PetscCall(DMNetworkGetComponent(networkdm, e, 0, &keye, &component, NULL));
245 if (keye != appctx_water.compkey_edge) PetscCheck(keye == appctx_power.compkey_branch, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not a power branch");
246 }
247 }
248
249 PetscCall(DMRestoreLocalVector(networkdm, &localX));
250
251 PetscCall(DMLocalToGlobalBegin(networkdm, localF, ADD_VALUES, F));
252 PetscCall(DMLocalToGlobalEnd(networkdm, localF, ADD_VALUES, F));
253 PetscCall(DMRestoreLocalVector(networkdm, &localF));
254 #if 0
255 if (rank == 0) printf("F:\n");
256 PetscCall(VecView(F,PETSC_VIEWER_STDOUT_WORLD));
257 #endif
258 PetscFunctionReturn(PETSC_SUCCESS);
259 }
260
SetInitialGuess(DM networkdm,Vec X,void * appctx)261 PetscErrorCode SetInitialGuess(DM networkdm, Vec X, void *appctx)
262 {
263 PetscInt nv, ne, i, j, ncomp, offset, key;
264 const PetscInt *vtx, *edges;
265 UserCtx *user = (UserCtx *)appctx;
266 Vec localX = user->localXold;
267 UserCtx_Power appctx_power = (*user).appctx_power;
268 AppCtx_Water appctx_water = (*user).appctx_water;
269 PetscBool ghost;
270 PetscScalar *xarr;
271 VERTEX_Power bus;
272 VERTEX_Water vertex;
273 void *component;
274 GEN gen;
275
276 PetscFunctionBegin;
277 PetscCall(VecSet(X, 0.0));
278 PetscCall(VecSet(localX, 0.0));
279
280 /* Set initial guess for power subnetwork */
281 PetscCall(DMNetworkGetSubnetwork(networkdm, 0, &nv, &ne, &vtx, &edges));
282 PetscCall(SetInitialGuess_Power(networkdm, localX, nv, ne, vtx, edges, &appctx_power));
283
284 /* Set initial guess for water subnetwork */
285 PetscCall(DMNetworkGetSubnetwork(networkdm, 1, &nv, &ne, &vtx, &edges));
286 PetscCall(SetInitialGuess_Water(networkdm, localX, nv, ne, vtx, edges, NULL));
287
288 /* Set initial guess at the coupling vertex */
289 PetscCall(VecGetArray(localX, &xarr));
290 PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
291 for (i = 0; i < nv; i++) {
292 PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[i], &ghost));
293 if (ghost) continue;
294
295 PetscCall(DMNetworkGetNumComponents(networkdm, vtx[i], &ncomp));
296 for (j = 0; j < ncomp; j++) {
297 PetscCall(DMNetworkGetLocalVecOffset(networkdm, vtx[i], j, &offset));
298 PetscCall(DMNetworkGetComponent(networkdm, vtx[i], j, &key, (void **)&component, NULL));
299 if (key == appctx_power.compkey_bus) {
300 bus = (VERTEX_Power)component;
301 xarr[offset] = bus->va * PETSC_PI / 180.0;
302 xarr[offset + 1] = bus->vm;
303 } else if (key == appctx_power.compkey_gen) {
304 gen = (GEN)component;
305 if (!gen->status) continue;
306 xarr[offset + 1] = gen->vs;
307 } else if (key == appctx_water.compkey_vtx) {
308 vertex = (VERTEX_Water)component;
309 if (vertex->type == VERTEX_TYPE_JUNCTION) {
310 xarr[offset] = 100;
311 } else if (vertex->type == VERTEX_TYPE_RESERVOIR) {
312 xarr[offset] = vertex->res.head;
313 } else {
314 xarr[offset] = vertex->tank.initlvl + vertex->tank.elev;
315 }
316 }
317 }
318 }
319 PetscCall(VecRestoreArray(localX, &xarr));
320
321 PetscCall(DMLocalToGlobalBegin(networkdm, localX, ADD_VALUES, X));
322 PetscCall(DMLocalToGlobalEnd(networkdm, localX, ADD_VALUES, X));
323 PetscFunctionReturn(PETSC_SUCCESS);
324 }
325
326 /* Set coordinates */
CoordinateVecSetUp(DM dmcoords,Vec coords)327 static PetscErrorCode CoordinateVecSetUp(DM dmcoords, Vec coords)
328 {
329 PetscInt i, gidx, offset, v, nv, Nsubnet;
330 const PetscInt *vtx;
331 PetscScalar *carray;
332 PetscReal *color;
333
334 PetscFunctionBeginUser;
335 PetscCall(VecGetArrayWrite(coords, &carray));
336 PetscCall(DMNetworkGetNumSubNetworks(dmcoords, NULL, &Nsubnet));
337 for (i = 0; i < Nsubnet; i++) {
338 PetscCall(DMNetworkGetSubnetwork(dmcoords, i, &nv, NULL, &vtx, NULL));
339 for (v = 0; v < nv; v++) {
340 PetscCall(DMNetworkGetGlobalVertexIndex(dmcoords, vtx[v], &gidx));
341 PetscCall(DMNetworkGetLocalVecOffset(dmcoords, vtx[v], 0, &offset));
342 PetscCall(DMNetworkGetComponent(dmcoords, vtx[v], 0, NULL, (void **)&color, NULL));
343 *color = 0.0;
344 switch (gidx) {
345 case 0:
346 carray[offset] = -1.0;
347 carray[offset + 1] = -1.0;
348 break;
349 case 1:
350 carray[offset] = -2.0;
351 carray[offset + 1] = 2.0;
352 break;
353 case 2:
354 carray[offset] = 0.0;
355 carray[offset + 1] = 2.0;
356 break;
357 case 3:
358 carray[offset] = -1.0;
359 carray[offset + 1] = 0.0;
360 break;
361 case 4:
362 carray[offset] = 0.0;
363 carray[offset + 1] = 0.0;
364 break;
365 case 5:
366 carray[offset] = 0.0;
367 carray[offset + 1] = 1.0;
368 break;
369 case 6:
370 carray[offset] = -1.0;
371 carray[offset + 1] = 1.0;
372 break;
373 case 7:
374 carray[offset] = -2.0;
375 carray[offset + 1] = 1.0;
376 break;
377 case 8:
378 carray[offset] = -2.0;
379 carray[offset + 1] = 0.0;
380 break;
381 case 9:
382 carray[offset] = 1.0;
383 carray[offset + 1] = 0.0;
384 break;
385 case 10:
386 carray[offset] = 1.0;
387 carray[offset + 1] = -1.0;
388 break;
389 case 11:
390 carray[offset] = 2.0;
391 carray[offset + 1] = -1.0;
392 break;
393 case 12:
394 carray[offset] = 2.0;
395 carray[offset + 1] = 0.0;
396 break;
397 case 13:
398 carray[offset] = 0.0;
399 carray[offset + 1] = -1.0;
400 break;
401 case 14:
402 carray[offset] = 2.0;
403 carray[offset + 1] = 1.0;
404 break;
405 default:
406 PetscCheck(gidx < 15 && gidx > -1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "gidx %" PetscInt_FMT "must between 0 and 14", gidx);
407 }
408 }
409 }
410 PetscCall(VecRestoreArrayWrite(coords, &carray));
411 PetscFunctionReturn(PETSC_SUCCESS);
412 }
413
CoordinatePrint(DM dm)414 static PetscErrorCode CoordinatePrint(DM dm)
415 {
416 DM dmcoords;
417 PetscInt cdim, v, off, vglobal, vStart, vEnd;
418 const PetscScalar *carray;
419 Vec coords;
420 MPI_Comm comm;
421 PetscMPIInt rank;
422
423 PetscFunctionBegin;
424 /* get info from dm */
425 PetscCall(DMGetCoordinateDim(dm, &cdim));
426 PetscCall(DMGetCoordinatesLocal(dm, &coords));
427
428 PetscCall(DMGetCoordinateDM(dm, &dmcoords));
429 PetscCall(PetscObjectGetComm((PetscObject)dmcoords, &comm));
430 PetscCallMPI(MPI_Comm_rank(comm, &rank));
431
432 /* print coordinates from dmcoords */
433 PetscCall(PetscPrintf(MPI_COMM_WORLD, "\nCoordinatePrint, cdim %" PetscInt_FMT ":\n", cdim));
434 PetscCall(PetscSynchronizedPrintf(MPI_COMM_WORLD, "[%d]\n", rank));
435
436 PetscCall(DMNetworkGetVertexRange(dmcoords, &vStart, &vEnd));
437 PetscCall(VecGetArrayRead(coords, &carray));
438 for (v = vStart; v < vEnd; v++) {
439 PetscCall(DMNetworkGetLocalVecOffset(dmcoords, v, 0, &off));
440 PetscCall(DMNetworkGetGlobalVertexIndex(dmcoords, v, &vglobal));
441 switch (cdim) {
442 case 2:
443 PetscCall(PetscSynchronizedPrintf(MPI_COMM_WORLD, "Vertex: %" PetscInt_FMT ", x = %f y = %f \n", vglobal, (double)PetscRealPart(carray[off]), (double)PetscRealPart(carray[off + 1])));
444 break;
445 default:
446 PetscCheck(cdim == 2, MPI_COMM_WORLD, PETSC_ERR_SUP, "Only supports Network embedding dimension of 2, not supplied %" PetscInt_FMT, cdim);
447 break;
448 }
449 }
450 PetscCall(PetscSynchronizedFlush(MPI_COMM_WORLD, NULL));
451 PetscCall(VecRestoreArrayRead(coords, &carray));
452 PetscFunctionReturn(PETSC_SUCCESS);
453 }
454
main(int argc,char ** argv)455 int main(int argc, char **argv)
456 {
457 DM networkdm, dmcoords;
458 PetscMPIInt rank, size;
459 PetscInt Nsubnet = 2, numVertices[2], numEdges[2], i, j, nv, ne, it_max = 10;
460 PetscInt vStart, vEnd, compkey;
461 const PetscInt *vtx, *edges;
462 PetscReal *color;
463 Vec X, F, coords;
464 SNES snes, snes_power, snes_water;
465 Mat Jac;
466 PetscBool ghost, viewJ = PETSC_FALSE, viewX = PETSC_FALSE, test = PETSC_FALSE, distribute = PETSC_TRUE, flg, printCoord = PETSC_FALSE, viewCSV = PETSC_FALSE, monitorIt = PETSC_FALSE;
467 UserCtx user;
468 SNESConvergedReason reason;
469 PetscLogStage stage[4];
470
471 /* Power subnetwork */
472 UserCtx_Power *appctx_power = &user.appctx_power;
473 char pfdata_file[PETSC_MAX_PATH_LEN] = "power/case9.m";
474 PFDATA *pfdata = NULL;
475 PetscInt genj, loadj, *edgelist_power = NULL, power_netnum;
476 PetscScalar Sbase = 0.0;
477
478 /* Water subnetwork */
479 AppCtx_Water *appctx_water = &user.appctx_water;
480 WATERDATA *waterdata = NULL;
481 char waterdata_file[PETSC_MAX_PATH_LEN] = "water/sample1.inp";
482 PetscInt *edgelist_water = NULL, water_netnum;
483
484 /* Shared vertices between subnetworks */
485 PetscInt power_svtx, water_svtx;
486
487 PetscFunctionBeginUser;
488 PetscCall(PetscInitialize(&argc, &argv, "ex1options", help));
489 PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
490 PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
491
492 /* (1) Read Data - Only rank 0 reads the data */
493 PetscCall(PetscLogStageRegister("Read Data", &stage[0]));
494 PetscCall(PetscLogStagePush(stage[0]));
495
496 for (i = 0; i < Nsubnet; i++) {
497 numVertices[i] = 0;
498 numEdges[i] = 0;
499 }
500
501 /* All processes READ THE DATA FOR THE FIRST SUBNETWORK: Electric Power Grid */
502 /* Used for shared vertex, because currently the coupling info must be available in all processes!!! */
503 PetscCall(PetscOptionsGetString(NULL, NULL, "-pfdata", pfdata_file, PETSC_MAX_PATH_LEN - 1, NULL));
504 PetscCall(PetscNew(&pfdata));
505 PetscCall(PFReadMatPowerData(pfdata, pfdata_file));
506 if (rank == 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, "Power network: nb = %" PetscInt_FMT ", ngen = %" PetscInt_FMT ", nload = %" PetscInt_FMT ", nbranch = %" PetscInt_FMT "\n", pfdata->nbus, pfdata->ngen, pfdata->nload, pfdata->nbranch));
507 Sbase = pfdata->sbase;
508 if (rank == 0) { /* proc[0] will create Electric Power Grid */
509 numEdges[0] = pfdata->nbranch;
510 numVertices[0] = pfdata->nbus;
511
512 PetscCall(PetscMalloc1(2 * numEdges[0], &edgelist_power));
513 PetscCall(GetListofEdges_Power(pfdata, edgelist_power));
514 }
515 /* Broadcast power Sbase to all processors */
516 PetscCallMPI(MPI_Bcast(&Sbase, 1, MPIU_SCALAR, 0, PETSC_COMM_WORLD));
517 appctx_power->Sbase = Sbase;
518 appctx_power->jac_error = PETSC_FALSE;
519 /* If external option activated. Introduce error in jacobian */
520 PetscCall(PetscOptionsHasName(NULL, NULL, "-jac_error", &appctx_power->jac_error));
521
522 /* All processes READ THE DATA FOR THE SECOND SUBNETWORK: Water */
523 /* Used for shared vertex, because currently the coupling info must be available in all processes!!! */
524 PetscCall(PetscNew(&waterdata));
525 PetscCall(PetscOptionsGetString(NULL, NULL, "-waterdata", waterdata_file, PETSC_MAX_PATH_LEN - 1, NULL));
526 PetscCall(WaterReadData(waterdata, waterdata_file));
527 if (size == 1 || (size > 1 && rank == 1)) {
528 PetscCall(PetscCalloc1(2 * waterdata->nedge, &edgelist_water));
529 PetscCall(GetListofEdges_Water(waterdata, edgelist_water));
530 numEdges[1] = waterdata->nedge;
531 numVertices[1] = waterdata->nvertex;
532 }
533 PetscCall(PetscLogStagePop());
534
535 /* (2) Create a network consist of two subnetworks */
536 PetscCall(PetscLogStageRegister("Net Setup", &stage[1]));
537 PetscCall(PetscLogStagePush(stage[1]));
538
539 PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewCSV", &viewCSV, NULL));
540 PetscCall(PetscOptionsGetBool(NULL, NULL, "-printCoord", &printCoord, NULL));
541 PetscCall(PetscOptionsGetBool(NULL, NULL, "-test", &test, NULL));
542 PetscCall(PetscOptionsGetBool(NULL, NULL, "-distribute", &distribute, NULL));
543
544 /* Create an empty network object */
545 PetscCall(DMNetworkCreate(PETSC_COMM_WORLD, &networkdm));
546
547 /* Register the components in the network */
548 PetscCall(DMNetworkRegisterComponent(networkdm, "branchstruct", sizeof(struct _p_EDGE_Power), &appctx_power->compkey_branch));
549 PetscCall(DMNetworkRegisterComponent(networkdm, "busstruct", sizeof(struct _p_VERTEX_Power), &appctx_power->compkey_bus));
550 PetscCall(DMNetworkRegisterComponent(networkdm, "genstruct", sizeof(struct _p_GEN), &appctx_power->compkey_gen));
551 PetscCall(DMNetworkRegisterComponent(networkdm, "loadstruct", sizeof(struct _p_LOAD), &appctx_power->compkey_load));
552
553 PetscCall(DMNetworkRegisterComponent(networkdm, "edge_water", sizeof(struct _p_EDGE_Water), &appctx_water->compkey_edge));
554 PetscCall(DMNetworkRegisterComponent(networkdm, "vertex_water", sizeof(struct _p_VERTEX_Water), &appctx_water->compkey_vtx));
555
556 PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] Total local nvertices %" PetscInt_FMT " + %" PetscInt_FMT " = %" PetscInt_FMT ", nedges %" PetscInt_FMT " + %" PetscInt_FMT " = %" PetscInt_FMT "\n", rank, numVertices[0], numVertices[1], numVertices[0] + numVertices[1], numEdges[0], numEdges[1], numEdges[0] + numEdges[1]));
557 PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
558
559 PetscCall(DMNetworkSetNumSubNetworks(networkdm, PETSC_DECIDE, Nsubnet));
560 PetscCall(DMNetworkAddSubnetwork(networkdm, "power", numEdges[0], edgelist_power, &power_netnum));
561 PetscCall(DMNetworkAddSubnetwork(networkdm, "water", numEdges[1], edgelist_water, &water_netnum));
562
563 /* vertex subnet[0].4 shares with vertex subnet[1].0 */
564 power_svtx = 4;
565 water_svtx = 0;
566 PetscCall(DMNetworkAddSharedVertices(networkdm, power_netnum, water_netnum, 1, &power_svtx, &water_svtx));
567
568 /* Set up the network layout */
569 PetscCall(DMNetworkLayoutSetUp(networkdm));
570
571 /* ADD VARIABLES AND COMPONENTS FOR THE POWER SUBNETWORK */
572 genj = 0;
573 loadj = 0;
574 PetscCall(DMNetworkGetSubnetwork(networkdm, power_netnum, &nv, &ne, &vtx, &edges));
575
576 for (i = 0; i < ne; i++) PetscCall(DMNetworkAddComponent(networkdm, edges[i], appctx_power->compkey_branch, &pfdata->branch[i], 0));
577
578 for (i = 0; i < nv; i++) {
579 PetscCall(DMNetworkIsSharedVertex(networkdm, vtx[i], &flg));
580 if (flg) continue;
581
582 PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_bus, &pfdata->bus[i], 2));
583 if (pfdata->bus[i].ngen) {
584 for (j = 0; j < pfdata->bus[i].ngen; j++) PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_gen, &pfdata->gen[genj++], 0));
585 }
586 if (pfdata->bus[i].nload) {
587 for (j = 0; j < pfdata->bus[i].nload; j++) PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_load, &pfdata->load[loadj++], 0));
588 }
589 }
590
591 /* ADD VARIABLES AND COMPONENTS FOR THE WATER SUBNETWORK */
592 PetscCall(DMNetworkGetSubnetwork(networkdm, water_netnum, &nv, &ne, &vtx, &edges));
593 for (i = 0; i < ne; i++) PetscCall(DMNetworkAddComponent(networkdm, edges[i], appctx_water->compkey_edge, &waterdata->edge[i], 0));
594
595 for (i = 0; i < nv; i++) {
596 PetscCall(DMNetworkIsSharedVertex(networkdm, vtx[i], &flg));
597 if (flg) continue;
598
599 PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_water->compkey_vtx, &waterdata->vertex[i], 1));
600 }
601
602 /* ADD VARIABLES AND COMPONENTS AT THE SHARED VERTEX: net[0].4 coupls with net[1].0 -- owning and all ghost ranks of the vertex do this */
603 PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
604 for (i = 0; i < nv; i++) {
605 /* power */
606 PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_bus, &pfdata->bus[4], 2));
607 /* bus[4] is a load, add its component */
608 PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_load, &pfdata->load[0], 0));
609
610 /* water */
611 PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_water->compkey_vtx, &waterdata->vertex[0], 1));
612 }
613
614 /* Set coordinates for visualization */
615 PetscCall(DMSetCoordinateDim(networkdm, 2));
616 PetscCall(DMGetCoordinateDM(networkdm, &dmcoords));
617 PetscCall(DMNetworkGetVertexRange(dmcoords, &vStart, &vEnd));
618
619 PetscCall(PetscCalloc1(vEnd - vStart, &color));
620 PetscCall(DMNetworkRegisterComponent(dmcoords, "coordinate&color", sizeof(PetscReal), &compkey));
621 for (i = vStart; i < vEnd; i++) PetscCall(DMNetworkAddComponent(dmcoords, i, compkey, &color[i - vStart], 2));
622 PetscCall(DMNetworkFinalizeComponents(dmcoords));
623
624 PetscCall(DMCreateLocalVector(dmcoords, &coords));
625 PetscCall(DMSetCoordinatesLocal(networkdm, coords)); /* set/get coords to/from networkdm */
626 PetscCall(CoordinateVecSetUp(dmcoords, coords));
627 if (printCoord) PetscCall(CoordinatePrint(networkdm));
628
629 /* Set up DM for use */
630 PetscCall(DMSetUp(networkdm));
631
632 /* Free user objects */
633 PetscCall(PetscFree(edgelist_power));
634 PetscCall(PetscFree(pfdata->bus));
635 PetscCall(PetscFree(pfdata->gen));
636 PetscCall(PetscFree(pfdata->branch));
637 PetscCall(PetscFree(pfdata->load));
638 PetscCall(PetscFree(pfdata));
639
640 PetscCall(PetscFree(edgelist_water));
641 PetscCall(PetscFree(waterdata->vertex));
642 PetscCall(PetscFree(waterdata->edge));
643 PetscCall(PetscFree(waterdata));
644
645 /* Re-distribute networkdm to multiple processes for better job balance */
646 if (distribute) {
647 PetscCall(DMNetworkDistribute(&networkdm, 0));
648
649 if (printCoord) PetscCall(CoordinatePrint(networkdm));
650 if (viewCSV) { /* CSV View of network with coordinates */
651 PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_CSV));
652 PetscCall(DMView(networkdm, PETSC_VIEWER_STDOUT_WORLD));
653 PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
654 }
655 }
656 PetscCall(VecDestroy(&coords));
657
658 /* Test DMNetworkGetSubnetwork() and DMNetworkGetSubnetworkSharedVertices() */
659 if (test) {
660 PetscInt v, gidx;
661 PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
662 for (i = 0; i < Nsubnet; i++) {
663 PetscCall(DMNetworkGetSubnetwork(networkdm, i, &nv, &ne, &vtx, &edges));
664 PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] After distribute, subnet[%" PetscInt_FMT "] ne %" PetscInt_FMT ", nv %" PetscInt_FMT "\n", rank, i, ne, nv));
665 PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
666
667 for (v = 0; v < nv; v++) {
668 PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[v], &ghost));
669 PetscCall(DMNetworkGetGlobalVertexIndex(networkdm, vtx[v], &gidx));
670 PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] subnet[%" PetscInt_FMT "] v %" PetscInt_FMT " %" PetscInt_FMT "; ghost %d\n", rank, i, vtx[v], gidx, ghost));
671 }
672 PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
673 }
674 PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
675
676 PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
677 PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] After distribute, num of shared vertices nsv = %" PetscInt_FMT "\n", rank, nv));
678 for (v = 0; v < nv; v++) {
679 PetscCall(DMNetworkGetGlobalVertexIndex(networkdm, vtx[v], &gidx));
680 PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] sv %" PetscInt_FMT ", gidx=%" PetscInt_FMT "\n", rank, vtx[v], gidx));
681 }
682 PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
683 }
684
685 /* Create solution vector X */
686 PetscCall(DMCreateGlobalVector(networkdm, &X));
687 PetscCall(VecDuplicate(X, &F));
688 PetscCall(DMGetLocalVector(networkdm, &user.localXold));
689 PetscCall(PetscLogStagePop());
690
691 /* (3) Setup Solvers */
692 PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewJ", &viewJ, NULL));
693 PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewX", &viewX, NULL));
694
695 PetscCall(PetscLogStageRegister("SNES Setup", &stage[2]));
696 PetscCall(PetscLogStagePush(stage[2]));
697
698 PetscCall(SetInitialGuess(networkdm, X, &user));
699
700 /* Create coupled snes */
701 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "SNES_coupled setup ......\n"));
702 user.subsnes_id = Nsubnet;
703 PetscCall(SNESCreate(PETSC_COMM_WORLD, &snes));
704 PetscCall(SNESSetDM(snes, networkdm));
705 PetscCall(SNESSetOptionsPrefix(snes, "coupled_"));
706 PetscCall(SNESSetFunction(snes, F, FormFunction, &user));
707 /* set maxit=1 which can be changed via option '-coupled_snes_max_it <>', see ex1options */
708 PetscCall(SNESSetTolerances(snes, PETSC_CURRENT, PETSC_CURRENT, PETSC_CURRENT, 1, PETSC_CURRENT));
709 PetscCall(SNESSetFromOptions(snes));
710
711 if (viewJ) {
712 /* View Jac structure */
713 PetscCall(SNESGetJacobian(snes, &Jac, NULL, NULL, NULL));
714 PetscCall(MatView(Jac, PETSC_VIEWER_DRAW_WORLD));
715 }
716
717 if (viewX) {
718 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Solution:\n"));
719 PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
720 }
721
722 if (viewJ) {
723 /* View assembled Jac */
724 PetscCall(MatView(Jac, PETSC_VIEWER_DRAW_WORLD));
725 }
726
727 /* Create snes_power */
728 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "SNES_power setup ......\n"));
729 user.subsnes_id = 0;
730 PetscCall(SNESCreate(PETSC_COMM_WORLD, &snes_power));
731 PetscCall(SNESSetDM(snes_power, networkdm));
732 PetscCall(SNESSetOptionsPrefix(snes_power, "power_"));
733 PetscCall(SNESSetFunction(snes_power, F, FormFunction, &user));
734 /* set maxit=1 which can be changed via option '-power_snes_max_it <>', see ex1options */
735 PetscCall(SNESSetTolerances(snes_power, PETSC_CURRENT, PETSC_CURRENT, PETSC_CURRENT, 1, PETSC_CURRENT));
736
737 /* Use user-provide Jacobian */
738 PetscCall(DMCreateMatrix(networkdm, &Jac));
739 PetscCall(SNESSetJacobian(snes_power, Jac, Jac, FormJacobian_subPower, &user));
740 PetscCall(SNESSetFromOptions(snes_power));
741
742 if (viewX) {
743 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Power Solution:\n"));
744 PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
745 }
746 if (viewJ) {
747 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Power Jac:\n"));
748 PetscCall(SNESGetJacobian(snes_power, &Jac, NULL, NULL, NULL));
749 PetscCall(MatView(Jac, PETSC_VIEWER_DRAW_WORLD));
750 }
751
752 /* Create snes_water */
753 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "SNES_water setup......\n"));
754
755 user.subsnes_id = 1;
756 PetscCall(SNESCreate(PETSC_COMM_WORLD, &snes_water));
757 PetscCall(SNESSetDM(snes_water, networkdm));
758 PetscCall(SNESSetOptionsPrefix(snes_water, "water_"));
759 PetscCall(SNESSetFunction(snes_water, F, FormFunction, &user));
760 /* set maxit=1 which can be changed via option '-water_snes_max_it <>', see ex1options */
761 PetscCall(SNESSetTolerances(snes_water, PETSC_CURRENT, PETSC_CURRENT, PETSC_CURRENT, 1, PETSC_CURRENT));
762 PetscCall(SNESSetFromOptions(snes_water));
763
764 if (viewX) {
765 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Water Solution:\n"));
766 PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
767 }
768
769 /* Monitor snes, snes_power and snes_water iterations */
770 PetscCall(PetscOptionsGetBool(NULL, NULL, "-monitorIteration", &monitorIt, NULL));
771 user.monitorColor = PETSC_FALSE;
772 PetscCall(PetscOptionsGetBool(NULL, NULL, "-monitorColor", &user.monitorColor, NULL));
773 if (user.monitorColor) monitorIt = PETSC_TRUE; /* require installation of pandas and matplotlib */
774 if (monitorIt) {
775 PetscCall(SNESMonitorSet(snes_power, UserMonitor, &user, NULL));
776 PetscCall(SNESMonitorSet(snes_water, UserMonitor, &user, NULL));
777 PetscCall(SNESMonitorSet(snes, UserMonitor, &user, NULL));
778 }
779 PetscCall(PetscLogStagePop());
780
781 /* (4) Solve: we must update user.localXold after each call of SNESSolve().
782 See "PETSc DMNetwork: A Library for Scalable Network PDE-Based Multiphysics Simulations",
783 https://dl.acm.org/doi/10.1145/3344587
784 */
785 PetscCall(PetscLogStageRegister("SNES Solve", &stage[3]));
786 PetscCall(PetscLogStagePush(stage[3]));
787 user.it = 0; /* total_snes_it */
788 reason = SNES_DIVERGED_DTOL;
789 while (user.it < it_max && (PetscInt)reason < 0) {
790 user.subsnes_id = 0;
791 PetscCall(SNESSolve(snes_power, NULL, X));
792 PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, user.localXold));
793 PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, user.localXold));
794
795 user.subsnes_id = 1;
796 PetscCall(SNESSolve(snes_water, NULL, X));
797 PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, user.localXold));
798 PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, user.localXold));
799
800 user.subsnes_id = Nsubnet;
801 PetscCall(SNESSolve(snes, NULL, X));
802 PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, user.localXold));
803 PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, user.localXold));
804
805 PetscCall(SNESGetConvergedReason(snes, &reason));
806 user.it++;
807 }
808 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Coupled_SNES converged in %" PetscInt_FMT " iterations\n", user.it));
809 if (viewX) {
810 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Final Solution:\n"));
811 PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
812 }
813 PetscCall(PetscLogStagePop());
814
815 /* Free objects */
816 /* -------------*/
817 PetscCall(PetscFree(color));
818 PetscCall(VecDestroy(&X));
819 PetscCall(VecDestroy(&F));
820 PetscCall(DMRestoreLocalVector(networkdm, &user.localXold));
821
822 PetscCall(SNESDestroy(&snes));
823 PetscCall(MatDestroy(&Jac));
824 PetscCall(SNESDestroy(&snes_power));
825 PetscCall(SNESDestroy(&snes_water));
826
827 PetscCall(DMDestroy(&networkdm));
828 PetscCall(PetscFinalize());
829 return 0;
830 }
831
832 /*TEST
833
834 build:
835 requires: !complex double defined(PETSC_HAVE_ATTRIBUTEALIGNED)
836 depends: power/PFReadData.c power/pffunctions.c water/waterreaddata.c water/waterfunctions.c
837
838 test:
839 args: -options_left no -dmnetwork_view
840 localrunfiles: ex1options power/case9.m water/sample1.inp
841 output_file: output/ex1.out
842
843 test:
844 suffix: 2
845 nsize: 3
846 args: -options_left no -petscpartitioner_type parmetis
847 args: -water_pc_type jacobi -power_pc_type jacobi -coupled_pc_type jacobi
848 localrunfiles: ex1options power/case9.m water/sample1.inp
849 output_file: output/ex1_2.out
850 requires: parmetis
851
852 test:
853 suffix: 3
854 nsize: 3
855 args: -options_left no -distribute false
856 args: -water_pc_type jacobi -power_pc_type jacobi -coupled_pc_type jacobi
857 localrunfiles: ex1options power/case9.m water/sample1.inp
858 output_file: output/ex1_3.out
859
860 test:
861 suffix: 4
862 nsize: 4
863 args: -options_left no -petscpartitioner_type simple -dmnetwork_view -dmnetwork_view_distributed
864 args: -water_pc_type jacobi -power_pc_type jacobi -coupled_pc_type jacobi
865 localrunfiles: ex1options power/case9.m water/sample1.inp
866 output_file: output/ex1_4.out
867
868 test:
869 suffix: 5
870 args: -options_left no -viewCSV
871 args: -water_pc_type jacobi -power_pc_type jacobi -coupled_pc_type jacobi
872 localrunfiles: ex1options power/case9.m water/sample1.inp
873 output_file: output/ex1_5.out
874
875 test:
876 suffix: 6
877 nsize: 3
878 args: -options_left no -petscpartitioner_type parmetis -dmnetwork_view_distributed draw:null
879 args: -water_pc_type jacobi -power_pc_type jacobi -coupled_pc_type jacobi
880 localrunfiles: ex1options power/case9.m water/sample1.inp
881 output_file: output/ex1_6.out
882 requires: parmetis
883
884 TEST*/
885