1 static char help[] = "Test PetscSFFetchAndOp on patterned SF graphs. PetscSFFetchAndOp internally uses PetscSFBcastAndOp \n\
2 and PetscSFReduce. So it is a good test to see if they all work for patterned graphs.\n\
3 Run with ./prog -op [replace | sum]\n\n";
4
5 #include <petscvec.h>
6 #include <petscsf.h>
main(int argc,char ** argv)7 int main(int argc, char **argv)
8 {
9 PetscInt i, N = 10, low, high, nleaves;
10 PetscMPIInt size, rank;
11 Vec x, y, y2, gy2;
12 PetscScalar *rootdata, *leafdata, *leafupdate;
13 PetscLayout layout;
14 PetscSF gathersf, allgathersf, alltoallsf;
15 MPI_Op op = MPI_SUM;
16 char opname[64];
17 const char *mpiopname;
18 PetscBool flag, isreplace, issum;
19
20 PetscFunctionBeginUser;
21 PetscCall(PetscInitialize(&argc, &argv, NULL, help));
22 PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
23 PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
24
25 PetscCall(PetscOptionsGetString(NULL, NULL, "-op", opname, sizeof(opname), &flag));
26 PetscCall(PetscStrcmp(opname, "replace", &isreplace));
27 PetscCall(PetscStrcmp(opname, "sum", &issum));
28
29 if (isreplace) {
30 op = MPI_REPLACE;
31 mpiopname = "MPI_REPLACE";
32 } else if (issum) {
33 op = MPIU_SUM;
34 mpiopname = "MPI_SUM";
35 } else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Unsupported argument (%s) to -op, which must be 'replace' or 'sum'", opname);
36
37 PetscCall(VecCreate(PETSC_COMM_WORLD, &x));
38 PetscCall(VecSetFromOptions(x));
39 PetscCall(VecSetSizes(x, PETSC_DECIDE, N));
40
41 /*-------------------------------------*/
42 /* PETSCSF_PATTERN_GATHER */
43 /*-------------------------------------*/
44
45 /* set MPI vec x to [1, 2, .., N] */
46 PetscCall(VecGetOwnershipRange(x, &low, &high));
47 for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i + 1.0, INSERT_VALUES));
48 PetscCall(VecAssemblyBegin(x));
49 PetscCall(VecAssemblyEnd(x));
50
51 /* Create the gather SF */
52 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_GATHER graph with op = %s\n", mpiopname));
53 PetscCall(VecGetLayout(x, &layout));
54 PetscCall(PetscSFCreate(PETSC_COMM_WORLD, &gathersf));
55 PetscCall(PetscSFSetGraphWithPattern(gathersf, layout, PETSCSF_PATTERN_GATHER));
56
57 /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */
58 PetscCall(PetscSFGetGraph(gathersf, NULL, &nleaves, NULL, NULL));
59 PetscCall(VecCreateSeq(PETSC_COMM_SELF, nleaves, &y));
60 PetscCall(VecDuplicate(y, &y2));
61
62 PetscCall(VecGetArray(x, &rootdata));
63 PetscCall(VecGetArray(y, &leafdata));
64 PetscCall(VecGetArray(y2, &leafupdate));
65
66 /* Bcast x to y,to initialize y = [1,N], then scale y to make leafupdate = y = [2,2*N] */
67 PetscCall(PetscSFBcastBegin(gathersf, MPIU_SCALAR, rootdata, leafdata, MPI_REPLACE));
68 PetscCall(PetscSFBcastEnd(gathersf, MPIU_SCALAR, rootdata, leafdata, MPI_REPLACE));
69 PetscCall(VecRestoreArray(y, &leafdata));
70 PetscCall(VecScale(y, 2));
71 PetscCall(VecGetArray(y, &leafdata));
72
73 /* FetchAndOp x to y */
74 PetscCall(PetscSFFetchAndOpBegin(gathersf, MPIU_SCALAR, rootdata, leafdata, leafupdate, op));
75 PetscCall(PetscSFFetchAndOpEnd(gathersf, MPIU_SCALAR, rootdata, leafdata, leafupdate, op));
76
77 /* View roots (x) and leafupdate (y2). Since this is a gather graph, leafudpate = rootdata = [1,N], then rootdata += leafdata, i.e., [3,3*N] */
78 PetscCall(VecCreateMPIWithArray(PETSC_COMM_WORLD, 1, nleaves, PETSC_DECIDE, leafupdate, &gy2));
79 PetscCall(PetscObjectSetName((PetscObject)x, "rootdata"));
80 PetscCall(PetscObjectSetName((PetscObject)gy2, "leafupdate"));
81
82 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
83 PetscCall(VecView(gy2, PETSC_VIEWER_STDOUT_WORLD));
84 PetscCall(VecDestroy(&gy2));
85
86 PetscCall(VecRestoreArray(y2, &leafupdate));
87 PetscCall(VecDestroy(&y2));
88
89 PetscCall(VecRestoreArray(y, &leafdata));
90 PetscCall(VecDestroy(&y));
91
92 PetscCall(VecRestoreArray(x, &rootdata));
93 /* PetscCall(VecDestroy(&x)); */ /* We will reuse x in ALLGATHER, so do not destroy it */
94
95 PetscCall(PetscSFDestroy(&gathersf));
96
97 /*-------------------------------------*/
98 /* PETSCSF_PATTERN_ALLGATHER */
99 /*-------------------------------------*/
100
101 /* set MPI vec x to [1, 2, .., N] */
102 for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i + 1.0, INSERT_VALUES));
103 PetscCall(VecAssemblyBegin(x));
104 PetscCall(VecAssemblyEnd(x));
105
106 /* Create the allgather SF */
107 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_ALLGATHER graph with op = %s\n", mpiopname));
108 PetscCall(VecGetLayout(x, &layout));
109 PetscCall(PetscSFCreate(PETSC_COMM_WORLD, &allgathersf));
110 PetscCall(PetscSFSetGraphWithPattern(allgathersf, layout, PETSCSF_PATTERN_ALLGATHER));
111
112 /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */
113 PetscCall(PetscSFGetGraph(allgathersf, NULL, &nleaves, NULL, NULL));
114 PetscCall(VecCreateSeq(PETSC_COMM_SELF, nleaves, &y));
115 PetscCall(VecDuplicate(y, &y2));
116
117 PetscCall(VecGetArray(x, &rootdata));
118 PetscCall(VecGetArray(y, &leafdata));
119 PetscCall(VecGetArray(y2, &leafupdate));
120
121 /* Bcast x to y, to initialize y = [1,N], then scale y to make leafupdate = y = [2,2*N] */
122 PetscCall(PetscSFBcastBegin(allgathersf, MPIU_SCALAR, rootdata, leafdata, MPI_REPLACE));
123 PetscCall(PetscSFBcastEnd(allgathersf, MPIU_SCALAR, rootdata, leafdata, MPI_REPLACE));
124 PetscCall(VecRestoreArray(y, &leafdata));
125 PetscCall(VecScale(y, 2));
126 PetscCall(VecGetArray(y, &leafdata));
127
128 /* FetchAndOp x to y */
129 PetscCall(PetscSFFetchAndOpBegin(allgathersf, MPIU_SCALAR, rootdata, leafdata, leafupdate, op));
130 PetscCall(PetscSFFetchAndOpEnd(allgathersf, MPIU_SCALAR, rootdata, leafdata, leafupdate, op));
131
132 /* View roots (x) and leafupdate (y2). Since this is an allgather graph, we have (suppose ranks get updates in ascending order)
133 rank 0: leafupdate = rootdata = [1,N], rootdata += leafdata = [3,3*N]
134 rank 1: leafupdate = rootdata = [3,3*N], rootdata += leafdata = [5,5*N]
135 rank 2: leafupdate = rootdata = [5,5*N], rootdata += leafdata = [7,7*N]
136 ...
137 */
138 PetscCall(VecCreateMPIWithArray(PETSC_COMM_WORLD, 1, nleaves, PETSC_DECIDE, leafupdate, &gy2));
139 PetscCall(PetscObjectSetName((PetscObject)x, "rootdata"));
140 PetscCall(PetscObjectSetName((PetscObject)gy2, "leafupdate"));
141
142 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
143 PetscCall(VecView(gy2, PETSC_VIEWER_STDOUT_WORLD));
144 PetscCall(VecDestroy(&gy2));
145
146 PetscCall(VecRestoreArray(y2, &leafupdate));
147 PetscCall(VecDestroy(&y2));
148
149 PetscCall(VecRestoreArray(y, &leafdata));
150 PetscCall(VecDestroy(&y));
151
152 PetscCall(VecRestoreArray(x, &rootdata));
153 PetscCall(VecDestroy(&x)); /* We won't reuse x in ALLGATHER, so destroy it */
154
155 PetscCall(PetscSFDestroy(&allgathersf));
156
157 /*-------------------------------------*/
158 /* PETSCSF_PATTERN_ALLTOALL */
159 /*-------------------------------------*/
160
161 PetscCall(VecCreate(PETSC_COMM_WORLD, &x));
162 PetscCall(VecSetFromOptions(x));
163 PetscCall(VecSetSizes(x, size, PETSC_DECIDE));
164
165 /* set MPI vec x to [1, 2, .., size^2] */
166 PetscCall(VecGetOwnershipRange(x, &low, &high));
167 for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i + 1.0, INSERT_VALUES));
168 PetscCall(VecAssemblyBegin(x));
169 PetscCall(VecAssemblyEnd(x));
170
171 /* Create the alltoall SF */
172 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_ALLTOALL graph with op = %s\n", mpiopname));
173 PetscCall(PetscSFCreate(PETSC_COMM_WORLD, &alltoallsf));
174 PetscCall(PetscSFSetGraphWithPattern(alltoallsf, NULL /*insignificant*/, PETSCSF_PATTERN_ALLTOALL));
175
176 /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */
177 PetscCall(PetscSFGetGraph(alltoallsf, NULL, &nleaves, NULL, NULL));
178 PetscCall(VecCreateSeq(PETSC_COMM_SELF, nleaves, &y));
179 PetscCall(VecDuplicate(y, &y2));
180
181 PetscCall(VecGetArray(x, &rootdata));
182 PetscCall(VecGetArray(y, &leafdata));
183 PetscCall(VecGetArray(y2, &leafupdate));
184
185 /* Bcast x to y, to initialize y = 1+rank+size*i, with i=0..size-1 */
186 PetscCall(PetscSFBcastBegin(alltoallsf, MPIU_SCALAR, rootdata, leafdata, MPI_REPLACE));
187 PetscCall(PetscSFBcastEnd(alltoallsf, MPIU_SCALAR, rootdata, leafdata, MPI_REPLACE));
188
189 /* FetchAndOp x to y */
190 PetscCall(PetscSFFetchAndOpBegin(alltoallsf, MPIU_SCALAR, rootdata, leafdata, leafupdate, op));
191 PetscCall(PetscSFFetchAndOpEnd(alltoallsf, MPIU_SCALAR, rootdata, leafdata, leafupdate, op));
192
193 /* View roots (x) and leafupdate (y2). Since this is an alltoall graph, each root has only one leaf.
194 So, leafupdate = rootdata = 1+rank+size*i, i=0..size-1; and rootdata += leafdata, i.e., rootdata = [2,2*N]
195 */
196 PetscCall(VecCreateMPIWithArray(PETSC_COMM_WORLD, 1, nleaves, PETSC_DECIDE, leafupdate, &gy2));
197 PetscCall(PetscObjectSetName((PetscObject)x, "rootdata"));
198 PetscCall(PetscObjectSetName((PetscObject)gy2, "leafupdate"));
199
200 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
201 PetscCall(VecView(gy2, PETSC_VIEWER_STDOUT_WORLD));
202 PetscCall(VecDestroy(&gy2));
203
204 PetscCall(VecRestoreArray(y2, &leafupdate));
205 PetscCall(VecDestroy(&y2));
206
207 PetscCall(VecRestoreArray(y, &leafdata));
208 PetscCall(VecDestroy(&y));
209
210 PetscCall(VecRestoreArray(x, &rootdata));
211 PetscCall(VecDestroy(&x));
212
213 PetscCall(PetscSFDestroy(&alltoallsf));
214
215 PetscCall(PetscFinalize());
216 return 0;
217 }
218
219 /*TEST
220
221 test:
222 # N=10 is divisible by nsize, to trigger Allgather/Gather in SF
223 #MPI_Sendrecv_replace is broken with 20210400300
224 requires: !defined(PETSC_HAVE_I_MPI)
225 nsize: 2
226 args: -op replace
227
228 test:
229 suffix: 2
230 nsize: 2
231 args: -op sum
232
233 # N=10 is not divisible by nsize, to trigger Allgatherv/Gatherv in SF
234 test:
235 #MPI_Sendrecv_replace is broken with 20210400300
236 requires: !defined(PETSC_HAVE_I_MPI)
237 suffix: 3
238 nsize: 3
239 args: -op replace
240
241 test:
242 suffix: 4
243 nsize: 3
244 args: -op sum
245
246 TEST*/
247