1 static char help[] = "Test VecScatterCreateToZero, VecScatterCreateToAll\n\n";
2
3 #include <petscvec.h>
main(int argc,char ** argv)4 int main(int argc, char **argv)
5 {
6 PetscInt i, N = 10, n = PETSC_DECIDE, low, high, onlylocal = -1;
7 PetscMPIInt size, rank;
8 Vec x, y;
9 VecScatter vscat;
10
11 PetscFunctionBeginUser;
12 PetscCall(PetscInitialize(&argc, &argv, NULL, help));
13 PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
14 PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
15 PetscCall(PetscOptionsGetInt(NULL, NULL, "-n", &N, NULL));
16
17 /* Trigger special case in VecScatterCreateToAll to deal with the one-to-all pattern */
18 PetscCall(PetscOptionsGetInt(NULL, NULL, "-onlylocal", &onlylocal, NULL));
19 if (onlylocal >= 0 && onlylocal < size) n = (rank == onlylocal ? N : 0);
20
21 PetscCall(VecCreate(PETSC_COMM_WORLD, &x));
22 PetscCall(VecSetFromOptions(x));
23 PetscCall(VecSetSizes(x, n, N));
24 PetscCall(VecGetOwnershipRange(x, &low, &high));
25 PetscCall(PetscObjectSetName((PetscObject)x, "x"));
26
27 /*-------------------------------------*/
28 /* VecScatterCreateToZero */
29 /*-------------------------------------*/
30
31 /* MPI vec x = [0, 1, 2, .., N-1] */
32 for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i, INSERT_VALUES));
33 PetscCall(VecAssemblyBegin(x));
34 PetscCall(VecAssemblyEnd(x));
35
36 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting VecScatterCreateToZero\n"));
37 PetscCall(VecScatterCreateToZero(x, &vscat, &y));
38 PetscCall(PetscObjectSetName((PetscObject)y, "y"));
39
40 /* Test PetscSFBcastAndOp with op = MPI_REPLACE, which does y = x on rank 0 */
41 PetscCall(VecScatterBegin(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
42 PetscCall(VecScatterEnd(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
43 if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));
44
45 /* Test PetscSFBcastAndOp with op = MPI_SUM, which does y += x */
46 PetscCall(VecScatterBegin(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
47 PetscCall(VecScatterEnd(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
48 if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));
49
50 /* Test PetscSFReduce with op = MPI_REPLACE, which does x = y */
51 PetscCall(VecScatterBegin(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
52 PetscCall(VecScatterEnd(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
53 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
54
55 /* Test PetscSFReduce with op = MPI_SUM, which does x += y on x's local part on rank 0*/
56 PetscCall(VecScatterBegin(vscat, y, x, ADD_VALUES, SCATTER_REVERSE_LOCAL));
57 PetscCall(VecScatterEnd(vscat, y, x, ADD_VALUES, SCATTER_REVERSE_LOCAL));
58 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
59
60 PetscCall(VecDestroy(&y));
61 PetscCall(VecScatterDestroy(&vscat));
62
63 /*-------------------------------------*/
64 /* VecScatterCreateToAll */
65 /*-------------------------------------*/
66 for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i, INSERT_VALUES));
67 PetscCall(VecAssemblyBegin(x));
68 PetscCall(VecAssemblyEnd(x));
69
70 PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting VecScatterCreateToAll\n"));
71
72 PetscCall(VecScatterCreateToAll(x, &vscat, &y));
73 PetscCall(PetscObjectSetName((PetscObject)y, "y"));
74
75 /* Test PetscSFBcastAndOp with op = MPI_REPLACE, which does y = x on all ranks */
76 PetscCall(VecScatterBegin(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
77 PetscCall(VecScatterEnd(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
78 if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));
79
80 /* Test PetscSFBcastAndOp with op = MPI_SUM, which does y += x */
81 PetscCall(VecScatterBegin(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
82 PetscCall(VecScatterEnd(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
83 if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));
84
85 /* Test PetscSFReduce with op = MPI_REPLACE, which does x = y */
86 PetscCall(VecScatterBegin(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
87 PetscCall(VecScatterEnd(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
88 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
89
90 /* Test PetscSFReduce with op = MPI_SUM, which does x += size*y */
91 PetscCall(VecScatterBegin(vscat, y, x, ADD_VALUES, SCATTER_REVERSE));
92 PetscCall(VecScatterEnd(vscat, y, x, ADD_VALUES, SCATTER_REVERSE));
93 PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
94 PetscCall(VecDestroy(&x));
95 PetscCall(VecDestroy(&y));
96 PetscCall(VecScatterDestroy(&vscat));
97
98 PetscCall(PetscFinalize());
99 return 0;
100 }
101
102 /*TEST
103
104 testset:
105 # N=10 is divisible by nsize, to trigger Allgather/Gather in SF
106 nsize: 2
107 # Exact numbers really matter here
108 diff_args: -j
109 filter: grep -v "type"
110 output_file: output/ex8_1.out
111
112 test:
113 suffix: 1_standard
114
115 test:
116 suffix: 1_cuda
117 # sf_backend cuda is not needed if compiling only with cuda
118 args: -vec_type cuda -sf_backend cuda
119 requires: cuda
120
121 test:
122 suffix: 1_hip
123 args: -vec_type hip -sf_backend hip
124 requires: hip
125
126 test:
127 suffix: 1_cuda_aware_mpi
128 # sf_backend cuda is not needed if compiling only with cuda
129 args: -vec_type cuda -sf_backend cuda
130 requires: cuda defined(PETSC_HAVE_MPI_GPU_AWARE)
131
132 testset:
133 # N=10 is not divisible by nsize, to trigger Allgatherv/Gatherv in SF
134 nsize: 3
135 # Exact numbers really matter here
136 diff_args: -j
137 filter: grep -v "type" | grep -v "Process "
138 output_file: output/ex8_2.out
139
140 test:
141 suffix: 2_standard
142
143 test:
144 suffix: 2_cuda
145 # sf_backend cuda is not needed if compiling only with cuda
146 args: -vec_type cuda -sf_backend cuda
147 requires: cuda
148
149 test:
150 suffix: 2_hip
151 # sf_backend hip is not needed if compiling only with hip
152 args: -vec_type hip -sf_backend hip
153 requires: hip
154
155 test:
156 suffix: 2_cuda_aware_mpi
157 args: -vec_type cuda
158 requires: cuda defined(PETSC_HAVE_MPI_GPU_AWARE)
159
160 testset:
161 # trigger one-to-all pattern in Allgatherv
162 nsize: 3
163 diff_args: -j
164 filter: grep -v "type" | grep -v "Process "
165 output_file: output/ex8_3.out
166 args: -onlylocal 1
167
168 test:
169 suffix: 2_standard_onetoall
170
171 test:
172 suffix: 2_cuda_onetoall
173 # sf_backend cuda is not needed if compiling only with cuda
174 args: -vec_type cuda -sf_backend cuda
175 requires: cuda
176
177 test:
178 suffix: 2_hip_onetoall
179 # sf_backend hip is not needed if compiling only with hip
180 args: -vec_type hip -sf_backend hip
181 requires: hip
182
183 test:
184 suffix: 2_cuda_aware_mpi_onetoall
185 args: -vec_type cuda
186 requires: cuda defined(PETSC_HAVE_MPI_GPU_AWARE)
187
188 TEST*/
189