1 static char help[]= "Test PetscSFFetchAndOp on patterned SF graphs. PetscSFFetchAndOp internally uses PetscSFBcastAndOp \n\ 2 and PetscSFReduce. So it is a good test to see if they all work for patterned graphs.\n\ 3 Run with ./prog -op [replace | sum]\n\n"; 4 5 #include <petscvec.h> 6 #include <petscsf.h> 7 int main(int argc,char **argv) 8 { 9 PetscErrorCode ierr; 10 PetscInt i,N=10,low,high,nleaves; 11 PetscMPIInt size,rank; 12 Vec x,y,y2,gy2; 13 PetscScalar *rootdata,*leafdata,*leafupdate; 14 PetscLayout layout; 15 PetscSF gathersf,allgathersf,alltoallsf; 16 MPI_Op op=MPI_SUM; 17 char opname[64]; 18 const char *mpiopname; 19 PetscBool flag,isreplace,issum; 20 21 ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; 22 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRMPI(ierr); 23 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRMPI(ierr); 24 25 ierr = PetscOptionsGetString(NULL,NULL,"-op",opname,sizeof(opname),&flag);CHKERRQ(ierr); 26 ierr = PetscStrcmp(opname,"replace",&isreplace);CHKERRQ(ierr); 27 ierr = PetscStrcmp(opname,"sum",&issum);CHKERRQ(ierr); 28 29 if (isreplace) {op = MPI_REPLACE; mpiopname = "MPI_REPLACE";} 30 else if (issum) {op = MPIU_SUM; mpiopname = "MPI_SUM";} 31 else SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,"Unsupported argument (%s) to -op, which must be 'replace' or 'sum'",opname); 32 33 ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); 34 ierr = VecSetFromOptions(x);CHKERRQ(ierr); 35 ierr = VecSetSizes(x,PETSC_DECIDE,N);CHKERRQ(ierr); 36 37 /*-------------------------------------*/ 38 /* PETSCSF_PATTERN_GATHER */ 39 /*-------------------------------------*/ 40 41 /* set MPI vec x to [1, 2, .., N] */ 42 ierr = VecGetOwnershipRange(x,&low,&high);CHKERRQ(ierr); 43 for (i=low; i<high; i++) {ierr = VecSetValue(x,i,(PetscScalar)i+1.0,INSERT_VALUES);CHKERRQ(ierr);} 44 ierr = VecAssemblyBegin(x);CHKERRQ(ierr); 45 ierr = VecAssemblyEnd(x);CHKERRQ(ierr); 46 47 /* Create the gather SF */ 48 ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_GATHER graph with op = %s\n",mpiopname);CHKERRQ(ierr); 49 ierr = VecGetLayout(x,&layout);CHKERRQ(ierr); 50 ierr = PetscSFCreate(PETSC_COMM_WORLD,&gathersf);CHKERRQ(ierr); 51 ierr = PetscSFSetGraphWithPattern(gathersf,layout,PETSCSF_PATTERN_GATHER);CHKERRQ(ierr); 52 53 /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */ 54 ierr = PetscSFGetGraph(gathersf,NULL,&nleaves,NULL,NULL);CHKERRQ(ierr); 55 ierr = VecCreateSeq(PETSC_COMM_SELF,nleaves,&y);CHKERRQ(ierr); 56 ierr = VecDuplicate(y,&y2);CHKERRQ(ierr); 57 58 ierr = VecGetArray(x,&rootdata);CHKERRQ(ierr); 59 ierr = VecGetArray(y,&leafdata);CHKERRQ(ierr); 60 ierr = VecGetArray(y2,&leafupdate);CHKERRQ(ierr); 61 62 /* Bcast x to y,to initialize y = [1,N], then scale y to make leafupdate = y = [2,2*N] */ 63 ierr = PetscSFBcastBegin(gathersf,MPIU_SCALAR,rootdata,leafdata,MPI_REPLACE);CHKERRQ(ierr); 64 ierr = PetscSFBcastEnd(gathersf,MPIU_SCALAR,rootdata,leafdata,MPI_REPLACE);CHKERRQ(ierr); 65 ierr = VecRestoreArray(y,&leafdata);CHKERRQ(ierr); 66 ierr = VecScale(y,2);CHKERRQ(ierr); 67 ierr = VecGetArray(y,&leafdata);CHKERRQ(ierr); 68 69 /* FetchAndOp x to y */ 70 ierr = PetscSFFetchAndOpBegin(gathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);CHKERRQ(ierr); 71 ierr = PetscSFFetchAndOpEnd(gathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);CHKERRQ(ierr); 72 73 /* View roots (x) and leafupdate (y2). Since this is a gather graph, leafudpate = rootdata = [1,N], then rootdata += leafdata, i.e., [3,3*N] */ 74 ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,nleaves,PETSC_DECIDE,leafupdate,&gy2);CHKERRQ(ierr); 75 ierr = PetscObjectSetName((PetscObject)x,"rootdata");CHKERRQ(ierr); 76 ierr = PetscObjectSetName((PetscObject)gy2,"leafupdate");CHKERRQ(ierr); 77 78 ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); 79 ierr = VecView(gy2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); 80 ierr = VecDestroy(&gy2);CHKERRQ(ierr); 81 82 ierr = VecRestoreArray(y2,&leafupdate);CHKERRQ(ierr); 83 ierr = VecDestroy(&y2);CHKERRQ(ierr); 84 85 ierr = VecRestoreArray(y,&leafdata);CHKERRQ(ierr); 86 ierr = VecDestroy(&y);CHKERRQ(ierr); 87 88 ierr = VecRestoreArray(x,&rootdata);CHKERRQ(ierr); 89 /* ierr = VecDestroy(&x);CHKERRQ(ierr); */ /* We will reuse x in ALLGATHER, so do not destroy it */ 90 91 ierr = PetscSFDestroy(&gathersf);CHKERRQ(ierr); 92 93 /*-------------------------------------*/ 94 /* PETSCSF_PATTERN_ALLGATHER */ 95 /*-------------------------------------*/ 96 97 /* set MPI vec x to [1, 2, .., N] */ 98 for (i=low; i<high; i++) {ierr = VecSetValue(x,i,(PetscScalar)i+1.0,INSERT_VALUES);CHKERRQ(ierr);} 99 ierr = VecAssemblyBegin(x);CHKERRQ(ierr); 100 ierr = VecAssemblyEnd(x);CHKERRQ(ierr); 101 102 /* Create the allgather SF */ 103 ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_ALLGATHER graph with op = %s\n",mpiopname);CHKERRQ(ierr); 104 ierr = VecGetLayout(x,&layout);CHKERRQ(ierr); 105 ierr = PetscSFCreate(PETSC_COMM_WORLD,&allgathersf);CHKERRQ(ierr); 106 ierr = PetscSFSetGraphWithPattern(allgathersf,layout,PETSCSF_PATTERN_ALLGATHER);CHKERRQ(ierr); 107 108 /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */ 109 ierr = PetscSFGetGraph(allgathersf,NULL,&nleaves,NULL,NULL);CHKERRQ(ierr); 110 ierr = VecCreateSeq(PETSC_COMM_SELF,nleaves,&y);CHKERRQ(ierr); 111 ierr = VecDuplicate(y,&y2);CHKERRQ(ierr); 112 113 ierr = VecGetArray(x,&rootdata);CHKERRQ(ierr); 114 ierr = VecGetArray(y,&leafdata);CHKERRQ(ierr); 115 ierr = VecGetArray(y2,&leafupdate);CHKERRQ(ierr); 116 117 /* Bcast x to y, to initialize y = [1,N], then scale y to make leafupdate = y = [2,2*N] */ 118 ierr = PetscSFBcastBegin(allgathersf,MPIU_SCALAR,rootdata,leafdata,MPI_REPLACE);CHKERRQ(ierr); 119 ierr = PetscSFBcastEnd(allgathersf,MPIU_SCALAR,rootdata,leafdata,MPI_REPLACE);CHKERRQ(ierr); 120 ierr = VecRestoreArray(y,&leafdata);CHKERRQ(ierr); 121 ierr = VecScale(y,2);CHKERRQ(ierr); 122 ierr = VecGetArray(y,&leafdata);CHKERRQ(ierr); 123 124 /* FetchAndOp x to y */ 125 ierr = PetscSFFetchAndOpBegin(allgathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);CHKERRQ(ierr); 126 ierr = PetscSFFetchAndOpEnd(allgathersf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);CHKERRQ(ierr); 127 128 /* View roots (x) and leafupdate (y2). Since this is an allgather graph, we have (suppose ranks get updates in ascending order) 129 rank 0: leafupdate = rootdata = [1,N], rootdata += leafdata = [3,3*N] 130 rank 1: leafupdate = rootdata = [3,3*N], rootdata += leafdata = [5,5*N] 131 rank 2: leafupdate = rootdata = [5,5*N], rootdata += leafdata = [7,7*N] 132 ... 133 */ 134 ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,nleaves,PETSC_DECIDE,leafupdate,&gy2);CHKERRQ(ierr); 135 ierr = PetscObjectSetName((PetscObject)x,"rootdata");CHKERRQ(ierr); 136 ierr = PetscObjectSetName((PetscObject)gy2,"leafupdate");CHKERRQ(ierr); 137 138 ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); 139 ierr = VecView(gy2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); 140 ierr = VecDestroy(&gy2);CHKERRQ(ierr); 141 142 ierr = VecRestoreArray(y2,&leafupdate);CHKERRQ(ierr); 143 ierr = VecDestroy(&y2);CHKERRQ(ierr); 144 145 ierr = VecRestoreArray(y,&leafdata);CHKERRQ(ierr); 146 ierr = VecDestroy(&y);CHKERRQ(ierr); 147 148 ierr = VecRestoreArray(x,&rootdata);CHKERRQ(ierr); 149 ierr = VecDestroy(&x);CHKERRQ(ierr); /* We won't reuse x in ALLGATHER, so destroy it */ 150 151 ierr = PetscSFDestroy(&allgathersf);CHKERRQ(ierr); 152 153 /*-------------------------------------*/ 154 /* PETSCSF_PATTERN_ALLTOALL */ 155 /*-------------------------------------*/ 156 157 ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); 158 ierr = VecSetFromOptions(x);CHKERRQ(ierr); 159 ierr = VecSetSizes(x,size,PETSC_DECIDE);CHKERRQ(ierr); 160 161 /* set MPI vec x to [1, 2, .., size^2] */ 162 ierr = VecGetOwnershipRange(x,&low,&high);CHKERRQ(ierr); 163 for (i=low; i<high; i++) {ierr = VecSetValue(x,i,(PetscScalar)i+1.0,INSERT_VALUES);CHKERRQ(ierr);} 164 ierr = VecAssemblyBegin(x);CHKERRQ(ierr); 165 ierr = VecAssemblyEnd(x);CHKERRQ(ierr); 166 167 /* Create the alltoall SF */ 168 ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTesting PetscSFFetchAndOp on a PETSCSF_PATTERN_ALLTOALL graph with op = %s\n",mpiopname);CHKERRQ(ierr); 169 ierr = PetscSFCreate(PETSC_COMM_WORLD,&alltoallsf);CHKERRQ(ierr); 170 ierr = PetscSFSetGraphWithPattern(alltoallsf,NULL/*insignificant*/,PETSCSF_PATTERN_ALLTOALL);CHKERRQ(ierr); 171 172 /* Create the leaf vector y (seq vector) and its duplicate y2 working as leafupdate */ 173 ierr = PetscSFGetGraph(alltoallsf,NULL,&nleaves,NULL,NULL);CHKERRQ(ierr); 174 ierr = VecCreateSeq(PETSC_COMM_SELF,nleaves,&y);CHKERRQ(ierr); 175 ierr = VecDuplicate(y,&y2);CHKERRQ(ierr); 176 177 ierr = VecGetArray(x,&rootdata);CHKERRQ(ierr); 178 ierr = VecGetArray(y,&leafdata);CHKERRQ(ierr); 179 ierr = VecGetArray(y2,&leafupdate);CHKERRQ(ierr); 180 181 /* Bcast x to y, to initialize y = 1+rank+size*i, with i=0..size-1 */ 182 ierr = PetscSFBcastBegin(alltoallsf,MPIU_SCALAR,rootdata,leafdata,MPI_REPLACE);CHKERRQ(ierr); 183 ierr = PetscSFBcastEnd(alltoallsf,MPIU_SCALAR,rootdata,leafdata,MPI_REPLACE);CHKERRQ(ierr); 184 185 /* FetchAndOp x to y */ 186 ierr = PetscSFFetchAndOpBegin(alltoallsf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);CHKERRQ(ierr); 187 ierr = PetscSFFetchAndOpEnd(alltoallsf,MPIU_SCALAR,rootdata,leafdata,leafupdate,op);CHKERRQ(ierr); 188 189 /* View roots (x) and leafupdate (y2). Since this is an alltoall graph, each root has only one leaf. 190 So, leafupdate = rootdata = 1+rank+size*i, i=0..size-1; and rootdata += leafdata, i.e., rootdata = [2,2*N] 191 */ 192 ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,nleaves,PETSC_DECIDE,leafupdate,&gy2);CHKERRQ(ierr); 193 ierr = PetscObjectSetName((PetscObject)x,"rootdata");CHKERRQ(ierr); 194 ierr = PetscObjectSetName((PetscObject)gy2,"leafupdate");CHKERRQ(ierr); 195 196 ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); 197 ierr = VecView(gy2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); 198 ierr = VecDestroy(&gy2);CHKERRQ(ierr); 199 200 ierr = VecRestoreArray(y2,&leafupdate);CHKERRQ(ierr); 201 ierr = VecDestroy(&y2);CHKERRQ(ierr); 202 203 ierr = VecRestoreArray(y,&leafdata);CHKERRQ(ierr); 204 ierr = VecDestroy(&y);CHKERRQ(ierr); 205 206 ierr = VecRestoreArray(x,&rootdata);CHKERRQ(ierr); 207 ierr = VecDestroy(&x);CHKERRQ(ierr); 208 209 ierr = PetscSFDestroy(&alltoallsf);CHKERRQ(ierr); 210 211 ierr = PetscFinalize(); 212 return ierr; 213 } 214 215 /*TEST 216 217 test: 218 # N=10 is divisible by nsize, to trigger Allgather/Gather in SF 219 nsize: 2 220 args: -op replace 221 222 test: 223 suffix: 2 224 nsize: 2 225 args: -op sum 226 227 # N=10 is not divisible by nsize, to trigger Allgatherv/Gatherv in SF 228 test: 229 suffix: 3 230 nsize: 3 231 args: -op replace 232 233 test: 234 suffix: 4 235 nsize: 3 236 args: -op sum 237 238 TEST*/ 239 240