1 static char help[] = "Test that MatPartitioning and PetscPartitioner interfaces are equivalent when using PETSCPARTITIONERMATPARTITIONING\n\n"; 2 static char FILENAME[] = "ex24.c"; 3 4 #include <petscdmplex.h> 5 #include <petscviewerhdf5.h> 6 7 #if defined(PETSC_HAVE_PTSCOTCH) 8 EXTERN_C_BEGIN 9 #include <ptscotch.h> 10 EXTERN_C_END 11 #endif 12 13 typedef struct { 14 PetscBool compare_is; /* Compare ISs and PetscSections */ 15 PetscBool compare_dm; /* Compare DM */ 16 PetscBool tpw; /* Use target partition weights */ 17 char partitioning[64]; 18 char repartitioning[64]; 19 } AppCtx; 20 21 static PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options) 22 { 23 PetscBool repartition = PETSC_TRUE; 24 PetscErrorCode ierr; 25 26 PetscFunctionBegin; 27 options->compare_is = PETSC_FALSE; 28 options->compare_dm = PETSC_FALSE; 29 30 ierr = PetscOptionsBegin(comm, "", "Meshing Interpolation Test Options", "DMPLEX");CHKERRQ(ierr); 31 ierr = PetscOptionsBool("-compare_is", "Compare ISs and PetscSections?", FILENAME, options->compare_is, &options->compare_is, NULL);CHKERRQ(ierr); 32 ierr = PetscOptionsBool("-compare_dm", "Compare DMs?", FILENAME, options->compare_dm, &options->compare_dm, NULL);CHKERRQ(ierr); 33 ierr = PetscStrncpy(options->partitioning,MATPARTITIONINGPARMETIS,sizeof(options->partitioning));CHKERRQ(ierr); 34 ierr = PetscOptionsString("-partitioning","The mat partitioning type to test","None",options->partitioning, options->partitioning,sizeof(options->partitioning),NULL);CHKERRQ(ierr); 35 ierr = PetscOptionsBool("-repartition", "Partition again after the first partition?", FILENAME, repartition, &repartition, NULL);CHKERRQ(ierr); 36 if (repartition) { 37 ierr = PetscStrncpy(options->repartitioning,MATPARTITIONINGPARMETIS,64);CHKERRQ(ierr); 38 ierr = PetscOptionsString("-repartitioning","The mat partitioning type to test (second partitioning)","None", options->repartitioning, options->repartitioning,sizeof(options->repartitioning),NULL);CHKERRQ(ierr); 39 } else { 40 options->repartitioning[0] = '\0'; 41 } 42 ierr = PetscOptionsBool("-tpweight", "Use target partition weights", FILENAME, options->tpw, &options->tpw, NULL);CHKERRQ(ierr); 43 ierr = PetscOptionsEnd();CHKERRQ(ierr); 44 PetscFunctionReturn(0); 45 } 46 47 static PetscErrorCode ScotchResetRandomSeed() 48 { 49 PetscFunctionBegin; 50 #if defined(PETSC_HAVE_PTSCOTCH) 51 SCOTCH_randomReset(); 52 #endif 53 PetscFunctionReturn(0); 54 } 55 56 static PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm) 57 { 58 PetscErrorCode ierr; 59 60 PetscFunctionBegin; 61 ierr = DMCreate(comm, dm);CHKERRQ(ierr); 62 ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr); 63 ierr = DMPlexDistributeSetDefault(*dm, PETSC_FALSE);CHKERRQ(ierr); 64 ierr = DMSetFromOptions(*dm);CHKERRQ(ierr); 65 ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr); 66 PetscFunctionReturn(0); 67 } 68 69 int main(int argc, char **argv) 70 { 71 MPI_Comm comm; 72 DM dm1, dm2, dmdist1, dmdist2; 73 DMPlexInterpolatedFlag interp; 74 MatPartitioning mp; 75 PetscPartitioner part1, part2; 76 AppCtx user; 77 IS is1=NULL, is2=NULL; 78 IS is1g, is2g; 79 PetscSection s1=NULL, s2=NULL, tpws = NULL; 80 PetscInt i; 81 PetscBool flg; 82 PetscErrorCode ierr; 83 PetscMPIInt size; 84 85 ierr = PetscInitialize(&argc, &argv, NULL,help);if (ierr) return ierr; 86 comm = PETSC_COMM_WORLD; 87 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 88 ierr = ProcessOptions(comm, &user);CHKERRQ(ierr); 89 ierr = CreateMesh(comm, &user, &dm1);CHKERRQ(ierr); 90 ierr = CreateMesh(comm, &user, &dm2);CHKERRQ(ierr); 91 92 if (user.tpw) { 93 ierr = PetscSectionCreate(comm, &tpws);CHKERRQ(ierr); 94 ierr = PetscSectionSetChart(tpws, 0, size);CHKERRQ(ierr); 95 for (i=0;i<size;i++) { 96 PetscInt tdof = i%2 ? 2*i -1 : i+2; 97 ierr = PetscSectionSetDof(tpws, i, tdof);CHKERRQ(ierr); 98 } 99 if (size > 1) { /* test zero tpw entry */ 100 ierr = PetscSectionSetDof(tpws, 0, 0);CHKERRQ(ierr); 101 } 102 ierr = PetscSectionSetUp(tpws);CHKERRQ(ierr); 103 } 104 105 /* partition dm1 using PETSCPARTITIONERPARMETIS */ 106 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 107 ierr = DMPlexGetPartitioner(dm1, &part1);CHKERRQ(ierr); 108 ierr = PetscObjectSetOptionsPrefix((PetscObject)part1,"p1_");CHKERRQ(ierr); 109 ierr = PetscPartitionerSetType(part1, user.partitioning);CHKERRQ(ierr); 110 ierr = PetscPartitionerSetFromOptions(part1);CHKERRQ(ierr); 111 ierr = PetscSectionCreate(comm, &s1);CHKERRQ(ierr); 112 ierr = PetscPartitionerDMPlexPartition(part1, dm1, tpws, s1, &is1);CHKERRQ(ierr); 113 114 /* partition dm2 using PETSCPARTITIONERMATPARTITIONING with MATPARTITIONINGPARMETIS */ 115 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 116 ierr = DMPlexGetPartitioner(dm2, &part2);CHKERRQ(ierr); 117 ierr = PetscObjectSetOptionsPrefix((PetscObject)part2,"p2_");CHKERRQ(ierr); 118 ierr = PetscPartitionerSetType(part2, PETSCPARTITIONERMATPARTITIONING);CHKERRQ(ierr); 119 ierr = PetscPartitionerMatPartitioningGetMatPartitioning(part2, &mp);CHKERRQ(ierr); 120 ierr = MatPartitioningSetType(mp, user.partitioning);CHKERRQ(ierr); 121 ierr = PetscPartitionerSetFromOptions(part2);CHKERRQ(ierr); 122 ierr = PetscSectionCreate(comm, &s2);CHKERRQ(ierr); 123 ierr = PetscPartitionerDMPlexPartition(part2, dm2, tpws, s2, &is2);CHKERRQ(ierr); 124 125 ierr = ISOnComm(is1, comm, PETSC_USE_POINTER, &is1g);CHKERRQ(ierr); 126 ierr = ISOnComm(is2, comm, PETSC_USE_POINTER, &is2g);CHKERRQ(ierr); 127 ierr = ISViewFromOptions(is1g, NULL, "-seq_is1_view");CHKERRQ(ierr); 128 ierr = ISViewFromOptions(is2g, NULL, "-seq_is2_view");CHKERRQ(ierr); 129 /* compare the two ISs */ 130 if (user.compare_is) { 131 ierr = ISEqualUnsorted(is1g, is2g, &flg);CHKERRQ(ierr); 132 if (!flg) {ierr = PetscPrintf(comm, "ISs are not equal with type %s with size %d.\n",user.partitioning,size);CHKERRQ(ierr);} 133 } 134 ierr = ISDestroy(&is1g);CHKERRQ(ierr); 135 ierr = ISDestroy(&is2g);CHKERRQ(ierr); 136 137 /* compare the two PetscSections */ 138 ierr = PetscSectionViewFromOptions(s1, NULL, "-seq_s1_view");CHKERRQ(ierr); 139 ierr = PetscSectionViewFromOptions(s2, NULL, "-seq_s2_view");CHKERRQ(ierr); 140 if (user.compare_is) { 141 ierr = PetscSectionCompare(s1, s2, &flg);CHKERRQ(ierr); 142 if (!flg) {ierr = PetscPrintf(comm, "PetscSections are not equal with %s with size %d.\n",user.partitioning,size);CHKERRQ(ierr);} 143 } 144 145 /* distribute both DMs */ 146 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 147 ierr = DMPlexDistribute(dm1, 0, NULL, &dmdist1);CHKERRQ(ierr); 148 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 149 ierr = DMPlexDistribute(dm2, 0, NULL, &dmdist2);CHKERRQ(ierr); 150 151 /* cleanup */ 152 ierr = PetscSectionDestroy(&tpws);CHKERRQ(ierr); 153 ierr = PetscSectionDestroy(&s1);CHKERRQ(ierr); 154 ierr = PetscSectionDestroy(&s2);CHKERRQ(ierr); 155 ierr = ISDestroy(&is1);CHKERRQ(ierr); 156 ierr = ISDestroy(&is2);CHKERRQ(ierr); 157 ierr = DMDestroy(&dm1);CHKERRQ(ierr); 158 ierr = DMDestroy(&dm2);CHKERRQ(ierr); 159 160 /* if distributed DMs are NULL (sequential case), then quit */ 161 if (!dmdist1 && !dmdist2) return ierr; 162 163 ierr = DMViewFromOptions(dmdist1, NULL, "-dm_dist1_view");CHKERRQ(ierr); 164 ierr = DMViewFromOptions(dmdist2, NULL, "-dm_dist2_view");CHKERRQ(ierr); 165 166 /* compare the two distributed DMs */ 167 if (user.compare_dm) { 168 ierr = DMPlexEqual(dmdist1, dmdist2, &flg);CHKERRQ(ierr); 169 if (!flg) {ierr = PetscPrintf(comm, "Distributed DMs are not equal %s with size %d.\n",user.partitioning,size);CHKERRQ(ierr);} 170 } 171 172 /* if repartitioning is disabled, then quit */ 173 if (user.repartitioning[0] == '\0') return ierr; 174 175 if (user.tpw) { 176 ierr = PetscSectionCreate(comm, &tpws);CHKERRQ(ierr); 177 ierr = PetscSectionSetChart(tpws, 0, size);CHKERRQ(ierr); 178 for (i=0;i<size;i++) { 179 PetscInt tdof = i%2 ? i+1 : size - i; 180 ierr = PetscSectionSetDof(tpws, i, tdof);CHKERRQ(ierr); 181 } 182 ierr = PetscSectionSetUp(tpws);CHKERRQ(ierr); 183 } 184 185 /* repartition distributed DM dmdist1 */ 186 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 187 ierr = DMPlexGetPartitioner(dmdist1, &part1);CHKERRQ(ierr); 188 ierr = PetscObjectSetOptionsPrefix((PetscObject)part1,"dp1_");CHKERRQ(ierr); 189 ierr = PetscPartitionerSetType(part1, user.repartitioning);CHKERRQ(ierr); 190 ierr = PetscPartitionerSetFromOptions(part1);CHKERRQ(ierr); 191 ierr = PetscSectionCreate(comm, &s1);CHKERRQ(ierr); 192 ierr = PetscPartitionerDMPlexPartition(part1, dmdist1, tpws, s1, &is1);CHKERRQ(ierr); 193 194 /* repartition distributed DM dmdist2 */ 195 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 196 ierr = DMPlexGetPartitioner(dmdist2, &part2);CHKERRQ(ierr); 197 ierr = PetscObjectSetOptionsPrefix((PetscObject)part2,"dp2_");CHKERRQ(ierr); 198 ierr = PetscPartitionerSetType(part2, PETSCPARTITIONERMATPARTITIONING);CHKERRQ(ierr); 199 ierr = PetscPartitionerMatPartitioningGetMatPartitioning(part2, &mp);CHKERRQ(ierr); 200 ierr = MatPartitioningSetType(mp, user.repartitioning);CHKERRQ(ierr); 201 ierr = PetscPartitionerSetFromOptions(part2);CHKERRQ(ierr); 202 ierr = PetscSectionCreate(comm, &s2);CHKERRQ(ierr); 203 ierr = PetscPartitionerDMPlexPartition(part2, dmdist2, tpws, s2, &is2);CHKERRQ(ierr); 204 205 /* compare the two ISs */ 206 ierr = ISOnComm(is1, comm, PETSC_USE_POINTER, &is1g);CHKERRQ(ierr); 207 ierr = ISOnComm(is2, comm, PETSC_USE_POINTER, &is2g);CHKERRQ(ierr); 208 ierr = ISViewFromOptions(is1g, NULL, "-dist_is1_view");CHKERRQ(ierr); 209 ierr = ISViewFromOptions(is2g, NULL, "-dist_is2_view");CHKERRQ(ierr); 210 if (user.compare_is) { 211 ierr = ISEqualUnsorted(is1g, is2g, &flg);CHKERRQ(ierr); 212 if (!flg) {ierr = PetscPrintf(comm, "Distributed ISs are not equal, with %s with size %d.\n",user.repartitioning,size);CHKERRQ(ierr);} 213 } 214 ierr = ISDestroy(&is1g);CHKERRQ(ierr); 215 ierr = ISDestroy(&is2g);CHKERRQ(ierr); 216 217 /* compare the two PetscSections */ 218 ierr = PetscSectionViewFromOptions(s1, NULL, "-dist_s1_view");CHKERRQ(ierr); 219 ierr = PetscSectionViewFromOptions(s2, NULL, "-dist_s2_view");CHKERRQ(ierr); 220 if (user.compare_is) { 221 ierr = PetscSectionCompare(s1, s2, &flg);CHKERRQ(ierr); 222 if (!flg) {ierr = PetscPrintf(comm, "Distributed PetscSections are not equal, with %s with size %d.\n",user.repartitioning,size);CHKERRQ(ierr);} 223 } 224 225 /* redistribute both distributed DMs */ 226 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 227 ierr = DMPlexDistribute(dmdist1, 0, NULL, &dm1);CHKERRQ(ierr); 228 ierr = ScotchResetRandomSeed();CHKERRQ(ierr); 229 ierr = DMPlexDistribute(dmdist2, 0, NULL, &dm2);CHKERRQ(ierr); 230 231 /* compare the two distributed DMs */ 232 ierr = DMPlexIsInterpolated(dm1, &interp);CHKERRQ(ierr); 233 if (interp == DMPLEX_INTERPOLATED_NONE) { 234 ierr = DMPlexEqual(dm1, dm2, &flg);CHKERRQ(ierr); 235 if (!flg) {ierr = PetscPrintf(comm, "Redistributed DMs are not equal, with %s with size %d.\n",user.repartitioning,size);CHKERRQ(ierr);} 236 } 237 238 /* cleanup */ 239 ierr = PetscSectionDestroy(&tpws);CHKERRQ(ierr); 240 ierr = PetscSectionDestroy(&s1);CHKERRQ(ierr); 241 ierr = PetscSectionDestroy(&s2);CHKERRQ(ierr); 242 ierr = ISDestroy(&is1);CHKERRQ(ierr); 243 ierr = ISDestroy(&is2);CHKERRQ(ierr); 244 ierr = DMDestroy(&dm1);CHKERRQ(ierr); 245 ierr = DMDestroy(&dm2);CHKERRQ(ierr); 246 ierr = DMDestroy(&dmdist1);CHKERRQ(ierr); 247 ierr = DMDestroy(&dmdist2);CHKERRQ(ierr); 248 ierr = PetscFinalize(); 249 return ierr; 250 } 251 252 /*TEST 253 254 test: 255 # partition sequential mesh loaded from Exodus file 256 suffix: 0 257 nsize: {{1 2 3 4 8}} 258 requires: chaco parmetis ptscotch exodusii 259 args: -dm_plex_filename ${PETSC_DIR}/share/petsc/datafiles/meshes/blockcylinder-50.exo 260 args: -partitioning {{chaco parmetis ptscotch}} -repartitioning {{parmetis ptscotch}} -tpweight {{0 1}} 261 test: 262 # repartition mesh already partitioned naively by MED loader 263 suffix: 1 264 nsize: {{1 2 3 4 8}} 265 TODO: MED 266 requires: parmetis ptscotch med 267 args: -dm_plex_filename ${PETSC_DIR}/share/petsc/datafiles/meshes/cylinder.med 268 args: -repartition 0 -partitioning {{parmetis ptscotch}} 269 test: 270 # partition mesh generated by ctetgen using scotch, then repartition with scotch, diff view 271 suffix: 3 272 nsize: 4 273 requires: ptscotch ctetgen 274 args: -dm_plex_dim 3 -dm_plex_box_faces 2,3,2 -partitioning ptscotch -repartitioning ptscotch 275 args: -p1_petscpartitioner_view -p2_petscpartitioner_view -dp1_petscpartitioner_view -dp2_petscpartitioner_view -tpweight {{0 1}} 276 test: 277 # partition mesh generated by ctetgen using partitioners supported both by MatPartitioning and PetscPartitioner 278 suffix: 4 279 nsize: {{1 2 3 4 8}} 280 requires: chaco parmetis ptscotch ctetgen 281 args: -dm_plex_dim 3 -dm_plex_box_faces {{2,3,4 5,4,3 7,11,5}} -partitioning {{chaco parmetis ptscotch}} -repartitioning {{parmetis ptscotch}} -tpweight {{0 1}} 282 283 TEST*/ 284