1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 /* 4 Note that tag of 0 is ok because comm is a private communicator 5 generated below just for these routines. 6 */ 7 8 #undef __FUNCT__ 9 #define __FUNCT__ "PetscSequentialPhaseBegin_Private" 10 PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng) 11 { 12 PetscErrorCode ierr; 13 PetscMPIInt rank,size,tag = 0; 14 MPI_Status status; 15 16 PetscFunctionBegin; 17 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 18 if (size == 1) PetscFunctionReturn(0); 19 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 20 if (rank) { 21 ierr = MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);CHKERRQ(ierr); 22 } 23 /* Send to the next process in the group unless we are the last process */ 24 if ((rank % ng) < ng - 1 && rank != size - 1) { 25 ierr = MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);CHKERRQ(ierr); 26 } 27 PetscFunctionReturn(0); 28 } 29 30 #undef __FUNCT__ 31 #define __FUNCT__ "PetscSequentialPhaseEnd_Private" 32 PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng) 33 { 34 PetscErrorCode ierr; 35 PetscMPIInt rank,size,tag = 0; 36 MPI_Status status; 37 38 PetscFunctionBegin; 39 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 40 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 41 if (size == 1) PetscFunctionReturn(0); 42 43 /* Send to the first process in the next group */ 44 if ((rank % ng) == ng - 1 || rank == size - 1) { 45 ierr = MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);CHKERRQ(ierr); 46 } 47 if (!rank) { 48 ierr = MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);CHKERRQ(ierr); 49 } 50 PetscFunctionReturn(0); 51 } 52 53 /* ---------------------------------------------------------------------*/ 54 /* 55 The variable Petsc_Seq_keyval is used to indicate an MPI attribute that 56 is attached to a communicator that manages the sequential phase code below. 57 */ 58 static int Petsc_Seq_keyval = MPI_KEYVAL_INVALID; 59 60 #undef __FUNCT__ 61 #define __FUNCT__ "PetscSequentialPhaseBegin" 62 /*@ 63 PetscSequentialPhaseBegin - Begins a sequential section of code. 64 65 Collective on MPI_Comm 66 67 Input Parameters: 68 + comm - Communicator to sequentialize. 69 - ng - Number in processor group. This many processes are allowed to execute 70 at the same time (usually 1) 71 72 Level: intermediate 73 74 Notes: 75 PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a 76 way to force a section of code to be executed by the processes in 77 rank order. Typically, this is done with 78 .vb 79 PetscSequentialPhaseBegin(comm, 1); 80 <code to be executed sequentially> 81 PetscSequentialPhaseEnd(comm, 1); 82 .ve 83 84 Often, the sequential code contains output statements (e.g., printf) to 85 be executed. Note that you may need to flush the I/O buffers before 86 calling PetscSequentialPhaseEnd(). Also, note that some systems do 87 not propagate I/O in any order to the controling terminal (in other words, 88 even if you flush the output, you may not get the data in the order 89 that you want). 90 91 .seealso: PetscSequentialPhaseEnd() 92 93 Concepts: sequential stage 94 95 @*/ 96 PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm,int ng) 97 { 98 PetscErrorCode ierr; 99 PetscMPIInt size; 100 MPI_Comm local_comm,*addr_local_comm; 101 102 PetscFunctionBegin; 103 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 104 if (size == 1) PetscFunctionReturn(0); 105 106 /* Get the private communicator for the sequential operations */ 107 if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) { 108 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);CHKERRQ(ierr); 109 } 110 111 ierr = MPI_Comm_dup(comm,&local_comm);CHKERRQ(ierr); 112 ierr = PetscMalloc1(1,&addr_local_comm);CHKERRQ(ierr); 113 114 *addr_local_comm = local_comm; 115 116 ierr = MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);CHKERRQ(ierr); 117 ierr = PetscSequentialPhaseBegin_Private(local_comm,ng);CHKERRQ(ierr); 118 PetscFunctionReturn(0); 119 } 120 121 #undef __FUNCT__ 122 #define __FUNCT__ "PetscSequentialPhaseEnd" 123 /*@ 124 PetscSequentialPhaseEnd - Ends a sequential section of code. 125 126 Collective on MPI_Comm 127 128 Input Parameters: 129 + comm - Communicator to sequentialize. 130 - ng - Number in processor group. This many processes are allowed to execute 131 at the same time (usually 1) 132 133 Level: intermediate 134 135 Notes: 136 See PetscSequentialPhaseBegin() for more details. 137 138 .seealso: PetscSequentialPhaseBegin() 139 140 Concepts: sequential stage 141 142 @*/ 143 PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm,int ng) 144 { 145 PetscErrorCode ierr; 146 PetscMPIInt size,flag; 147 MPI_Comm local_comm,*addr_local_comm; 148 149 PetscFunctionBegin; 150 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 151 if (size == 1) PetscFunctionReturn(0); 152 153 ierr = MPI_Attr_get(comm,Petsc_Seq_keyval,(void**)&addr_local_comm,&flag);CHKERRQ(ierr); 154 if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()"); 155 local_comm = *addr_local_comm; 156 157 ierr = PetscSequentialPhaseEnd_Private(local_comm,ng);CHKERRQ(ierr); 158 159 ierr = PetscFree(addr_local_comm);CHKERRQ(ierr); 160 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 161 ierr = MPI_Attr_delete(comm,Petsc_Seq_keyval);CHKERRQ(ierr); 162 PetscFunctionReturn(0); 163 } 164