1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 /* 4 Note that tag of 0 is ok because comm is a private communicator 5 generated below just for these routines. 6 */ 7 8 #undef __FUNCT__ 9 #define __FUNCT__ "PetscSequentialPhaseBegin_Private" 10 PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng) 11 { 12 PetscErrorCode ierr; 13 PetscMPIInt rank,size,tag = 0; 14 MPI_Status status; 15 16 PetscFunctionBegin; 17 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 18 if (size == 1) PetscFunctionReturn(0); 19 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 20 if (rank) { 21 ierr = MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);CHKERRQ(ierr); 22 } 23 /* Send to the next process in the group unless we are the last process */ 24 if ((rank % ng) < ng - 1 && rank != size - 1) { 25 ierr = MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);CHKERRQ(ierr); 26 } 27 PetscFunctionReturn(0); 28 } 29 30 #undef __FUNCT__ 31 #define __FUNCT__ "PetscSequentialPhaseEnd_Private" 32 PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng) 33 { 34 PetscErrorCode ierr; 35 PetscMPIInt rank,size,tag = 0; 36 MPI_Status status; 37 38 PetscFunctionBegin; 39 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 40 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 41 if (size == 1) PetscFunctionReturn(0); 42 43 /* Send to the first process in the next group */ 44 if ((rank % ng) == ng - 1 || rank == size - 1) { 45 ierr = MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);CHKERRQ(ierr); 46 } 47 if (!rank) { 48 ierr = MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);CHKERRQ(ierr); 49 } 50 PetscFunctionReturn(0); 51 } 52 53 /* ---------------------------------------------------------------------*/ 54 /* 55 The variable Petsc_Seq_keyval is used to indicate an MPI attribute that 56 is attached to a communicator that manages the sequential phase code below. 57 */ 58 static int Petsc_Seq_keyval = MPI_KEYVAL_INVALID; 59 60 #undef __FUNCT__ 61 #define __FUNCT__ "PetscSequentialPhaseBegin" 62 /*@ 63 PetscSequentialPhaseBegin - Begins a sequential section of code. 64 65 Collective on MPI_Comm 66 67 Input Parameters: 68 + comm - Communicator to sequentialize. 69 - ng - Number in processor group. This many processes are allowed to execute 70 at the same time (usually 1) 71 72 Level: intermediate 73 74 Notes: 75 PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a 76 way to force a section of code to be executed by the processes in 77 rank order. Typically, this is done with 78 .vb 79 PetscSequentialPhaseBegin(comm, 1); 80 <code to be executed sequentially> 81 PetscSequentialPhaseEnd(comm, 1); 82 .ve 83 84 Often, the sequential code contains output statements (e.g., printf) to 85 be executed. Note that you may need to flush the I/O buffers before 86 calling PetscSequentialPhaseEnd(). Also, note that some systems do 87 not propagate I/O in any order to the controling terminal (in other words, 88 even if you flush the output, you may not get the data in the order 89 that you want). 90 91 .seealso: PetscSequentialPhaseEnd() 92 93 Concepts: sequential stage 94 95 @*/ 96 PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm,int ng) 97 { 98 PetscErrorCode ierr; 99 PetscMPIInt size; 100 MPI_Comm local_comm,*addr_local_comm; 101 102 PetscFunctionBegin; 103 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 104 if (size == 1) PetscFunctionReturn(0); 105 106 /* Get the private communicator for the sequential operations */ 107 if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) { 108 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);CHKERRQ(ierr); 109 } 110 111 ierr = MPI_Comm_dup(comm,&local_comm);CHKERRQ(ierr); 112 ierr = PetscMalloc(sizeof(MPI_Comm),&addr_local_comm);CHKERRQ(ierr); 113 *addr_local_comm = local_comm; 114 ierr = MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);CHKERRQ(ierr); 115 ierr = PetscSequentialPhaseBegin_Private(local_comm,ng);CHKERRQ(ierr); 116 PetscFunctionReturn(0); 117 } 118 119 #undef __FUNCT__ 120 #define __FUNCT__ "PetscSequentialPhaseEnd" 121 /*@ 122 PetscSequentialPhaseEnd - Ends a sequential section of code. 123 124 Collective on MPI_Comm 125 126 Input Parameters: 127 + comm - Communicator to sequentialize. 128 - ng - Number in processor group. This many processes are allowed to execute 129 at the same time (usually 1) 130 131 Level: intermediate 132 133 Notes: 134 See PetscSequentialPhaseBegin() for more details. 135 136 .seealso: PetscSequentialPhaseBegin() 137 138 Concepts: sequential stage 139 140 @*/ 141 PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm,int ng) 142 { 143 PetscErrorCode ierr; 144 PetscMPIInt size,flag; 145 MPI_Comm local_comm,*addr_local_comm; 146 147 PetscFunctionBegin; 148 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 149 if (size == 1) PetscFunctionReturn(0); 150 151 ierr = MPI_Attr_get(comm,Petsc_Seq_keyval,(void **)&addr_local_comm,&flag);CHKERRQ(ierr); 152 if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()"); 153 local_comm = *addr_local_comm; 154 155 ierr = PetscSequentialPhaseEnd_Private(local_comm,ng);CHKERRQ(ierr); 156 157 ierr = PetscFree(addr_local_comm);CHKERRQ(ierr); 158 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 159 ierr = MPI_Attr_delete(comm,Petsc_Seq_keyval);CHKERRQ(ierr); 160 PetscFunctionReturn(0); 161 } 162