1 #define PETSC_DLL 2 3 #include "petscsys.h" /*I "petscsys.h" I*/ 4 /* 5 Note that tag of 0 is ok because comm is a private communicator 6 generated below just for these routines. 7 */ 8 9 #undef __FUNCT__ 10 #define __FUNCT__ "PetscSequentialPhaseBegin_Private" 11 PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng) 12 { 13 PetscErrorCode ierr; 14 PetscMPIInt rank,size,tag = 0; 15 MPI_Status status; 16 17 PetscFunctionBegin; 18 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 19 if (size == 1) PetscFunctionReturn(0); 20 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 21 if (rank) { 22 ierr = MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);CHKERRQ(ierr); 23 } 24 /* Send to the next process in the group unless we are the last process */ 25 if ((rank % ng) < ng - 1 && rank != size - 1) { 26 ierr = MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);CHKERRQ(ierr); 27 } 28 PetscFunctionReturn(0); 29 } 30 31 #undef __FUNCT__ 32 #define __FUNCT__ "PetscSequentialPhaseEnd_Private" 33 PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng) 34 { 35 PetscErrorCode ierr; 36 PetscMPIInt rank,size,tag = 0; 37 MPI_Status status; 38 39 PetscFunctionBegin; 40 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 41 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 42 if (size == 1) PetscFunctionReturn(0); 43 44 /* Send to the first process in the next group */ 45 if ((rank % ng) == ng - 1 || rank == size - 1) { 46 ierr = MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);CHKERRQ(ierr); 47 } 48 if (!rank) { 49 ierr = MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);CHKERRQ(ierr); 50 } 51 PetscFunctionReturn(0); 52 } 53 54 /* ---------------------------------------------------------------------*/ 55 /* 56 The variable Petsc_Seq_keyval is used to indicate an MPI attribute that 57 is attached to a communicator that manages the sequential phase code below. 58 */ 59 static int Petsc_Seq_keyval = MPI_KEYVAL_INVALID; 60 61 #undef __FUNCT__ 62 #define __FUNCT__ "PetscSequentialPhaseBegin" 63 /*@ 64 PetscSequentialPhaseBegin - Begins a sequential section of code. 65 66 Collective on MPI_Comm 67 68 Input Parameters: 69 + comm - Communicator to sequentialize. 70 - ng - Number in processor group. This many processes are allowed to execute 71 at the same time (usually 1) 72 73 Level: intermediate 74 75 Notes: 76 PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a 77 way to force a section of code to be executed by the processes in 78 rank order. Typically, this is done with 79 .vb 80 PetscSequentialPhaseBegin(comm, 1); 81 <code to be executed sequentially> 82 PetscSequentialPhaseEnd(comm, 1); 83 .ve 84 85 Often, the sequential code contains output statements (e.g., printf) to 86 be executed. Note that you may need to flush the I/O buffers before 87 calling PetscSequentialPhaseEnd(). Also, note that some systems do 88 not propagate I/O in any order to the controling terminal (in other words, 89 even if you flush the output, you may not get the data in the order 90 that you want). 91 92 .seealso: PetscSequentialPhaseEnd() 93 94 Concepts: sequential stage 95 96 @*/ 97 PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm,int ng) 98 { 99 PetscErrorCode ierr; 100 PetscMPIInt size; 101 MPI_Comm local_comm,*addr_local_comm; 102 103 PetscFunctionBegin; 104 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 105 if (size == 1) PetscFunctionReturn(0); 106 107 /* Get the private communicator for the sequential operations */ 108 if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) { 109 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);CHKERRQ(ierr); 110 } 111 112 ierr = MPI_Comm_dup(comm,&local_comm);CHKERRQ(ierr); 113 ierr = PetscMalloc(sizeof(MPI_Comm),&addr_local_comm);CHKERRQ(ierr); 114 *addr_local_comm = local_comm; 115 ierr = MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);CHKERRQ(ierr); 116 ierr = PetscSequentialPhaseBegin_Private(local_comm,ng);CHKERRQ(ierr); 117 PetscFunctionReturn(0); 118 } 119 120 #undef __FUNCT__ 121 #define __FUNCT__ "PetscSequentialPhaseEnd" 122 /*@ 123 PetscSequentialPhaseEnd - Ends a sequential section of code. 124 125 Collective on MPI_Comm 126 127 Input Parameters: 128 + comm - Communicator to sequentialize. 129 - ng - Number in processor group. This many processes are allowed to execute 130 at the same time (usually 1) 131 132 Level: intermediate 133 134 Notes: 135 See PetscSequentialPhaseBegin() for more details. 136 137 .seealso: PetscSequentialPhaseBegin() 138 139 Concepts: sequential stage 140 141 @*/ 142 PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm,int ng) 143 { 144 PetscErrorCode ierr; 145 PetscMPIInt size,flag; 146 MPI_Comm local_comm,*addr_local_comm; 147 148 PetscFunctionBegin; 149 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 150 if (size == 1) PetscFunctionReturn(0); 151 152 ierr = MPI_Attr_get(comm,Petsc_Seq_keyval,(void **)&addr_local_comm,&flag);CHKERRQ(ierr); 153 if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()"); 154 local_comm = *addr_local_comm; 155 156 ierr = PetscSequentialPhaseEnd_Private(local_comm,ng);CHKERRQ(ierr); 157 158 ierr = PetscFree(addr_local_comm);CHKERRQ(ierr); 159 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 160 ierr = MPI_Attr_delete(comm,Petsc_Seq_keyval);CHKERRQ(ierr); 161 PetscFunctionReturn(0); 162 } 163