1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 /* 4 Note that tag of 0 is ok because comm is a private communicator 5 generated below just for these routines. 6 */ 7 8 PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng) 9 { 10 PetscErrorCode ierr; 11 PetscMPIInt rank,size,tag = 0; 12 MPI_Status status; 13 14 PetscFunctionBegin; 15 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 16 if (size == 1) PetscFunctionReturn(0); 17 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 18 if (rank) { 19 ierr = MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);CHKERRQ(ierr); 20 } 21 /* Send to the next process in the group unless we are the last process */ 22 if ((rank % ng) < ng - 1 && rank != size - 1) { 23 ierr = MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);CHKERRQ(ierr); 24 } 25 PetscFunctionReturn(0); 26 } 27 28 PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng) 29 { 30 PetscErrorCode ierr; 31 PetscMPIInt rank,size,tag = 0; 32 MPI_Status status; 33 34 PetscFunctionBegin; 35 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 36 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 37 if (size == 1) PetscFunctionReturn(0); 38 39 /* Send to the first process in the next group */ 40 if ((rank % ng) == ng - 1 || rank == size - 1) { 41 ierr = MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);CHKERRQ(ierr); 42 } 43 if (!rank) { 44 ierr = MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);CHKERRQ(ierr); 45 } 46 PetscFunctionReturn(0); 47 } 48 49 /* ---------------------------------------------------------------------*/ 50 /* 51 The variable Petsc_Seq_keyval is used to indicate an MPI attribute that 52 is attached to a communicator that manages the sequential phase code below. 53 */ 54 static int Petsc_Seq_keyval = MPI_KEYVAL_INVALID; 55 56 /*@ 57 PetscSequentialPhaseBegin - Begins a sequential section of code. 58 59 Collective on MPI_Comm 60 61 Input Parameters: 62 + comm - Communicator to sequentialize. 63 - ng - Number in processor group. This many processes are allowed to execute 64 at the same time (usually 1) 65 66 Level: intermediate 67 68 Notes: 69 PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a 70 way to force a section of code to be executed by the processes in 71 rank order. Typically, this is done with 72 .vb 73 PetscSequentialPhaseBegin(comm, 1); 74 <code to be executed sequentially> 75 PetscSequentialPhaseEnd(comm, 1); 76 .ve 77 78 Often, the sequential code contains output statements (e.g., printf) to 79 be executed. Note that you may need to flush the I/O buffers before 80 calling PetscSequentialPhaseEnd(). Also, note that some systems do 81 not propagate I/O in any order to the controling terminal (in other words, 82 even if you flush the output, you may not get the data in the order 83 that you want). 84 85 .seealso: PetscSequentialPhaseEnd() 86 87 Concepts: sequential stage 88 89 @*/ 90 PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm,int ng) 91 { 92 PetscErrorCode ierr; 93 PetscMPIInt size; 94 MPI_Comm local_comm,*addr_local_comm; 95 96 PetscFunctionBegin; 97 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 98 if (size == 1) PetscFunctionReturn(0); 99 100 /* Get the private communicator for the sequential operations */ 101 if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) { 102 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);CHKERRQ(ierr); 103 } 104 105 ierr = MPI_Comm_dup(comm,&local_comm);CHKERRQ(ierr); 106 ierr = PetscMalloc1(1,&addr_local_comm);CHKERRQ(ierr); 107 108 *addr_local_comm = local_comm; 109 110 ierr = MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);CHKERRQ(ierr); 111 ierr = PetscSequentialPhaseBegin_Private(local_comm,ng);CHKERRQ(ierr); 112 PetscFunctionReturn(0); 113 } 114 115 /*@ 116 PetscSequentialPhaseEnd - Ends a sequential section of code. 117 118 Collective on MPI_Comm 119 120 Input Parameters: 121 + comm - Communicator to sequentialize. 122 - ng - Number in processor group. This many processes are allowed to execute 123 at the same time (usually 1) 124 125 Level: intermediate 126 127 Notes: 128 See PetscSequentialPhaseBegin() for more details. 129 130 .seealso: PetscSequentialPhaseBegin() 131 132 Concepts: sequential stage 133 134 @*/ 135 PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm,int ng) 136 { 137 PetscErrorCode ierr; 138 PetscMPIInt size,flag; 139 MPI_Comm local_comm,*addr_local_comm; 140 141 PetscFunctionBegin; 142 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 143 if (size == 1) PetscFunctionReturn(0); 144 145 ierr = MPI_Attr_get(comm,Petsc_Seq_keyval,(void**)&addr_local_comm,&flag);CHKERRQ(ierr); 146 if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()"); 147 local_comm = *addr_local_comm; 148 149 ierr = PetscSequentialPhaseEnd_Private(local_comm,ng);CHKERRQ(ierr); 150 151 ierr = PetscFree(addr_local_comm);CHKERRQ(ierr); 152 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 153 ierr = MPI_Attr_delete(comm,Petsc_Seq_keyval);CHKERRQ(ierr); 154 PetscFunctionReturn(0); 155 } 156