1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 #include <petsc/private/petscimpl.h> 4 /* 5 Note that tag of 0 is ok because comm is a private communicator 6 generated below just for these routines. 7 */ 8 9 PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng) 10 { 11 PetscErrorCode ierr; 12 PetscMPIInt rank,size,tag = 0; 13 MPI_Status status; 14 15 PetscFunctionBegin; 16 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 17 if (size == 1) PetscFunctionReturn(0); 18 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 19 if (rank) { 20 ierr = MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);CHKERRQ(ierr); 21 } 22 /* Send to the next process in the group unless we are the last process */ 23 if ((rank % ng) < ng - 1 && rank != size - 1) { 24 ierr = MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);CHKERRQ(ierr); 25 } 26 PetscFunctionReturn(0); 27 } 28 29 PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng) 30 { 31 PetscErrorCode ierr; 32 PetscMPIInt rank,size,tag = 0; 33 MPI_Status status; 34 35 PetscFunctionBegin; 36 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 37 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 38 if (size == 1) PetscFunctionReturn(0); 39 40 /* Send to the first process in the next group */ 41 if ((rank % ng) == ng - 1 || rank == size - 1) { 42 ierr = MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);CHKERRQ(ierr); 43 } 44 if (!rank) { 45 ierr = MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);CHKERRQ(ierr); 46 } 47 PetscFunctionReturn(0); 48 } 49 50 /* ---------------------------------------------------------------------*/ 51 /* 52 The variable Petsc_Seq_keyval is used to indicate an MPI attribute that 53 is attached to a communicator that manages the sequential phase code below. 54 */ 55 PetscMPIInt Petsc_Seq_keyval = MPI_KEYVAL_INVALID; 56 57 /*@ 58 PetscSequentialPhaseBegin - Begins a sequential section of code. 59 60 Collective on MPI_Comm 61 62 Input Parameters: 63 + comm - Communicator to sequentialize. 64 - ng - Number in processor group. This many processes are allowed to execute 65 at the same time (usually 1) 66 67 Level: intermediate 68 69 Notes: 70 PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a 71 way to force a section of code to be executed by the processes in 72 rank order. Typically, this is done with 73 .vb 74 PetscSequentialPhaseBegin(comm, 1); 75 <code to be executed sequentially> 76 PetscSequentialPhaseEnd(comm, 1); 77 .ve 78 79 Often, the sequential code contains output statements (e.g., printf) to 80 be executed. Note that you may need to flush the I/O buffers before 81 calling PetscSequentialPhaseEnd(). Also, note that some systems do 82 not propagate I/O in any order to the controling terminal (in other words, 83 even if you flush the output, you may not get the data in the order 84 that you want). 85 86 .seealso: PetscSequentialPhaseEnd() 87 88 Concepts: sequential stage 89 90 @*/ 91 PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm,int ng) 92 { 93 PetscErrorCode ierr; 94 PetscMPIInt size; 95 MPI_Comm local_comm,*addr_local_comm; 96 97 PetscFunctionBegin; 98 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 99 if (size == 1) PetscFunctionReturn(0); 100 101 /* Get the private communicator for the sequential operations */ 102 if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) { 103 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);CHKERRQ(ierr); 104 } 105 106 ierr = MPI_Comm_dup(comm,&local_comm);CHKERRQ(ierr); 107 ierr = PetscMalloc1(1,&addr_local_comm);CHKERRQ(ierr); 108 109 *addr_local_comm = local_comm; 110 111 ierr = MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);CHKERRQ(ierr); 112 ierr = PetscSequentialPhaseBegin_Private(local_comm,ng);CHKERRQ(ierr); 113 PetscFunctionReturn(0); 114 } 115 116 /*@ 117 PetscSequentialPhaseEnd - Ends a sequential section of code. 118 119 Collective on MPI_Comm 120 121 Input Parameters: 122 + comm - Communicator to sequentialize. 123 - ng - Number in processor group. This many processes are allowed to execute 124 at the same time (usually 1) 125 126 Level: intermediate 127 128 Notes: 129 See PetscSequentialPhaseBegin() for more details. 130 131 .seealso: PetscSequentialPhaseBegin() 132 133 Concepts: sequential stage 134 135 @*/ 136 PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm,int ng) 137 { 138 PetscErrorCode ierr; 139 PetscMPIInt size,flag; 140 MPI_Comm local_comm,*addr_local_comm; 141 142 PetscFunctionBegin; 143 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 144 if (size == 1) PetscFunctionReturn(0); 145 146 ierr = MPI_Attr_get(comm,Petsc_Seq_keyval,(void**)&addr_local_comm,&flag);CHKERRQ(ierr); 147 if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()"); 148 local_comm = *addr_local_comm; 149 150 ierr = PetscSequentialPhaseEnd_Private(local_comm,ng);CHKERRQ(ierr); 151 152 ierr = PetscFree(addr_local_comm);CHKERRQ(ierr); 153 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 154 ierr = MPI_Attr_delete(comm,Petsc_Seq_keyval);CHKERRQ(ierr); 155 PetscFunctionReturn(0); 156 } 157