1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 #include <petsc/private/petscimpl.h> 4 /* 5 Note that tag of 0 is ok because comm is a private communicator 6 generated below just for these routines. 7 */ 8 9 PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm,int ng) 10 { 11 PetscErrorCode ierr; 12 PetscMPIInt rank,size,tag = 0; 13 MPI_Status status; 14 15 PetscFunctionBegin; 16 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 17 if (size == 1) PetscFunctionReturn(0); 18 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 19 if (rank) { 20 ierr = MPI_Recv(0,0,MPI_INT,rank-1,tag,comm,&status);CHKERRQ(ierr); 21 } 22 /* Send to the next process in the group unless we are the last process */ 23 if ((rank % ng) < ng - 1 && rank != size - 1) { 24 ierr = MPI_Send(0,0,MPI_INT,rank + 1,tag,comm);CHKERRQ(ierr); 25 } 26 PetscFunctionReturn(0); 27 } 28 29 PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm,int ng) 30 { 31 PetscErrorCode ierr; 32 PetscMPIInt rank,size,tag = 0; 33 MPI_Status status; 34 35 PetscFunctionBegin; 36 ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); 37 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 38 if (size == 1) PetscFunctionReturn(0); 39 40 /* Send to the first process in the next group */ 41 if ((rank % ng) == ng - 1 || rank == size - 1) { 42 ierr = MPI_Send(0,0,MPI_INT,(rank + 1) % size,tag,comm);CHKERRQ(ierr); 43 } 44 if (!rank) { 45 ierr = MPI_Recv(0,0,MPI_INT,size-1,tag,comm,&status);CHKERRQ(ierr); 46 } 47 PetscFunctionReturn(0); 48 } 49 50 /* ---------------------------------------------------------------------*/ 51 /* 52 The variable Petsc_Seq_keyval is used to indicate an MPI attribute that 53 is attached to a communicator that manages the sequential phase code below. 54 */ 55 PetscMPIInt Petsc_Seq_keyval = MPI_KEYVAL_INVALID; 56 57 /*@ 58 PetscSequentialPhaseBegin - Begins a sequential section of code. 59 60 Collective on MPI_Comm 61 62 Input Parameters: 63 + comm - Communicator to sequentialize. 64 - ng - Number in processor group. This many processes are allowed to execute 65 at the same time (usually 1) 66 67 Level: intermediate 68 69 Notes: 70 PetscSequentialPhaseBegin() and PetscSequentialPhaseEnd() provide a 71 way to force a section of code to be executed by the processes in 72 rank order. Typically, this is done with 73 .vb 74 PetscSequentialPhaseBegin(comm, 1); 75 <code to be executed sequentially> 76 PetscSequentialPhaseEnd(comm, 1); 77 .ve 78 79 Often, the sequential code contains output statements (e.g., printf) to 80 be executed. Note that you may need to flush the I/O buffers before 81 calling PetscSequentialPhaseEnd(). Also, note that some systems do 82 not propagate I/O in any order to the controling terminal (in other words, 83 even if you flush the output, you may not get the data in the order 84 that you want). 85 86 .seealso: PetscSequentialPhaseEnd() 87 88 Concepts: sequential stage 89 90 @*/ 91 PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm,int ng) 92 { 93 PetscErrorCode ierr; 94 PetscMPIInt size; 95 MPI_Comm local_comm,*addr_local_comm; 96 97 PetscFunctionBegin; 98 ierr = PetscSysInitializePackage();CHKERRQ(ierr); 99 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 100 if (size == 1) PetscFunctionReturn(0); 101 102 /* Get the private communicator for the sequential operations */ 103 if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) { 104 ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,MPI_NULL_DELETE_FN,&Petsc_Seq_keyval,0);CHKERRQ(ierr); 105 } 106 107 ierr = MPI_Comm_dup(comm,&local_comm);CHKERRQ(ierr); 108 ierr = PetscMalloc1(1,&addr_local_comm);CHKERRQ(ierr); 109 110 *addr_local_comm = local_comm; 111 112 ierr = MPI_Attr_put(comm,Petsc_Seq_keyval,(void*)addr_local_comm);CHKERRQ(ierr); 113 ierr = PetscSequentialPhaseBegin_Private(local_comm,ng);CHKERRQ(ierr); 114 PetscFunctionReturn(0); 115 } 116 117 /*@ 118 PetscSequentialPhaseEnd - Ends a sequential section of code. 119 120 Collective on MPI_Comm 121 122 Input Parameters: 123 + comm - Communicator to sequentialize. 124 - ng - Number in processor group. This many processes are allowed to execute 125 at the same time (usually 1) 126 127 Level: intermediate 128 129 Notes: 130 See PetscSequentialPhaseBegin() for more details. 131 132 .seealso: PetscSequentialPhaseBegin() 133 134 Concepts: sequential stage 135 136 @*/ 137 PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm,int ng) 138 { 139 PetscErrorCode ierr; 140 PetscMPIInt size,flag; 141 MPI_Comm local_comm,*addr_local_comm; 142 143 PetscFunctionBegin; 144 ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); 145 if (size == 1) PetscFunctionReturn(0); 146 147 ierr = MPI_Attr_get(comm,Petsc_Seq_keyval,(void**)&addr_local_comm,&flag);CHKERRQ(ierr); 148 if (!flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()"); 149 local_comm = *addr_local_comm; 150 151 ierr = PetscSequentialPhaseEnd_Private(local_comm,ng);CHKERRQ(ierr); 152 153 ierr = PetscFree(addr_local_comm);CHKERRQ(ierr); 154 ierr = MPI_Comm_free(&local_comm);CHKERRQ(ierr); 155 ierr = MPI_Attr_delete(comm,Petsc_Seq_keyval);CHKERRQ(ierr); 156 PetscFunctionReturn(0); 157 } 158