1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 4 /*@ 5 PetscSplitOwnershipBlock - Given a global (or local) length determines a local 6 (or global) length via a simple formula. Splits so each processors local size 7 is divisible by the block size. 8 9 Collective (if N is PETSC_DECIDE) 10 11 Input Parameters: 12 + comm - MPI communicator that shares the object being divided 13 . bs - block size 14 . n - local length (or PETSC_DECIDE to have it set) 15 - N - global length (or PETSC_DECIDE) 16 17 Level: developer 18 19 Notes: 20 n and N cannot be both PETSC_DECIDE 21 22 If one processor calls this with N of PETSC_DECIDE then all processors 23 must, otherwise the program will hang. 24 25 .seealso: PetscSplitOwnership() 26 27 @*/ 28 PetscErrorCode PetscSplitOwnershipBlock(MPI_Comm comm,PetscInt bs,PetscInt *n,PetscInt *N) 29 { 30 PetscErrorCode ierr; 31 PetscMPIInt size,rank; 32 33 PetscFunctionBegin; 34 if (*N == PETSC_DECIDE && *n == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Both n and N cannot be PETSC_DECIDE"); 35 36 if (*N == PETSC_DECIDE) { 37 if (*n % bs != 0) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"local size %D not divisible by block size %D",*n,bs); 38 ierr = MPIU_Allreduce(n,N,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr); 39 } else if (*n == PETSC_DECIDE) { 40 PetscInt Nbs = *N/bs; 41 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 42 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 43 *n = bs*(Nbs/size + ((Nbs % size) > rank)); 44 } 45 PetscFunctionReturn(0); 46 } 47 48 49 /*@ 50 PetscSplitOwnership - Given a global (or local) length determines a local 51 (or global) length via a simple formula 52 53 Collective (if n or N is PETSC_DECIDE) 54 55 Input Parameters: 56 + comm - MPI communicator that shares the object being divided 57 . n - local length (or PETSC_DECIDE to have it set) 58 - N - global length (or PETSC_DECIDE) 59 60 Level: developer 61 62 Notes: 63 n and N cannot be both PETSC_DECIDE 64 65 If one processor calls this with n or N of PETSC_DECIDE then all processors 66 must. Otherwise, an error is thrown in debug mode while the program will hang 67 in optimized (i.e. configured --with-debugging=0) mode. 68 69 .seealso: PetscSplitOwnershipBlock() 70 71 @*/ 72 PetscErrorCode PetscSplitOwnership(MPI_Comm comm,PetscInt *n,PetscInt *N) 73 { 74 PetscErrorCode ierr; 75 PetscMPIInt size,rank; 76 77 PetscFunctionBegin; 78 if (*N == PETSC_DECIDE && *n == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Both n and N cannot be PETSC_DECIDE\n likely a call to VecSetSizes() or MatSetSizes() is wrong.\nSee https://petsc.org/release/faq/#split-ownership"); 79 if (PetscDefined(USE_DEBUG)) { 80 PetscMPIInt l[2],g[2]; 81 l[0] = (*n == PETSC_DECIDE) ? 1 : 0; 82 l[1] = (*N == PETSC_DECIDE) ? 1 : 0; 83 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 84 ierr = MPIU_Allreduce(l,g,2,MPI_INT,MPI_SUM,comm);CHKERRQ(ierr); 85 if (g[0] && g[0] != size) SETERRQ(comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for local size"); 86 if (g[1] && g[1] != size) SETERRQ(comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for global size"); 87 } 88 89 if (*N == PETSC_DECIDE) { 90 PetscInt64 m = *n, M; 91 ierr = MPIU_Allreduce(&m,&M,1,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr); 92 if (M > PETSC_MAX_INT) SETERRQ1(comm,PETSC_ERR_INT_OVERFLOW,"Global size overflow %" PetscInt64_FMT ". You may consider ./configure PETSc with --with-64-bit-indices for the case you are running", M); 93 else *N = (PetscInt)M; 94 } else if (*n == PETSC_DECIDE) { 95 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 96 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 97 *n = *N/size + ((*N % size) > rank); 98 } else if (PetscDefined(USE_DEBUG)) { 99 PetscInt tmp; 100 ierr = MPIU_Allreduce(n,&tmp,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr); 101 if (tmp != *N) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Sum of local lengths %D does not equal global length %D, my local length %D\n likely a call to VecSetSizes() or MatSetSizes() is wrong.\nSee https://petsc.org/release/faq/#split-ownership",tmp,*N,*n); 102 } 103 PetscFunctionReturn(0); 104 } 105 106 107 /*@ 108 PetscSplitOwnershipEqual - Given a global (or local) length determines a local 109 (or global) length via a simple formula, trying to have all local lengths equal 110 111 Collective (if n or N is PETSC_DECIDE) 112 113 Input Parameters: 114 + comm - MPI communicator that shares the object being divided 115 . n - local length (or PETSC_DECIDE to have it set) 116 - N - global length (or PETSC_DECIDE) 117 118 Level: developer 119 120 Notes: 121 This is intended to be used with MATSCALAPACK, where the local size must 122 be equal in all processes (except possibly the last one). For instance, 123 the local sizes when spliting N=50 with 6 processes are 9,9,9,9,9,5 124 125 n and N cannot be both PETSC_DECIDE 126 127 If one processor calls this with n or N of PETSC_DECIDE then all processors 128 must. Otherwise, an error is thrown in debug mode while the program will hang 129 in optimized (i.e. configured --with-debugging=0) mode. 130 131 .seealso: PetscSplitOwnership(), PetscSplitOwnershipBlock() 132 133 @*/ 134 PetscErrorCode PetscSplitOwnershipEqual(MPI_Comm comm,PetscInt *n,PetscInt *N) 135 { 136 PetscErrorCode ierr; 137 PetscMPIInt size,rank; 138 139 PetscFunctionBegin; 140 if (*N == PETSC_DECIDE && *n == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Both n and N cannot be PETSC_DECIDE"); 141 if (PetscDefined(USE_DEBUG)) { 142 PetscMPIInt l[2],g[2]; 143 l[0] = (*n == PETSC_DECIDE) ? 1 : 0; 144 l[1] = (*N == PETSC_DECIDE) ? 1 : 0; 145 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 146 ierr = MPIU_Allreduce(l,g,2,MPI_INT,MPI_SUM,comm);CHKERRQ(ierr); 147 if (g[0] && g[0] != size) SETERRQ(comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for local size"); 148 if (g[1] && g[1] != size) SETERRQ(comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for global size"); 149 } 150 151 if (*N == PETSC_DECIDE) { 152 PetscInt64 m = *n, M; 153 ierr = MPIU_Allreduce(&m,&M,1,MPIU_INT64,MPI_SUM,comm);CHKERRQ(ierr); 154 if (M > PETSC_MAX_INT) SETERRQ1(comm,PETSC_ERR_INT_OVERFLOW,"Global size overflow %" PetscInt64_FMT ". You may consider ./configure PETSc with --with-64-bit-indices for the case you are running", M); 155 else *N = (PetscInt)M; 156 } else if (*n == PETSC_DECIDE) { 157 ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr); 158 ierr = MPI_Comm_rank(comm,&rank);CHKERRMPI(ierr); 159 *n = *N/size; 160 if (*N % size) { 161 if ((rank+1)*(*n+1)<=*N) *n = *n+1; 162 else if (rank*(*n+1)<=*N) *n = *N-rank*(*n+1); 163 else *n = 0; 164 } 165 } else if (PetscDefined(USE_DEBUG)) { 166 PetscInt tmp; 167 ierr = MPIU_Allreduce(n,&tmp,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr); 168 if (tmp != *N) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Sum of local lengths %D does not equal global length %D, my local length %D",tmp,*N,*n); 169 } 170 PetscFunctionReturn(0); 171 } 172