1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 4 /*@ 5 PetscSplitOwnershipBlock - Given a global (or local) length determines a local 6 (or global) length via a simple formula. Splits so each processors local size 7 is divisible by the block size. 8 9 Collective (if N is PETSC_DECIDE) 10 11 Input Parameters: 12 + comm - MPI communicator that shares the object being divided 13 . bs - block size 14 . n - local length (or PETSC_DECIDE to have it set) 15 - N - global length (or PETSC_DECIDE) 16 17 Level: developer 18 19 Notes: 20 n and N cannot be both PETSC_DECIDE 21 22 If one processor calls this with N of PETSC_DECIDE then all processors 23 must, otherwise the program will hang. 24 25 .seealso: `PetscSplitOwnership()` 26 27 @*/ 28 PetscErrorCode PetscSplitOwnershipBlock(MPI_Comm comm,PetscInt bs,PetscInt *n,PetscInt *N) 29 { 30 PetscMPIInt size,rank; 31 32 PetscFunctionBegin; 33 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Both n and N cannot be PETSC_DECIDE"); 34 35 if (*N == PETSC_DECIDE) { 36 PetscCheck(*n % bs == 0,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"local size %" PetscInt_FMT " not divisible by block size %" PetscInt_FMT,*n,bs); 37 PetscCall(MPIU_Allreduce(n,N,1,MPIU_INT,MPI_SUM,comm)); 38 } else if (*n == PETSC_DECIDE) { 39 PetscInt Nbs = *N/bs; 40 PetscCallMPI(MPI_Comm_size(comm,&size)); 41 PetscCallMPI(MPI_Comm_rank(comm,&rank)); 42 *n = bs*(Nbs/size + ((Nbs % size) > rank)); 43 } 44 PetscFunctionReturn(0); 45 } 46 47 /*@ 48 PetscSplitOwnership - Given a global (or local) length determines a local 49 (or global) length via a simple formula 50 51 Collective (if n or N is PETSC_DECIDE) 52 53 Input Parameters: 54 + comm - MPI communicator that shares the object being divided 55 . n - local length (or PETSC_DECIDE to have it set) 56 - N - global length (or PETSC_DECIDE) 57 58 Level: developer 59 60 Notes: 61 n and N cannot be both PETSC_DECIDE 62 63 If one processor calls this with n or N of PETSC_DECIDE then all processors 64 must. Otherwise, an error is thrown in debug mode while the program will hang 65 in optimized (i.e. configured --with-debugging=0) mode. 66 67 .seealso: `PetscSplitOwnershipBlock()` 68 69 @*/ 70 PetscErrorCode PetscSplitOwnership(MPI_Comm comm,PetscInt *n,PetscInt *N) 71 { 72 PetscMPIInt size,rank; 73 74 PetscFunctionBegin; 75 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Both n and N cannot be PETSC_DECIDE\n likely a call to VecSetSizes() or MatSetSizes() is wrong.\nSee https://petsc.org/release/faq/#split-ownership"); 76 if (PetscDefined(USE_DEBUG)) { 77 PetscMPIInt l[2],g[2]; 78 l[0] = (*n == PETSC_DECIDE) ? 1 : 0; 79 l[1] = (*N == PETSC_DECIDE) ? 1 : 0; 80 PetscCallMPI(MPI_Comm_size(comm,&size)); 81 PetscCall(MPIU_Allreduce(l,g,2,MPI_INT,MPI_SUM,comm)); 82 PetscCheck(!g[0] || g[0] == size,comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for local size"); 83 PetscCheck(!g[1] || g[1] == size,comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for global size"); 84 } 85 86 if (*N == PETSC_DECIDE) { 87 PetscInt64 m = *n, M; 88 PetscCall(MPIU_Allreduce(&m,&M,1,MPIU_INT64,MPI_SUM,comm)); 89 PetscCheck(M <= PETSC_MAX_INT,comm,PETSC_ERR_INT_OVERFLOW,"Global size overflow %" PetscInt64_FMT ". You may consider ./configure PETSc with --with-64-bit-indices for the case you are running", M); 90 else *N = (PetscInt)M; 91 } else if (*n == PETSC_DECIDE) { 92 PetscCallMPI(MPI_Comm_size(comm,&size)); 93 PetscCallMPI(MPI_Comm_rank(comm,&rank)); 94 *n = *N/size + ((*N % size) > rank); 95 } else if (PetscDefined(USE_DEBUG)) { 96 PetscInt tmp; 97 PetscCall(MPIU_Allreduce(n,&tmp,1,MPIU_INT,MPI_SUM,comm)); 98 PetscCheck(tmp == *N,PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Sum of local lengths %" PetscInt_FMT " does not equal global length %" PetscInt_FMT ", my local length %" PetscInt_FMT "\n likely a call to VecSetSizes() or MatSetSizes() is wrong.\nSee https://petsc.org/release/faq/#split-ownership",tmp,*N,*n); 99 } 100 PetscFunctionReturn(0); 101 } 102 103 /*@ 104 PetscSplitOwnershipEqual - Given a global (or local) length determines a local 105 (or global) length via a simple formula, trying to have all local lengths equal 106 107 Collective (if n or N is PETSC_DECIDE) 108 109 Input Parameters: 110 + comm - MPI communicator that shares the object being divided 111 . n - local length (or PETSC_DECIDE to have it set) 112 - N - global length (or PETSC_DECIDE) 113 114 Level: developer 115 116 Notes: 117 This is intended to be used with MATSCALAPACK, where the local size must 118 be equal in all processes (except possibly the last one). For instance, 119 the local sizes when spliting N=50 with 6 processes are 9,9,9,9,9,5 120 121 n and N cannot be both PETSC_DECIDE 122 123 If one processor calls this with n or N of PETSC_DECIDE then all processors 124 must. Otherwise, an error is thrown in debug mode while the program will hang 125 in optimized (i.e. configured --with-debugging=0) mode. 126 127 .seealso: `PetscSplitOwnership()`, `PetscSplitOwnershipBlock()` 128 129 @*/ 130 PetscErrorCode PetscSplitOwnershipEqual(MPI_Comm comm,PetscInt *n,PetscInt *N) 131 { 132 PetscMPIInt size,rank; 133 134 PetscFunctionBegin; 135 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE,PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Both n and N cannot be PETSC_DECIDE"); 136 if (PetscDefined(USE_DEBUG)) { 137 PetscMPIInt l[2],g[2]; 138 l[0] = (*n == PETSC_DECIDE) ? 1 : 0; 139 l[1] = (*N == PETSC_DECIDE) ? 1 : 0; 140 PetscCallMPI(MPI_Comm_size(comm,&size)); 141 PetscCall(MPIU_Allreduce(l,g,2,MPI_INT,MPI_SUM,comm)); 142 PetscCheck(!g[0] || g[0] == size,comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for local size"); 143 PetscCheck(!g[1] || g[1] == size,comm,PETSC_ERR_ARG_INCOMP,"All processes must supply PETSC_DECIDE for global size"); 144 } 145 146 if (*N == PETSC_DECIDE) { 147 PetscInt64 m = *n, M; 148 PetscCall(MPIU_Allreduce(&m,&M,1,MPIU_INT64,MPI_SUM,comm)); 149 PetscCheck(M <= PETSC_MAX_INT,comm,PETSC_ERR_INT_OVERFLOW,"Global size overflow %" PetscInt64_FMT ". You may consider ./configure PETSc with --with-64-bit-indices for the case you are running", M); 150 else *N = (PetscInt)M; 151 } else if (*n == PETSC_DECIDE) { 152 PetscCallMPI(MPI_Comm_size(comm,&size)); 153 PetscCallMPI(MPI_Comm_rank(comm,&rank)); 154 *n = *N/size; 155 if (*N % size) { 156 if ((rank+1)*(*n+1)<=*N) *n = *n+1; 157 else if (rank*(*n+1)<=*N) *n = *N-rank*(*n+1); 158 else *n = 0; 159 } 160 } else if (PetscDefined(USE_DEBUG)) { 161 PetscInt tmp; 162 PetscCall(MPIU_Allreduce(n,&tmp,1,MPIU_INT,MPI_SUM,comm)); 163 PetscCheck(tmp == *N,PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Sum of local lengths %" PetscInt_FMT " does not equal global length %" PetscInt_FMT ", my local length %" PetscInt_FMT,tmp,*N,*n); 164 } 165 PetscFunctionReturn(0); 166 } 167