1 2 #include <petscsys.h> /*I "petscsys.h" I*/ 3 4 /*@ 5 PetscSplitOwnershipBlock - Given a global (or local) length determines a local 6 (or global) length via a simple formula. Splits so each processors local size 7 is divisible by the block size. 8 9 Collective (if N is PETSC_DECIDE) 10 11 Input Parameters: 12 + comm - MPI communicator that shares the object being divided 13 . bs - block size 14 . n - local length (or PETSC_DECIDE to have it set) 15 - N - global length (or PETSC_DECIDE) 16 17 Level: developer 18 19 Notes: 20 n and N cannot be both PETSC_DECIDE 21 22 If one processor calls this with N of PETSC_DECIDE then all processors 23 must, otherwise the program will hang. 24 25 .seealso: `PetscSplitOwnership()` 26 27 @*/ 28 PetscErrorCode PetscSplitOwnershipBlock(MPI_Comm comm, PetscInt bs, PetscInt *n, PetscInt *N) { 29 PetscMPIInt size, rank; 30 31 PetscFunctionBegin; 32 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Both n and N cannot be PETSC_DECIDE"); 33 34 if (*N == PETSC_DECIDE) { 35 PetscCheck(*n % bs == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "local size %" PetscInt_FMT " not divisible by block size %" PetscInt_FMT, *n, bs); 36 PetscCall(MPIU_Allreduce(n, N, 1, MPIU_INT, MPI_SUM, comm)); 37 } else if (*n == PETSC_DECIDE) { 38 PetscInt Nbs = *N / bs; 39 PetscCallMPI(MPI_Comm_size(comm, &size)); 40 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 41 *n = bs * (Nbs / size + ((Nbs % size) > rank)); 42 } 43 PetscFunctionReturn(0); 44 } 45 46 /*@ 47 PetscSplitOwnership - Given a global (or local) length determines a local 48 (or global) length via a simple formula 49 50 Collective (if n or N is PETSC_DECIDE) 51 52 Input Parameters: 53 + comm - MPI communicator that shares the object being divided 54 . n - local length (or PETSC_DECIDE to have it set) 55 - N - global length (or PETSC_DECIDE) 56 57 Level: developer 58 59 Notes: 60 n and N cannot be both PETSC_DECIDE 61 62 If one processor calls this with n or N of PETSC_DECIDE then all processors 63 must. Otherwise, an error is thrown in debug mode while the program will hang 64 in optimized (i.e. configured --with-debugging=0) mode. 65 66 .seealso: `PetscSplitOwnershipBlock()` 67 68 @*/ 69 PetscErrorCode PetscSplitOwnership(MPI_Comm comm, PetscInt *n, PetscInt *N) { 70 PetscMPIInt size, rank; 71 72 PetscFunctionBegin; 73 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Both n and N cannot be PETSC_DECIDE\n likely a call to VecSetSizes() or MatSetSizes() is wrong.\nSee https://petsc.org/release/faq/#split-ownership"); 74 if (PetscDefined(USE_DEBUG)) { 75 PetscMPIInt l[2], g[2]; 76 l[0] = (*n == PETSC_DECIDE) ? 1 : 0; 77 l[1] = (*N == PETSC_DECIDE) ? 1 : 0; 78 PetscCallMPI(MPI_Comm_size(comm, &size)); 79 PetscCall(MPIU_Allreduce(l, g, 2, MPI_INT, MPI_SUM, comm)); 80 PetscCheck(!g[0] || g[0] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for local size"); 81 PetscCheck(!g[1] || g[1] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for global size"); 82 } 83 84 if (*N == PETSC_DECIDE) { 85 PetscInt64 m = *n, M; 86 PetscCall(MPIU_Allreduce(&m, &M, 1, MPIU_INT64, MPI_SUM, comm)); 87 PetscCheck(M <= PETSC_MAX_INT, comm, PETSC_ERR_INT_OVERFLOW, "Global size overflow %" PetscInt64_FMT ". You may consider ./configure PETSc with --with-64-bit-indices for the case you are running", M); 88 *N = (PetscInt)M; 89 } else if (*n == PETSC_DECIDE) { 90 PetscCallMPI(MPI_Comm_size(comm, &size)); 91 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 92 *n = *N / size + ((*N % size) > rank); 93 } else if (PetscDefined(USE_DEBUG)) { 94 PetscInt tmp; 95 PetscCall(MPIU_Allreduce(n, &tmp, 1, MPIU_INT, MPI_SUM, comm)); 96 PetscCheck(tmp == *N, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Sum of local lengths %" PetscInt_FMT " does not equal global length %" PetscInt_FMT ", my local length %" PetscInt_FMT "\n likely a call to VecSetSizes() or MatSetSizes() is wrong.\nSee https://petsc.org/release/faq/#split-ownership", tmp, *N, *n); 97 } 98 PetscFunctionReturn(0); 99 } 100 101 /*@ 102 PetscSplitOwnershipEqual - Given a global (or local) length determines a local 103 (or global) length via a simple formula, trying to have all local lengths equal 104 105 Collective (if n or N is PETSC_DECIDE) 106 107 Input Parameters: 108 + comm - MPI communicator that shares the object being divided 109 . n - local length (or PETSC_DECIDE to have it set) 110 - N - global length (or PETSC_DECIDE) 111 112 Level: developer 113 114 Notes: 115 This is intended to be used with MATSCALAPACK, where the local size must 116 be equal in all processes (except possibly the last one). For instance, 117 the local sizes when spliting N=50 with 6 processes are 9,9,9,9,9,5 118 119 n and N cannot be both PETSC_DECIDE 120 121 If one processor calls this with n or N of PETSC_DECIDE then all processors 122 must. Otherwise, an error is thrown in debug mode while the program will hang 123 in optimized (i.e. configured --with-debugging=0) mode. 124 125 .seealso: `PetscSplitOwnership()`, `PetscSplitOwnershipBlock()` 126 127 @*/ 128 PetscErrorCode PetscSplitOwnershipEqual(MPI_Comm comm, PetscInt *n, PetscInt *N) { 129 PetscMPIInt size, rank; 130 131 PetscFunctionBegin; 132 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Both n and N cannot be PETSC_DECIDE"); 133 if (PetscDefined(USE_DEBUG)) { 134 PetscMPIInt l[2], g[2]; 135 l[0] = (*n == PETSC_DECIDE) ? 1 : 0; 136 l[1] = (*N == PETSC_DECIDE) ? 1 : 0; 137 PetscCallMPI(MPI_Comm_size(comm, &size)); 138 PetscCall(MPIU_Allreduce(l, g, 2, MPI_INT, MPI_SUM, comm)); 139 PetscCheck(!g[0] || g[0] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for local size"); 140 PetscCheck(!g[1] || g[1] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for global size"); 141 } 142 143 if (*N == PETSC_DECIDE) { 144 PetscInt64 m = *n, M; 145 PetscCall(MPIU_Allreduce(&m, &M, 1, MPIU_INT64, MPI_SUM, comm)); 146 PetscCheck(M <= PETSC_MAX_INT, comm, PETSC_ERR_INT_OVERFLOW, "Global size overflow %" PetscInt64_FMT ". You may consider ./configure PETSc with --with-64-bit-indices for the case you are running", M); 147 *N = (PetscInt)M; 148 } else if (*n == PETSC_DECIDE) { 149 PetscCallMPI(MPI_Comm_size(comm, &size)); 150 PetscCallMPI(MPI_Comm_rank(comm, &rank)); 151 *n = *N / size; 152 if (*N % size) { 153 if ((rank + 1) * (*n + 1) <= *N) *n = *n + 1; 154 else if (rank * (*n + 1) <= *N) *n = *N - rank * (*n + 1); 155 else *n = 0; 156 } 157 } else if (PetscDefined(USE_DEBUG)) { 158 PetscInt tmp; 159 PetscCall(MPIU_Allreduce(n, &tmp, 1, MPIU_INT, MPI_SUM, comm)); 160 PetscCheck(tmp == *N, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Sum of local lengths %" PetscInt_FMT " does not equal global length %" PetscInt_FMT ", my local length %" PetscInt_FMT, tmp, *N, *n); 161 } 162 PetscFunctionReturn(0); 163 } 164