1 #include <petscsys.h> /*I "petscsys.h" I*/
2
3 /*@
4 PetscSplitOwnershipBlock - Given a global (or local) length determines a local
5 (or global) length via a simple formula. Splits so each processors local size
6 is divisible by the block size.
7
8 Collective (if `N` is `PETSC_DECIDE`)
9
10 Input Parameters:
11 + comm - MPI communicator that shares the object being divided
12 . bs - block size
13 . n - local length (or `PETSC_DECIDE` to have it set)
14 - N - global length (or `PETSC_DECIDE`)
15
16 Level: developer
17
18 Notes:
19 `n` and `N` cannot be both `PETSC_DECIDE`
20
21 If one processor calls this with `N` of `PETSC_DECIDE` then all processors
22 must, otherwise the program will hang.
23
24 .seealso: `PetscSplitOwnership()`, `PetscSplitOwnershipEqual()`
25 @*/
PetscSplitOwnershipBlock(MPI_Comm comm,PetscInt bs,PetscInt * n,PetscInt * N)26 PetscErrorCode PetscSplitOwnershipBlock(MPI_Comm comm, PetscInt bs, PetscInt *n, PetscInt *N)
27 {
28 PetscMPIInt size, rank;
29
30 PetscFunctionBegin;
31 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Both n and N cannot be PETSC_DECIDE");
32
33 if (*N == PETSC_DECIDE) {
34 PetscCheck(*n % bs == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "local size %" PetscInt_FMT " not divisible by block size %" PetscInt_FMT, *n, bs);
35 PetscCallMPI(MPIU_Allreduce(n, N, 1, MPIU_INT, MPI_SUM, comm));
36 } else if (*n == PETSC_DECIDE) {
37 PetscInt Nbs = *N / bs;
38 PetscCallMPI(MPI_Comm_size(comm, &size));
39 PetscCallMPI(MPI_Comm_rank(comm, &rank));
40 *n = bs * (Nbs / size + ((Nbs % size) > rank));
41 }
42 PetscFunctionReturn(PETSC_SUCCESS);
43 }
44
45 /*@
46 PetscSplitOwnership - Given a global (or local) length determines a local
47 (or global) length via a simple formula
48
49 Collective (if `n` or `N` is `PETSC_DECIDE` or `PETSC_DETERMINE`)
50
51 Input Parameters:
52 + comm - MPI communicator that shares the object being divided
53 . n - local length (or `PETSC_DECIDE` to have it set)
54 - N - global length (or `PETSC_DETERMINE` to have it set)
55
56 Level: developer
57
58 Notes:
59 `n` and `N` cannot be both `PETSC_DECIDE` and `PETSC_DETERMINE`
60
61 If one processor calls this with `n` of `PETSC_DECIDE` (or with `N` `PETSC_DETERMINE`) then all processors
62 must. Otherwise, an error is thrown in debug mode while the program will hang
63 in optimized (i.e. configured --with-debugging=0) mode.
64
65 .seealso: `PetscSplitOwnershipBlock()`, `PetscSplitOwnershipEqual()`, `PETSC_DECIDE`, `PETSC_DETERMINE`
66 @*/
PetscSplitOwnership(MPI_Comm comm,PetscInt * n,PetscInt * N)67 PetscErrorCode PetscSplitOwnership(MPI_Comm comm, PetscInt *n, PetscInt *N)
68 {
69 PetscMPIInt size, rank;
70
71 PetscFunctionBegin;
72 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Both n and N cannot be PETSC_DECIDE, likely a call to VecSetSizes() or MatSetSizes() is wrong. See https://petsc.org/release/faq/#split-ownership");
73 if (PetscDefined(USE_DEBUG)) {
74 PetscMPIInt l[2], g[2];
75 l[0] = (*n == PETSC_DECIDE) ? 1 : 0;
76 l[1] = (*N == PETSC_DECIDE) ? 1 : 0;
77 PetscCallMPI(MPI_Comm_size(comm, &size));
78 PetscCallMPI(MPIU_Allreduce(l, g, 2, MPI_INT, MPI_SUM, comm));
79 PetscCheck(!g[0] || g[0] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for local size");
80 PetscCheck(!g[1] || g[1] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for global size");
81 }
82
83 if (*N == PETSC_DECIDE) {
84 PetscCallMPI(MPIU_Allreduce(n, N, 1, MPIU_INT, MPI_SUM, comm));
85 } else if (*n == PETSC_DECIDE) {
86 PetscCallMPI(MPI_Comm_size(comm, &size));
87 PetscCallMPI(MPI_Comm_rank(comm, &rank));
88 *n = *N / size + ((*N % size) > rank);
89 } else if (PetscDefined(USE_DEBUG)) {
90 PetscInt tmp;
91 PetscCallMPI(MPIU_Allreduce(n, &tmp, 1, MPIU_INT, MPI_SUM, comm));
92 PetscCheck(tmp == *N, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Sum of local lengths %" PetscInt_FMT " does not equal global length %" PetscInt_FMT ", my local length %" PetscInt_FMT ", likely a call to VecSetSizes() or MatSetSizes() is wrong. See https://petsc.org/release/faq/#split-ownership", tmp, *N, *n);
93 }
94 PetscFunctionReturn(PETSC_SUCCESS);
95 }
96
97 /*@
98 PetscSplitOwnershipEqual - Given a global (or local) length determines a local
99 (or global) length via a simple formula, trying to have all local lengths equal
100
101 Collective (if `n` or `N` is `PETSC_DECIDE`)
102
103 Input Parameters:
104 + comm - MPI communicator that shares the object being divided
105 . n - local length (or `PETSC_DECIDE` to have it set)
106 - N - global length (or `PETSC_DECIDE`)
107
108 Level: developer
109
110 Notes:
111 This is intended to be used with `MATSCALAPACK`, where the local size must
112 be equal in all processes (except possibly the last one). For instance,
113 the local sizes when splitting `N`=50 with 6 processes are 9,9,9,9,9,5
114
115 n and N cannot be both `PETSC_DECIDE`
116
117 If one processor calls this with `n` or `N` of `PETSC_DECIDE` then all processors
118 must. Otherwise, an error is thrown in debug mode while the program will hang
119 in optimized (i.e. configured --with-debugging=0) mode.
120
121 .seealso: `PetscSplitOwnership()`, `PetscSplitOwnershipBlock()`
122 @*/
PetscSplitOwnershipEqual(MPI_Comm comm,PetscInt * n,PetscInt * N)123 PetscErrorCode PetscSplitOwnershipEqual(MPI_Comm comm, PetscInt *n, PetscInt *N)
124 {
125 PetscMPIInt size, rank;
126
127 PetscFunctionBegin;
128 PetscCheck(*N != PETSC_DECIDE || *n != PETSC_DECIDE, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Both n and N cannot be PETSC_DECIDE");
129 if (PetscDefined(USE_DEBUG)) {
130 PetscMPIInt l[2], g[2];
131 l[0] = (*n == PETSC_DECIDE) ? 1 : 0;
132 l[1] = (*N == PETSC_DECIDE) ? 1 : 0;
133 PetscCallMPI(MPI_Comm_size(comm, &size));
134 PetscCallMPI(MPIU_Allreduce(l, g, 2, MPI_INT, MPI_SUM, comm));
135 PetscCheck(!g[0] || g[0] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for local size");
136 PetscCheck(!g[1] || g[1] == size, comm, PETSC_ERR_ARG_INCOMP, "All processes must supply PETSC_DECIDE for global size");
137 }
138
139 if (*N == PETSC_DECIDE) {
140 PetscCallMPI(MPIU_Allreduce(n, N, 1, MPIU_INT, MPI_SUM, comm));
141 } else if (*n == PETSC_DECIDE) {
142 PetscCallMPI(MPI_Comm_size(comm, &size));
143 PetscCallMPI(MPI_Comm_rank(comm, &rank));
144 *n = *N / size;
145 if (*N % size) {
146 if ((rank + 1) * (*n + 1) <= *N) *n = *n + 1;
147 else if (rank * (*n + 1) <= *N) *n = *N - rank * (*n + 1);
148 else *n = 0;
149 }
150 } else if (PetscDefined(USE_DEBUG)) {
151 PetscInt tmp;
152 PetscCallMPI(MPIU_Allreduce(n, &tmp, 1, MPIU_INT, MPI_SUM, comm));
153 PetscCheck(tmp == *N, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Sum of local lengths %" PetscInt_FMT " does not equal global length %" PetscInt_FMT ", my local length %" PetscInt_FMT, tmp, *N, *n);
154 }
155 PetscFunctionReturn(PETSC_SUCCESS);
156 }
157