1 #include <stdio.h>
2 #include <stdlib.h>
3 #include <string.h>
4 #include <assert.h>
5
6 #include <mpi.h>
7
8 #include "common_c.h"
9
10 #include "FCMangle.h"
11
12 //commu_int.f
13 #define commu_int FortranCInterface_GLOBAL_(commu_int, COMMU_INT)
14 void commu_int(int* global, int* ilwork, int* n, char* code);
15
16 #define gen_ncorp FortranCInterface_GLOBAL_(gen_ncorp, GEN_NCORP)
17
18 // KEJ changed ncorp_t to lcorp_t (used for on proc numbering
19 // and introduced gcorp_t for global numbering
20 typedef int lcorp_t;
21 #define NCORP_MPI_T MPI_INTEGER
22 typedef long long int gcorp_t;
23
24 static lcorp_t count_owned(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes);
25 static lcorp_t count_local(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes);
26
27
gen_ncorp(gcorp_t * ncorp,int * ilwork,lcorp_t * nilwork_f,int * num_nodes)28 void gen_ncorp(gcorp_t* ncorp, int* ilwork, lcorp_t* nilwork_f, int* num_nodes)
29 {
30 int part;
31 int num_parts;
32 int i;
33 lcorp_t nilwork = *nilwork_f;
34 lcorp_t owned;
35 lcorp_t local;
36 lcorp_t* owner_counts;
37 gcorp_t local_start_id;
38 gcorp_t gid;
39
40 MPI_Comm_rank(MPI_COMM_WORLD, &part);
41 MPI_Comm_size(MPI_COMM_WORLD, &num_parts);
42
43 memset(ncorp, 0, sizeof(gcorp_t)*(*num_nodes));
44 owned = count_owned(ilwork, nilwork, ncorp, num_nodes);
45 local = count_local(ilwork, nilwork, ncorp, num_nodes);
46 conpar.iownnodes = owned+local;
47 #ifdef PRINT_EVERYTHING
48 printf("%d: %d local only nodes\n", part, local);
49 printf("%d: %d owned nodes\n", part, owned);
50 #endif
51 assert( owned <= *num_nodes );
52 assert( owned+local <= *num_nodes );
53
54 owner_counts = (lcorp_t*) malloc(sizeof(lcorp_t)*num_parts);
55 memset(owner_counts, 0, sizeof(lcorp_t)*num_parts);
56 owner_counts[part] = owned+local;
57 #ifdef PRINT_EVERYTHING
58 for(i=0;i<num_parts;i++)
59 {
60 printf("%d,", owner_counts[i]);
61 }
62 printf("\n");
63 #endif
64 MPI_Allgather(MPI_IN_PLACE, 1, NCORP_MPI_T, owner_counts,
65 1, NCORP_MPI_T, MPI_COMM_WORLD);
66 #ifdef PRINT_EVERYTHING
67 for(i=0;i<num_parts;i++)
68 {
69 printf("%d,", owner_counts[i]);
70 }
71 printf("\n");
72 #endif
73 local_start_id=0;
74 for(i=0;i<part;i++) //TODO: MPI_Exscan()?
75 {
76 // global so needs long long
77 local_start_id += owner_counts[i];
78 }
79 local_start_id++; //Fortran numbering
80 #ifdef PRINT_EVERYTHING
81 printf("%d: %d\n", part, local_start_id);
82 #endif
83 // global so needs long long
84 gid = local_start_id;
85 if(gid<0) printf("part,gid, %d %ld",part,gid);
86 assert(gid>=0);
87 for(i=0;i<*num_nodes;i++) //assign owned node's numbers
88 {
89 //if shared, owned 1
90 //if shared, slave -1
91 //if local only, 0
92 if(ncorp[i] == 1)
93 {
94 // global so needs long long
95 ncorp[i]=gid;
96 assert(ncorp[i]>=0);
97
98 // global so needs long long
99 gid++;
100 continue;
101 }
102 if(ncorp[i] == 0)
103 {
104 ncorp[i] = gid;
105 assert(ncorp[i]>=0);
106 gid++;
107 continue;
108 }
109 if(ncorp[i] == -1)
110 {
111 ncorp[i] = 0; //commu() adds, so zero slaves
112 }
113
114 }
115 //char code[] = "out";
116 //int ione = 1;
117 //commu_int(ncorp, ilwork, &ione, code);
118
119 }
120
count_local(int * ilwork,int nlwork,gcorp_t * ncorp_tmp,int * num_nodes)121 static lcorp_t count_local(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes)
122 {
123 int i;
124 lcorp_t num_local = 0;
125 for(i=0;i<*num_nodes;i++)
126 {
127 if(ncorp_tmp[i] == 0)
128 num_local++; //nodes away from part boundary
129 assert(!(ncorp_tmp[i] < -1 || ncorp_tmp[i] > 1));
130 }
131 return(num_local);
132 }
count_owned(int * ilwork,int nlwork,gcorp_t * ncorp_tmp,int * num_nodes)133 static lcorp_t count_owned(int* ilwork, int nlwork,gcorp_t* ncorp_tmp, int* num_nodes)
134 {
135 int numtask = ilwork[0];
136 int itkbeg = 0; //task offset
137 int owned = 0;
138 int i,j,k;
139 for(i=0;i<numtask;i++)
140 {
141 int itag = ilwork[itkbeg+1]; //mpi tag
142 int iacc = ilwork[itkbeg+2]; //0 for slave, 1 for master
143 assert(iacc >= 0 && iacc <= 1);
144 int iother = ilwork[itkbeg+3]-1; //other rank (see ctypes.f for off by one)
145 int numseg = ilwork[itkbeg+4]; //number of segments
146 for(j=0;j<numseg;j++)
147 {
148 int isgbeg = ilwork[itkbeg+5+(j*2)]; //first idx of seg
149 int lenseg = ilwork[itkbeg+6+(j*2)]; //length of seg
150 assert(iacc == 0 || iacc == 1);
151 if(iacc)
152 {
153 for(k=0;k<lenseg;k++)
154 {
155 if(ncorp_tmp[isgbeg-1+k] == 0)
156 owned++;
157 //make sure we're not both master and slave
158 assert(ncorp_tmp[isgbeg-1+k] != -1);
159 ncorp_tmp[isgbeg-1+k] = 1;
160 assert(isgbeg-1+k < *num_nodes);
161 }
162 assert(owned <= *num_nodes);
163 }
164 else
165 {
166 for(k=0;k<lenseg;k++)
167 {
168 ncorp_tmp[isgbeg-1+k] = -1;
169 assert(isgbeg-1+k < *num_nodes);
170 }
171 }
172 //ncorp_tmp init'd to 0
173 //if shared, owned 1
174 //if shared, slave -1
175 //if local only, 0
176
177 assert(itkbeg+6+(j*2) < nlwork);
178 }
179 itkbeg+= 4+2*numseg;
180 }
181 return(owned);
182 }
183