xref: /petsc/include/petsclog.h (revision efa12513287cff49a2b9648ae83199dcbfaad71a)
1 /*
2     Defines profile/logging in PETSc.
3 */
4 
5 #if !defined(PETSCLOG_H)
6 #define PETSCLOG_H
7 #include <petscsys.h>
8 #include <petsctime.h>
9 
10 /* General logging of information; different from event logging */
11 PETSC_EXTERN PetscErrorCode PetscInfo_Private(const char[],PetscObject,const char[],...);
12 #if defined(PETSC_USE_INFO)
13 #define PetscInfo(A,S)                             PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S)
14 #define PetscInfo1(A,S,a1)                         PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1)
15 #define PetscInfo2(A,S,a1,a2)                      PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2)
16 #define PetscInfo3(A,S,a1,a2,a3)                   PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3)
17 #define PetscInfo4(A,S,a1,a2,a3,a4)                PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4)
18 #define PetscInfo5(A,S,a1,a2,a3,a4,a5)             PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5)
19 #define PetscInfo6(A,S,a1,a2,a3,a4,a5,a6)          PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5,a6)
20 #define PetscInfo7(A,S,a1,a2,a3,a4,a5,a6,a7)       PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5,a6,a7)
21 #define PetscInfo8(A,S,a1,a2,a3,a4,a5,a6,a7,a8)    PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5,a6,a7,a8)
22 #define PetscInfo9(A,S,a1,a2,a3,a4,a5,a6,a7,a8,a9) PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5,a6,a7,a8,a9)
23 #else
24 #define PetscInfo(A,S)                             0
25 #define PetscInfo1(A,S,a1)                         0
26 #define PetscInfo2(A,S,a1,a2)                      0
27 #define PetscInfo3(A,S,a1,a2,a3)                   0
28 #define PetscInfo4(A,S,a1,a2,a3,a4)                0
29 #define PetscInfo5(A,S,a1,a2,a3,a4,a5)             0
30 #define PetscInfo6(A,S,a1,a2,a3,a4,a5,a6)          0
31 #define PetscInfo7(A,S,a1,a2,a3,a4,a5,a6,a7)       0
32 #define PetscInfo8(A,S,a1,a2,a3,a4,a5,a6,a7,a8)    0
33 #define PetscInfo9(A,S,a1,a2,a3,a4,a5,a6,a7,a8,a9) 0
34 #endif
35 
36 /*E
37     PetscInfoCommFlag - Describes the method by which to filter PetscInfo() by communicator size
38 
39     Used as an input for PetscInfoSetFilterCommSelf()
40 
41 $   PETSC_INFO_COMM_ALL - Default uninitialized value. PetscInfo() will not filter based on communicator size (i.e. will
42 print for all communicators)
43 $   PETSC_INFO_COMM_NO_SELF - PetscInfo() will NOT print for communicators with size = 1 (i.e. *_COMM_SELF)
44 $   PETSC_INFO_COMM_ONLY_SELF - PetscInfo will ONLY print for communicators with size = 1
45 
46     Level: intermediate
47 
48 .seealso: PetscInfo(), PetscInfoSetFromOptions(), PetscInfoSetFilterCommSelf()
49 E*/
50 typedef enum {
51   PETSC_INFO_COMM_ALL = -1,
52   PETSC_INFO_COMM_NO_SELF = 0,
53   PETSC_INFO_COMM_ONLY_SELF = 1
54 } PetscInfoCommFlag;
55 
56 PETSC_EXTERN const char * const PetscInfoCommFlags[];
57 PETSC_EXTERN PetscErrorCode PetscInfoDeactivateClass(PetscClassId);
58 PETSC_EXTERN PetscErrorCode PetscInfoActivateClass(PetscClassId);
59 PETSC_EXTERN PetscErrorCode PetscInfoEnabled(PetscClassId, PetscBool *);
60 PETSC_EXTERN PetscErrorCode PetscInfoAllow(PetscBool);
61 PETSC_EXTERN PetscErrorCode PetscInfoSetFile(const char[],const char[]);
62 PETSC_EXTERN PetscErrorCode PetscInfoGetFile(char **,FILE **);
63 PETSC_EXTERN PetscErrorCode PetscInfoSetClasses(PetscBool,PetscInt,const char *const *);
64 PETSC_EXTERN PetscErrorCode PetscInfoGetClass(const char *, PetscBool *);
65 PETSC_EXTERN PetscErrorCode PetscInfoGetInfo(PetscBool *,PetscBool *,PetscBool *,PetscBool *,PetscInfoCommFlag *);
66 PETSC_EXTERN PetscErrorCode PetscInfoProcessClass(const char[],PetscInt,PetscClassId[]);
67 PETSC_EXTERN PetscErrorCode PetscInfoSetFilterCommSelf(PetscInfoCommFlag);
68 PETSC_EXTERN PetscErrorCode PetscInfoSetFromOptions(PetscOptions);
69 PETSC_EXTERN PetscErrorCode PetscInfoDestroy(void);
70 PETSC_EXTERN PetscBool      PetscLogPrintInfo;  /* if true, indicates PetscInfo() is turned on */
71 
72 /*MC
73     PetscLogEvent - id used to identify PETSc or user events which timed portions (blocks of executable)
74      code.
75 
76     Level: intermediate
77 
78 .seealso: PetscLogEventRegister(), PetscLogEventBegin(), PetscLogEventEnd(), PetscLogStage
79 M*/
80 typedef int PetscLogEvent;
81 
82 /*MC
83     PetscLogStage - id used to identify user stages (phases, sections) of runs - for logging
84 
85     Level: intermediate
86 
87 .seealso: PetscLogStageRegister(), PetscLogStagePush(), PetscLogStagePop(), PetscLogEvent
88 M*/
89 typedef int PetscLogStage;
90 
91 #define PETSC_EVENT  1311311
92 PETSC_EXTERN PetscLogEvent PETSC_LARGEST_EVENT;
93 
94 /* Global flop counter */
95 PETSC_EXTERN PetscLogDouble petsc_TotalFlops;
96 PETSC_EXTERN PetscLogDouble petsc_tmp_flops;
97 
98 /* We must make the following structures available to access the event
99      activation flags in the PetscLogEventBegin/End() macros. These are not part of the PETSc public
100      API and are not intended to be used by other parts of PETSc or by users.
101 
102      The code that manipulates these structures is in src/sys/logging/utils.
103 */
104 typedef struct _n_PetscIntStack *PetscIntStack;
105 
106 /* -----------------------------------------------------------------------------------------------------*/
107 /*
108     PetscClassRegInfo, PetscClassPerfInfo - Each class has two data structures associated with it. The first has
109        static information about it, the second collects statistics on how many objects of the class are created,
110        how much memory they use, etc.
111 
112     PetscClassRegLog, PetscClassPerfLog - arrays of the PetscClassRegInfo and PetscClassPerfInfo for all classes.
113 */
114 typedef struct  {
115   char           *name;   /* The class name */
116   PetscClassId   classid; /* The integer identifying this class */
117 } PetscClassRegInfo;
118 
119 typedef struct {
120   PetscClassId   id;           /* The integer identifying this class */
121   int            creations;    /* The number of objects of this class created */
122   int            destructions; /* The number of objects of this class destroyed */
123   PetscLogDouble mem;          /* The total memory allocated by objects of this class */
124   PetscLogDouble descMem;      /* The total memory allocated by descendents of these objects */
125 } PetscClassPerfInfo;
126 
127 typedef struct _n_PetscClassRegLog *PetscClassRegLog;
128 struct _n_PetscClassRegLog {
129   int               numClasses; /* The number of classes registered */
130   int               maxClasses; /* The maximum number of classes */
131   PetscClassRegInfo *classInfo; /* The structure for class information (classids are monotonicly increasing) */
132 };
133 
134 typedef struct _n_PetscClassPerfLog *PetscClassPerfLog;
135 struct _n_PetscClassPerfLog {
136   int                numClasses; /* The number of logging classes */
137   int                maxClasses; /* The maximum number of classes */
138   PetscClassPerfInfo *classInfo; /* The structure for class information (classids are monotonicly increasing) */
139 };
140 /* -----------------------------------------------------------------------------------------------------*/
141 /*
142     PetscEventRegInfo, PetscEventPerfInfo - Each event has two data structures associated with it. The first has
143        static information about it, the second collects statistics on how many times the event is used, how
144        much time it takes, etc.
145 
146     PetscEventRegLog, PetscEventPerfLog - an array of all PetscEventRegInfo and PetscEventPerfInfo for all events. There is one
147       of these for each stage.
148 
149 */
150 typedef struct {
151   char         *name;         /* The name of this event */
152   PetscClassId classid;       /* The class the event is associated with */
153   PetscBool    collective;    /* Flag this event as collective */
154 #if defined (PETSC_HAVE_MPE)
155   int          mpe_id_begin;  /* MPE IDs that define the event */
156   int          mpe_id_end;
157 #endif
158 } PetscEventRegInfo;
159 
160 typedef struct {
161   int            id;            /* The integer identifying this event */
162   PetscBool      active;        /* The flag to activate logging */
163   PetscBool      visible;       /* The flag to print info in summary */
164   int            depth;         /* The nesting depth of the event call */
165   int            count;         /* The number of times this event was executed */
166   PetscLogDouble flops, flops2, flopsTmp; /* The flops and flops^2 used in this event */
167   PetscLogDouble time, time2, timeTmp;    /* The time and time^2 taken for this event */
168   PetscLogDouble syncTime;                /* The synchronization barrier time */
169   PetscLogDouble dof[8];        /* The number of degrees of freedom associated with this event */
170   PetscLogDouble errors[8];     /* The errors (user-defined) associated with this event */
171   PetscLogDouble numMessages;   /* The number of messages in this event */
172   PetscLogDouble messageLength; /* The total message lengths in this event */
173   PetscLogDouble numReductions; /* The number of reductions in this event */
174   PetscLogDouble memIncrease;   /* How much the resident memory has increased in this event */
175   PetscLogDouble mallocIncrease;/* How much the maximum malloced space has increased in this event */
176   PetscLogDouble mallocSpace;   /* How much the space was malloced and kept during this event */
177   PetscLogDouble mallocIncreaseEvent;  /* Maximum of the high water mark with in event minus memory available at the end of the event */
178   #if defined(PETSC_HAVE_DEVICE)
179   PetscLogDouble CpuToGpuCount; /* The total number of CPU to GPU copies */
180   PetscLogDouble GpuToCpuCount; /* The total number of GPU to CPU copies */
181   PetscLogDouble CpuToGpuSize;  /* The total size of CPU to GPU copies */
182   PetscLogDouble GpuToCpuSize;  /* The total size of GPU to CPU copies */
183   PetscLogDouble GpuFlops;      /* The flops done on a GPU in this event */
184   PetscLogDouble GpuTime;       /* The time spent on a GPU in this event */
185   #endif
186 } PetscEventPerfInfo;
187 
188 typedef struct _n_PetscEventRegLog *PetscEventRegLog;
189 struct _n_PetscEventRegLog {
190   int               numEvents;  /* The number of registered events */
191   int               maxEvents;  /* The maximum number of events */
192   PetscEventRegInfo *eventInfo; /* The registration information for each event */
193 };
194 
195 typedef struct _n_PetscEventPerfLog *PetscEventPerfLog;
196 struct _n_PetscEventPerfLog {
197   int                numEvents;  /* The number of logging events */
198   int                maxEvents;  /* The maximum number of events */
199   PetscEventPerfInfo *eventInfo; /* The performance information for each event */
200 };
201 /* ------------------------------------------------------------------------------------------------------------*/
202 /*
203    PetscStageInfo - Contains all the information about a particular stage.
204 
205    PetscStageLog - An array of PetscStageInfo for each registered stage. There is a single one of these in the code.
206 */
207 typedef struct _PetscStageInfo {
208   char               *name;     /* The stage name */
209   PetscBool          used;      /* The stage was pushed on this processor */
210   PetscEventPerfInfo perfInfo;  /* The stage performance information */
211   PetscEventPerfLog  eventLog;  /* The event information for this stage */
212   PetscClassPerfLog  classLog;  /* The class information for this stage */
213 } PetscStageInfo;
214 
215 typedef struct _n_PetscStageLog *PetscStageLog;
216 struct _n_PetscStageLog {
217   int              numStages;   /* The number of registered stages */
218   int              maxStages;   /* The maximum number of stages */
219   PetscIntStack    stack;       /* The stack for active stages */
220   int              curStage;    /* The current stage (only used in macros so we don't call PetscIntStackTop) */
221   PetscStageInfo   *stageInfo;  /* The information for each stage */
222   PetscEventRegLog eventLog;    /* The registered events */
223   PetscClassRegLog classLog;    /* The registered classes */
224 };
225 /* -----------------------------------------------------------------------------------------------------*/
226 
227 PETSC_EXTERN PetscErrorCode PetscLogObjectParent(PetscObject,PetscObject);
228 PETSC_EXTERN PetscErrorCode PetscLogObjectMemory(PetscObject,PetscLogDouble);
229 
230 #if defined(PETSC_USE_LOG)  /* --- Logging is turned on --------------------------------*/
231 PETSC_EXTERN PetscStageLog petsc_stageLog;
232 PETSC_EXTERN PetscErrorCode PetscLogGetStageLog(PetscStageLog*);
233 PETSC_EXTERN PetscErrorCode PetscStageLogGetCurrent(PetscStageLog,int*);
234 PETSC_EXTERN PetscErrorCode PetscStageLogGetEventPerfLog(PetscStageLog,int,PetscEventPerfLog*);
235 
236 /*
237    Flop counting:  We count each arithmetic operation (e.g., addition, multiplication) separately.
238 
239    For the complex numbers version, note that
240        1 complex addition = 2 flops
241        1 complex multiplication = 6 flops,
242    where we define 1 flop as that for a double precision scalar.  We roughly approximate
243    flop counting for complex numbers by multiplying the total flops by 4; this corresponds
244    to the assumption that we're counting mostly additions and multiplications -- and
245    roughly the same number of each.  More accurate counting could be done by distinguishing
246    among the various arithmetic operations.
247  */
248 
249 #if defined(PETSC_USE_COMPLEX)
250 #define PETSC_FLOPS_PER_OP 4.0
251 #else
252 #define PETSC_FLOPS_PER_OP 1.0
253 #endif
254 
255 /*@
256        PetscLogFlops - Log how many flops are performed in a calculation
257 
258    Input Paramters:
259     flops - the number of flops
260 
261    Notes:
262      To limit the chance of integer overflow when multiplying by a constant, represent the constant as a double,
263      not an integer. Use PetscLogFlops(4.0*n) not PetscLogFlops(4*n)
264 
265    Level: intermediate
266 
267 .seealso: PetscLogView(), PetscLogGpuFlops()
268 @*/
269 
270 PETSC_STATIC_INLINE PetscErrorCode PetscLogFlops(PetscLogDouble n)
271 {
272   PetscFunctionBegin;
273 #if defined(PETSC_USE_DEBUG)
274   if (n < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Cannot log negative flops");
275 #endif
276   petsc_TotalFlops += PETSC_FLOPS_PER_OP*n;
277   PetscFunctionReturn(0);
278 }
279 
280 #if defined(PETSC_HAVE_DEVICE)
281 /* Global GPU counters */
282 PETSC_EXTERN PetscLogDouble petsc_ctog_ct;
283 PETSC_EXTERN PetscLogDouble petsc_gtoc_ct;
284 PETSC_EXTERN PetscLogDouble petsc_ctog_sz;
285 PETSC_EXTERN PetscLogDouble petsc_gtoc_sz;
286 PETSC_EXTERN PetscLogDouble petsc_gflops;
287 PETSC_EXTERN PetscLogDouble petsc_gtime;
288 #if defined(PETSC_USE_DEBUG)
289 PETSC_EXTERN PetscBool      petsc_gtime_inuse;
290 #endif
291 
292 PETSC_STATIC_INLINE PetscErrorCode PetscLogCpuToGpu(PetscLogDouble size)
293 {
294   PetscFunctionBegin;
295   petsc_ctog_ct += 1;
296   petsc_ctog_sz += size;
297   PetscFunctionReturn(0);
298 }
299 
300 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuToCpu(PetscLogDouble size)
301 {
302   PetscFunctionBegin;
303   petsc_gtoc_ct += 1;
304   petsc_gtoc_sz += size;
305   PetscFunctionReturn(0);
306 }
307 
308 /*@
309        PetscLogGpuFlops - Log how many flops are performed in a calculation on the device
310 
311    Input Paramters:
312     flops - the number of flops
313 
314    Notes:
315      To limit the chance of integer overflow when multiplying by a constant, represent the constant as a double,
316      not an integer. Use PetscLogFlops(4.0*n) not PetscLogFlops(4*n)
317 
318    Level: intermediate
319 
320 .seealso: PetscLogView(), PetscLogFlops(), PetscLogGpuTimeBegin(), PetscLogGpuTimeEnd()
321 @*/
322 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuFlops(PetscLogDouble n)
323 {
324   PetscFunctionBegin;
325 #if defined(PETSC_USE_DEBUG)
326   if (n < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Cannot log negative flops");
327 #endif
328   petsc_TotalFlops += PETSC_FLOPS_PER_OP*n;
329   petsc_gflops += PETSC_FLOPS_PER_OP*n;
330   PetscFunctionReturn(0);
331 }
332 
333 /*@
334        PetscLogGpuTimeBegin - Start timer for device
335 
336    Level: intermediate
337 
338       Notes:
339         The timer is run on the CPU, it is a separate logging of time devoted to GPU computations (including kernel launch times).
340         This timer should NOT include times for data transfers between the GPU and CPU, nor setup actions such as allocating space.
341         The regular logging captures the time for data transfers and any CPU activites during the event
342         It is used to compute the flop rate on the GPU as it is actively engaged in running a kernel.
343 
344 
345 .seealso:  PetscLogView(), PetscLogGpuFlops(), PetscLogGpuTimeEnd()
346 @*/
347 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuTimeBegin()
348 {
349   PetscErrorCode ierr;
350   PetscFunctionBegin;
351 #if defined(PETSC_USE_DEBUG)
352   if (petsc_gtime_inuse) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Forgot to call PetscLogGpuTimeEnd()?");
353   petsc_gtime_inuse = PETSC_TRUE;
354 #endif
355   ierr = PetscTimeSubtract(&petsc_gtime);CHKERRQ(ierr);
356   PetscFunctionReturn(0);
357 }
358 /*@
359        PetscLogGpuTimeEnd - Stop timer for device
360 
361    Level: intermediate
362 
363 .seealso:  PetscLogView(), PetscLogGpuFlops(), PetscLogGpuTimeBegin()
364 @*/
365 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuTimeEnd()
366 {
367   PetscErrorCode ierr;
368   PetscFunctionBegin;
369 #if defined(PETSC_USE_DEBUG)
370   if (!petsc_gtime_inuse) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Forgot to call PetscLogGpuTimeBegin()?");
371   petsc_gtime_inuse = PETSC_FALSE;
372 #endif
373   ierr = PetscTimeAdd(&petsc_gtime);CHKERRQ(ierr);
374   PetscFunctionReturn(0);
375 }
376 
377 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuTimeAdd(PetscLogDouble t)
378 {
379   PetscFunctionBegin;
380   petsc_gtime += t;
381   PetscFunctionReturn(0);
382 }
383 #endif
384 
385 PETSC_EXTERN PetscErrorCode PetscGetFlops(PetscLogDouble *);
386 
387 #if defined (PETSC_HAVE_MPE)
388 PETSC_EXTERN PetscErrorCode PetscLogMPEBegin(void);
389 PETSC_EXTERN PetscErrorCode PetscLogMPEDump(const char[]);
390 #endif
391 
392 PETSC_EXTERN PetscErrorCode (*PetscLogPLB)(PetscLogEvent,int,PetscObject,PetscObject,PetscObject,PetscObject);
393 PETSC_EXTERN PetscErrorCode (*PetscLogPLE)(PetscLogEvent,int,PetscObject,PetscObject,PetscObject,PetscObject);
394 PETSC_EXTERN PetscErrorCode (*PetscLogPHC)(PetscObject);
395 PETSC_EXTERN PetscErrorCode (*PetscLogPHD)(PetscObject);
396 
397 #define PetscLogObjectParents(p,n,d)  0;do{int _i; for (_i=0; _i<(n); _i++) {ierr = PetscLogObjectParent((PetscObject)(p),(PetscObject)(d)[_i]);CHKERRQ(ierr);}}while (0)
398 #define PetscLogObjectCreate(h)      ((PetscLogPHC) ? (*PetscLogPHC)((PetscObject)(h)) : 0)
399 #define PetscLogObjectDestroy(h)     ((PetscLogPHD) ? (*PetscLogPHD)((PetscObject)(h)) : 0)
400 PETSC_EXTERN PetscErrorCode PetscLogObjectState(PetscObject, const char[], ...);
401 
402 /* Initialization functions */
403 PETSC_EXTERN PetscErrorCode PetscLogDefaultBegin(void);
404 PETSC_EXTERN PetscErrorCode PetscLogAllBegin(void);
405 PETSC_EXTERN PetscErrorCode PetscLogNestedBegin(void);
406 PETSC_EXTERN PetscErrorCode PetscLogTraceBegin(FILE *);
407 PETSC_EXTERN PetscErrorCode PetscLogActions(PetscBool);
408 PETSC_EXTERN PetscErrorCode PetscLogObjects(PetscBool);
409 PETSC_EXTERN PetscErrorCode PetscLogSetThreshold(PetscLogDouble,PetscLogDouble*);
410 PETSC_EXTERN PetscErrorCode PetscLogSet(PetscErrorCode (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject),
411                                         PetscErrorCode (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject));
412 
413 /* Output functions */
414 PETSC_EXTERN PetscErrorCode PetscLogView(PetscViewer);
415 PETSC_EXTERN PetscErrorCode PetscLogViewFromOptions(void);
416 PETSC_EXTERN PetscErrorCode PetscLogDump(const char[]);
417 
418 /* Stage functions */
419 PETSC_EXTERN PetscErrorCode PetscLogStageRegister(const char[],PetscLogStage*);
420 PETSC_EXTERN PetscErrorCode PetscLogStagePush(PetscLogStage);
421 PETSC_EXTERN PetscErrorCode PetscLogStagePop(void);
422 PETSC_EXTERN PetscErrorCode PetscLogStageSetActive(PetscLogStage,PetscBool);
423 PETSC_EXTERN PetscErrorCode PetscLogStageGetActive(PetscLogStage,PetscBool*);
424 PETSC_EXTERN PetscErrorCode PetscLogStageSetVisible(PetscLogStage,PetscBool);
425 PETSC_EXTERN PetscErrorCode PetscLogStageGetVisible(PetscLogStage,PetscBool*);
426 PETSC_EXTERN PetscErrorCode PetscLogStageGetId(const char[],PetscLogStage*);
427 
428 /* Event functions */
429 PETSC_EXTERN PetscErrorCode PetscLogEventRegister(const char[],PetscClassId,PetscLogEvent*);
430 PETSC_EXTERN PetscErrorCode PetscLogEventSetCollective(PetscLogEvent,PetscBool);
431 PETSC_EXTERN PetscErrorCode PetscLogEventIncludeClass(PetscClassId);
432 PETSC_EXTERN PetscErrorCode PetscLogEventExcludeClass(PetscClassId);
433 PETSC_EXTERN PetscErrorCode PetscLogEventActivate(PetscLogEvent);
434 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivate(PetscLogEvent);
435 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivatePush(PetscLogEvent);
436 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivatePop(PetscLogEvent);
437 PETSC_EXTERN PetscErrorCode PetscLogEventSetActiveAll(PetscLogEvent,PetscBool);
438 PETSC_EXTERN PetscErrorCode PetscLogEventActivateClass(PetscClassId);
439 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivateClass(PetscClassId);
440 PETSC_EXTERN PetscErrorCode PetscLogEventGetId(const char[],PetscLogEvent*);
441 PETSC_EXTERN PetscErrorCode PetscLogEventGetPerfInfo(int,PetscLogEvent,PetscEventPerfInfo*);
442 PETSC_EXTERN PetscErrorCode PetscLogEventSetDof(PetscLogEvent, PetscInt, PetscLogDouble);
443 PETSC_EXTERN PetscErrorCode PetscLogEventSetError(PetscLogEvent, PetscInt, PetscLogDouble);
444 
445 /* Global counters */
446 PETSC_EXTERN PetscLogDouble petsc_irecv_ct;
447 PETSC_EXTERN PetscLogDouble petsc_isend_ct;
448 PETSC_EXTERN PetscLogDouble petsc_recv_ct;
449 PETSC_EXTERN PetscLogDouble petsc_send_ct;
450 PETSC_EXTERN PetscLogDouble petsc_irecv_len;
451 PETSC_EXTERN PetscLogDouble petsc_isend_len;
452 PETSC_EXTERN PetscLogDouble petsc_recv_len;
453 PETSC_EXTERN PetscLogDouble petsc_send_len;
454 PETSC_EXTERN PetscLogDouble petsc_allreduce_ct;
455 PETSC_EXTERN PetscLogDouble petsc_gather_ct;
456 PETSC_EXTERN PetscLogDouble petsc_scatter_ct;
457 PETSC_EXTERN PetscLogDouble petsc_wait_ct;
458 PETSC_EXTERN PetscLogDouble petsc_wait_any_ct;
459 PETSC_EXTERN PetscLogDouble petsc_wait_all_ct;
460 PETSC_EXTERN PetscLogDouble petsc_sum_of_waits_ct;
461 
462 PETSC_EXTERN PetscBool      PetscLogMemory;
463 
464 PETSC_EXTERN PetscBool PetscLogSyncOn;  /* true if logging synchronization is enabled */
465 PETSC_EXTERN PetscErrorCode PetscLogEventSynchronize(PetscLogEvent, MPI_Comm);
466 
467 #define PetscLogEventSync(e,comm) \
468   (((PetscLogPLB && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? \
469     PetscLogEventSynchronize((e),(comm)) : 0))
470 
471 #define PetscLogEventBegin(e,o1,o2,o3,o4) \
472   (((PetscLogPLB && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? \
473     (*PetscLogPLB)((e),0,(PetscObject)(o1),(PetscObject)(o2),(PetscObject)(o3),(PetscObject)(o4)) : 0))
474 
475 #define PetscLogEventEnd(e,o1,o2,o3,o4) \
476   (((PetscLogPLE && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? \
477     (*PetscLogPLE)((e),0,(PetscObject)(o1),(PetscObject)(o2),(PetscObject)(o3),(PetscObject)(o4)) : 0))
478 
479 PETSC_EXTERN PetscErrorCode PetscLogEventGetFlops(PetscLogEvent,PetscLogDouble*);
480 PETSC_EXTERN PetscErrorCode PetscLogEventZeroFlops(PetscLogEvent);
481 
482 /*
483      These are used internally in the PETSc routines to keep a count of MPI messages and
484    their sizes.
485 
486      This does not work for MPI-Uni because our include/petsc/mpiuni/mpi.h file
487    uses macros to defined the MPI operations.
488 
489      It does not work correctly from HP-UX because it processes the
490    macros in a way that sometimes it double counts, hence
491    PETSC_HAVE_BROKEN_RECURSIVE_MACRO
492 
493      It does not work with Windows because winmpich lacks MPI_Type_size()
494 */
495 #if !defined(MPIUNI_H) && !defined(PETSC_HAVE_BROKEN_RECURSIVE_MACRO) && !defined (PETSC_HAVE_MPI_MISSING_TYPESIZE)
496 /*
497    Logging of MPI activities
498 */
499 PETSC_STATIC_INLINE PetscErrorCode PetscMPITypeSize(PetscInt count,MPI_Datatype type,PetscLogDouble *length)
500 {
501   PetscMPIInt    typesize;
502   PetscErrorCode ierr;
503   if (type == MPI_DATATYPE_NULL) return 0;
504   ierr     = MPI_Type_size(type,&typesize);CHKERRQ(ierr);
505   *length += (PetscLogDouble) (count*typesize);
506   return 0;
507 }
508 
509 PETSC_STATIC_INLINE PetscErrorCode PetscMPITypeSizeComm(MPI_Comm comm,const PetscMPIInt *counts,MPI_Datatype type,PetscLogDouble *length)
510 {
511   PetscMPIInt    typesize,size,p;
512   PetscErrorCode ierr;
513 
514   if (type == MPI_DATATYPE_NULL) return 0;
515   ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr);
516   ierr = MPI_Type_size(type,&typesize);CHKERRMPI(ierr);
517   for (p=0; p<size; ++p) {
518     *length += (PetscLogDouble) (counts[p]*typesize);
519   }
520   return 0;
521 }
522 
523 PETSC_STATIC_INLINE PetscErrorCode PetscMPITypeSizeCount(PetscInt n,const PetscMPIInt *counts,MPI_Datatype type,PetscLogDouble *length)
524 {
525   PetscMPIInt    typesize,p;
526   PetscErrorCode ierr;
527 
528   if (type == MPI_DATATYPE_NULL) return 0;
529   ierr = MPI_Type_size(type,&typesize);CHKERRMPI(ierr);
530   for (p=0; p<n; ++p) {
531     *length += (PetscLogDouble) (counts[p]*typesize);
532   }
533   return 0;
534 }
535 
536 /*
537     Returns 1 if the communicator is parallel else zero
538 */
539 PETSC_STATIC_INLINE int PetscMPIParallelComm(MPI_Comm comm)
540 {
541   PetscMPIInt size; MPI_Comm_size(comm,&size); return size > 1;
542 }
543 
544 #define MPI_Irecv(buf,count,datatype,source,tag,comm,request) \
545   ((petsc_irecv_ct++,0) || PetscMPITypeSize((count),(datatype),&(petsc_irecv_len)) || MPI_Irecv((buf),(count),(datatype),(source),(tag),(comm),(request)))
546 
547 #define MPI_Isend(buf,count,datatype,dest,tag,comm,request) \
548   ((petsc_isend_ct++,0) || PetscMPITypeSize((count),(datatype),&(petsc_isend_len)) || MPI_Isend((buf),(count),(datatype),(dest),(tag),(comm),(request)))
549 
550 #define MPI_Startall_irecv(count,datatype,number,requests) \
551   ((petsc_irecv_ct += (PetscLogDouble)(number),0) || PetscMPITypeSize((count),(datatype),&(petsc_irecv_len)) || ((number) && MPI_Startall((number),(requests))))
552 
553 #define MPI_Startall_isend(count,datatype,number,requests) \
554   ((petsc_isend_ct += (PetscLogDouble)(number),0) || PetscMPITypeSize((count),(datatype),&(petsc_isend_len)) || ((number) && MPI_Startall((number),(requests))))
555 
556 #define MPI_Start_isend(count,datatype,requests) \
557   ((petsc_isend_ct++,0) || PetscMPITypeSize((count),(datatype),(&petsc_isend_len)) || MPI_Start((requests)))
558 
559 #define MPI_Recv(buf,count,datatype,source,tag,comm,status) \
560   ((petsc_recv_ct++,0) || PetscMPITypeSize((count),(datatype),(&petsc_recv_len)) || MPI_Recv((buf),(count),(datatype),(source),(tag),(comm),(status)))
561 
562 #define MPI_Send(buf,count,datatype,dest,tag,comm) \
563   ((petsc_send_ct++,0) || PetscMPITypeSize((count),(datatype),(&petsc_send_len)) || MPI_Send((buf),(count),(datatype),(dest),(tag),(comm)))
564 
565 #define MPI_Wait(request,status) \
566   ((petsc_wait_ct++,petsc_sum_of_waits_ct++,0) || MPI_Wait((request),(status)))
567 
568 #define MPI_Waitany(a,b,c,d) \
569   ((petsc_wait_any_ct++,petsc_sum_of_waits_ct++,0) || MPI_Waitany((a),(b),(c),(d)))
570 
571 #define MPI_Waitall(count,array_of_requests,array_of_statuses) \
572   ((petsc_wait_all_ct++,petsc_sum_of_waits_ct += (PetscLogDouble) (count),0) || MPI_Waitall((count),(array_of_requests),(array_of_statuses)))
573 
574 #define MPI_Allreduce(sendbuf,recvbuf,count,datatype,op,comm) \
575   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || MPI_Allreduce((sendbuf),(recvbuf),(count),(datatype),(op),(comm)))
576 
577 #define MPI_Bcast(buffer,count,datatype,root,comm) \
578   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || MPI_Bcast((buffer),(count),(datatype),(root),(comm)))
579 
580 #define MPI_Reduce_scatter_block(sendbuf,recvbuf,recvcount,datatype,op,comm) \
581   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || MPI_Reduce_scatter_block((sendbuf),(recvbuf),(recvcount),(datatype),(op),(comm)))
582 
583 #define MPI_Alltoall(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm) \
584   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Alltoall((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm)))
585 
586 #define MPI_Alltoallv(sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
587   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSizeComm((comm),(sendcnts),(sendtype),(&petsc_send_len)) || MPI_Alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm)))
588 
589 #define MPI_Allgather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm) \
590   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Allgather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm)))
591 
592 #define MPI_Allgatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,comm) \
593   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Allgatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(comm)))
594 
595 #define MPI_Gather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm) \
596   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Gather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm)))
597 
598 #define MPI_Gatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,root,comm) \
599   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Gatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(root),(comm)))
600 
601 #define MPI_Scatter(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm) \
602   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Scatter((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm)))
603 
604 #define MPI_Scatterv(sendbuf,sendcount,displs,sendtype,recvbuf,recvcount,recvtype,root,comm) \
605   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Scatterv((sendbuf),(sendcount),(displs),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm)))
606 
607 #define MPI_Ialltoall(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,request) \
608   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Ialltoall((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm),(request)))
609 
610 #define MPI_Ialltoallv(sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
611   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSizeComm((comm),(sendcnts),(sendtype),(&petsc_send_len)) || MPI_Ialltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request)))
612 
613 #define MPI_Iallgather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,request) \
614   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Iallgather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm),(request)))
615 
616 #define MPI_Iallgatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,comm,request) \
617   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Iallgatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(comm),(request)))
618 
619 #define MPI_Igather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm,request) \
620   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Igather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm),(request)))
621 
622 #define MPI_Igatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,root,comm,request) \
623   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Igatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(root),(comm),(request)))
624 
625 #define MPI_Iscatter(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm,request) \
626   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Iscatter((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm),(request)))
627 
628 #define MPI_Iscatterv(sendbuf,sendcount,displs,sendtype,recvbuf,recvcount,recvtype,root,comm,request) \
629   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Iscatterv((sendbuf),(sendcount),(displs),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm),(request)))
630 
631 /* We treat MPI_Ineighbor_alltoallv as a set of isend/irecv instead of a traditional MPI collective.
632    OpenMPI-3.0 ran into error with outdegree = indegree = 0, so we use ((outdegree) || (indegree)) as a workaround.
633  */
634 #define MPI_Start_ineighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
635   ((petsc_isend_ct += (PetscLogDouble)(outdegree),0) || (petsc_irecv_ct += (PetscLogDouble)(indegree),0) || PetscMPITypeSizeCount((outdegree),(sendcnts),(sendtype),(&petsc_isend_len)) || PetscMPITypeSizeCount((indegree),(recvcnts),(recvtype),(&petsc_irecv_len)) || (((outdegree) || (indegree)) && MPI_Ineighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request))))
636 
637 #define MPI_Start_neighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
638   ((petsc_isend_ct += (PetscLogDouble)(outdegree),0) || (petsc_irecv_ct += (PetscLogDouble)(indegree),0) || PetscMPITypeSizeCount((outdegree),(sendcnts),(sendtype),(&petsc_isend_len)) || PetscMPITypeSizeCount((indegree),(recvcnts),(recvtype),(&petsc_irecv_len)) || (((outdegree) || (indegree)) && MPI_Neighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm))))
639 
640 #else
641 
642 #define MPI_Startall_irecv(count,datatype,number,requests) \
643   ((number) && MPI_Startall((number),(requests)))
644 
645 #define MPI_Startall_isend(count,datatype,number,requests) \
646   ((number) && MPI_Startall((number),(requests)))
647 
648 #define MPI_Start_isend(count,datatype,requests) \
649   (MPI_Start((requests)))
650 
651 #define MPI_Start_ineighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
652   (((outdegree) || (indegree)) && MPI_Ineighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request)))
653 
654 #define MPI_Start_neighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
655   (((outdegree) || (indegree)) && MPI_Neighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm)))
656 #endif /* !MPIUNI_H && ! PETSC_HAVE_BROKEN_RECURSIVE_MACRO */
657 
658 #else  /* ---Logging is turned off --------------------------------------------*/
659 
660 #define PetscLogMemory                     PETSC_FALSE
661 
662 #define PetscLogFlops(n)                   0
663 #define PetscGetFlops(a)                   (*(a) = 0.0,0)
664 
665 #define PetscLogStageRegister(a,b)         0
666 #define PetscLogStagePush(a)               0
667 #define PetscLogStagePop()                 0
668 #define PetscLogStageSetActive(a,b)        0
669 #define PetscLogStageGetActive(a,b)        0
670 #define PetscLogStageGetVisible(a,b)       0
671 #define PetscLogStageSetVisible(a,b)       0
672 #define PetscLogStageGetId(a,b)            (*(b)=0,0)
673 
674 #define PetscLogEventRegister(a,b,c)       0
675 #define PetscLogEventSetCollective(a,b)    0
676 #define PetscLogEventIncludeClass(a)       0
677 #define PetscLogEventExcludeClass(a)       0
678 #define PetscLogEventActivate(a)           0
679 #define PetscLogEventDeactivate(a)         0
680 #define PetscLogEventDeactivatePush(a)     0
681 #define PetscLogEventDeactivatePop(a)      0
682 #define PetscLogEventActivateClass(a)      0
683 #define PetscLogEventDeactivateClass(a)    0
684 #define PetscLogEventSetActiveAll(a,b)     0
685 #define PetscLogEventGetId(a,b)            (*(b)=0,0)
686 #define PetscLogEventGetPerfInfo(a,b,c)    0
687 #define PetscLogEventSetDof(a,b,c)         0
688 #define PetscLogEventSetError(a,b,c)       0
689 
690 #define PetscLogPLB                        0
691 #define PetscLogPLE                        0
692 #define PetscLogPHC                        0
693 #define PetscLogPHD                        0
694 
695 #define PetscLogObjectParents(p,n,c)       0
696 #define PetscLogObjectCreate(h)            0
697 #define PetscLogObjectDestroy(h)           0
698 PETSC_EXTERN PetscErrorCode PetscLogObjectState(PetscObject,const char[],...);
699 
700 #define PetscLogDefaultBegin()             0
701 #define PetscLogAllBegin()                 0
702 #define PetscLogNestedBegin()              0
703 #define PetscLogTraceBegin(file)           0
704 #define PetscLogActions(a)                 0
705 #define PetscLogObjects(a)                 0
706 #define PetscLogSetThreshold(a,b)          0
707 #define PetscLogSet(lb,le)                 0
708 
709 #define PetscLogView(viewer)               0
710 #define PetscLogViewFromOptions()          0
711 #define PetscLogDump(c)                    0
712 
713 #define PetscLogEventSync(e,comm)          0
714 #define PetscLogEventBegin(e,o1,o2,o3,o4)  0
715 #define PetscLogEventEnd(e,o1,o2,o3,o4)    0
716 
717 #if defined(PETSC_HAVE_DEVICE)
718 #define PetscLogCpuToGpu(a)                0
719 #define PetscLogGpuToCpu(a)                0
720 #define PetscLogGpuFlops(a)                0
721 #define PetscLogGpuTimeBegin()             0
722 #define PetscLogGpuTimeEnd()               0
723 #define PetscLogGpuTimeAdd(a)              0
724 #endif
725 
726 /* If PETSC_USE_LOG is NOT defined, these still need to be! */
727 #define MPI_Startall_irecv(count,datatype,number,requests) ((number) && MPI_Startall(number,requests))
728 #define MPI_Startall_isend(count,datatype,number,requests) ((number) && MPI_Startall(number,requests))
729 #define MPI_Start_isend(count,datatype,requests)           MPI_Start(requests)
730 #define MPI_Start_ineighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
731   (((outdegree) || (indegree)) && MPI_Ineighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request)))
732 #define MPI_Start_neighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
733   (((outdegree) || (indegree)) && MPI_Neighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm)))
734 
735 #endif   /* PETSC_USE_LOG */
736 
737 #define PetscPreLoadBegin(flag,name) \
738 do {\
739   PetscBool      PetscPreLoading = flag;\
740   int            PetscPreLoadMax,PetscPreLoadIt;\
741   PetscLogStage  _stageNum;\
742   PetscErrorCode _3_ierr; \
743   _3_ierr = PetscOptionsGetBool(NULL,NULL,"-preload",&PetscPreLoading,NULL);CHKERRQ(_3_ierr); \
744   PetscPreLoadMax = (int)(PetscPreLoading);\
745   PetscPreLoadingUsed = PetscPreLoading ? PETSC_TRUE : PetscPreLoadingUsed;\
746   for (PetscPreLoadIt=0; PetscPreLoadIt<=PetscPreLoadMax; PetscPreLoadIt++) {\
747     PetscPreLoadingOn = PetscPreLoading;\
748     _3_ierr = PetscBarrier(NULL);CHKERRQ(_3_ierr);\
749     if (PetscPreLoadIt>0) {\
750       _3_ierr = PetscLogStageGetId(name,&_stageNum);CHKERRQ(_3_ierr);\
751     } else {\
752       _3_ierr = PetscLogStageRegister(name,&_stageNum);CHKERRQ(_3_ierr); \
753     }\
754     _3_ierr = PetscLogStageSetActive(_stageNum,(PetscBool)(!PetscPreLoadMax || PetscPreLoadIt));\
755     _3_ierr = PetscLogStagePush(_stageNum);CHKERRQ(_3_ierr);
756 
757 #define PetscPreLoadEnd() \
758     _3_ierr = PetscLogStagePop();CHKERRQ(_3_ierr);\
759     PetscPreLoading = PETSC_FALSE;\
760   }\
761 } while (0)
762 
763 #define PetscPreLoadStage(name) do {                                         \
764     _3_ierr = PetscLogStagePop();CHKERRQ(_3_ierr);                      \
765     if (PetscPreLoadIt>0) {                                                  \
766       _3_ierr = PetscLogStageGetId(name,&_stageNum);CHKERRQ(_3_ierr);   \
767     } else {                                                            \
768       _3_ierr = PetscLogStageRegister(name,&_stageNum);CHKERRQ(_3_ierr); \
769     }                                                                   \
770     _3_ierr = PetscLogStageSetActive(_stageNum,(PetscBool)(!PetscPreLoadMax || PetscPreLoadIt)); \
771     _3_ierr = PetscLogStagePush(_stageNum);CHKERRQ(_3_ierr);            \
772   } while (0)
773 
774 /* some vars for logging */
775 PETSC_EXTERN PetscBool PetscPreLoadingUsed;       /* true if we are or have done preloading */
776 PETSC_EXTERN PetscBool PetscPreLoadingOn;         /* true if we are currently in a preloading calculation */
777 
778 #endif
779