xref: /petsc/include/petsclog.h (revision 0db4d2e0165a0ea245cec0549c2de0bb7b39e2c0)
1 /*
2     Defines profile/logging in PETSc.
3 */
4 
5 #if !defined(PETSCLOG_H)
6 #define PETSCLOG_H
7 #include <petscsys.h>
8 #include <petsctime.h>
9 
10 /* General logging of information; different from event logging */
11 PETSC_EXTERN PetscErrorCode PetscInfo_Private(const char[],PetscObject,const char[],...);
12 #if defined(PETSC_USE_INFO)
13 #define PetscInfo(A,S)                       PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S)
14 #define PetscInfo1(A,S,a1)                   PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1)
15 #define PetscInfo2(A,S,a1,a2)                PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2)
16 #define PetscInfo3(A,S,a1,a2,a3)             PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3)
17 #define PetscInfo4(A,S,a1,a2,a3,a4)          PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4)
18 #define PetscInfo5(A,S,a1,a2,a3,a4,a5)       PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5)
19 #define PetscInfo6(A,S,a1,a2,a3,a4,a5,a6)    PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5,a6)
20 #define PetscInfo7(A,S,a1,a2,a3,a4,a5,a6,a7) PetscInfo_Private(PETSC_FUNCTION_NAME,((PetscObject)A),S,a1,a2,a3,a4,a5,a6,a7)
21 #else
22 #define PetscInfo(A,S)                       0
23 #define PetscInfo1(A,S,a1)                   0
24 #define PetscInfo2(A,S,a1,a2)                0
25 #define PetscInfo3(A,S,a1,a2,a3)             0
26 #define PetscInfo4(A,S,a1,a2,a3,a4)          0
27 #define PetscInfo5(A,S,a1,a2,a3,a4,a5)       0
28 #define PetscInfo6(A,S,a1,a2,a3,a4,a5,a6)    0
29 #define PetscInfo7(A,S,a1,a2,a3,a4,a5,a6,a7) 0
30 #endif
31 
32 /*E
33     PetscInfoCommFlag - Describes the method by which to filter PetscInfo() by communicator size
34 
35     Used as an input for PetscInfoSetFilterCommSelf()
36 
37 $   PETSC_INFO_COMM_ALL - Default uninitialized value. PetscInfo() will not filter based on communicator size (i.e. will
38 print for all communicators)
39 $   PETSC_INFO_COMM_NO_SELF - PetscInfo() will NOT print for communicators with size = 1 (i.e. *_COMM_SELF)
40 $   PETSC_INFO_COMM_ONLY_SELF - PetscInfo will ONLY print for communicators with size = 1
41 
42     Level: intermediate
43 
44 .seealso: PetscInfo(), PetscInfoSetFromOptions(), PetscInfoSetFilterCommSelf()
45 E*/
46 typedef enum {
47   PETSC_INFO_COMM_ALL = -1,
48   PETSC_INFO_COMM_NO_SELF = 0,
49   PETSC_INFO_COMM_ONLY_SELF = 1
50 } PetscInfoCommFlag;
51 
52 PETSC_EXTERN const char * const PetscInfoCommFlags[];
53 PETSC_EXTERN PetscErrorCode PetscInfoDeactivateClass(PetscClassId);
54 PETSC_EXTERN PetscErrorCode PetscInfoActivateClass(PetscClassId);
55 PETSC_EXTERN PetscErrorCode PetscInfoEnabled(PetscClassId, PetscBool *);
56 PETSC_EXTERN PetscErrorCode PetscInfoAllow(PetscBool);
57 PETSC_EXTERN PetscErrorCode PetscInfoSetFile(const char[],const char[]);
58 PETSC_EXTERN PetscErrorCode PetscInfoGetFile(char **,FILE **);
59 PETSC_EXTERN PetscErrorCode PetscInfoSetClasses(PetscBool,PetscInt,const char *const *);
60 PETSC_EXTERN PetscErrorCode PetscInfoGetClass(const char *, PetscBool *);
61 PETSC_EXTERN PetscErrorCode PetscInfoGetInfo(PetscBool *,PetscBool *,PetscBool *,PetscBool *,PetscInfoCommFlag *);
62 PETSC_EXTERN PetscErrorCode PetscInfoProcessClass(const char[],PetscInt,PetscClassId[]);
63 PETSC_EXTERN PetscErrorCode PetscInfoSetFilterCommSelf(PetscInfoCommFlag);
64 PETSC_EXTERN PetscErrorCode PetscInfoSetFromOptions(PetscOptions);
65 PETSC_EXTERN PetscErrorCode PetscInfoDestroy(void);
66 PETSC_EXTERN PetscBool      PetscLogPrintInfo;  /* if true, indicates PetscInfo() is turned on */
67 
68 /*MC
69     PetscLogEvent - id used to identify PETSc or user events which timed portions (blocks of executable)
70      code.
71 
72     Level: intermediate
73 
74 .seealso: PetscLogEventRegister(), PetscLogEventBegin(), PetscLogEventEnd(), PetscLogStage
75 M*/
76 typedef int PetscLogEvent;
77 
78 /*MC
79     PetscLogStage - id used to identify user stages (phases, sections) of runs - for logging
80 
81     Level: intermediate
82 
83 .seealso: PetscLogStageRegister(), PetscLogStagePush(), PetscLogStagePop(), PetscLogEvent
84 M*/
85 typedef int PetscLogStage;
86 
87 #define PETSC_EVENT  1311311
88 PETSC_EXTERN PetscLogEvent PETSC_LARGEST_EVENT;
89 
90 /* Global flop counter */
91 PETSC_EXTERN PetscLogDouble petsc_TotalFlops;
92 PETSC_EXTERN PetscLogDouble petsc_tmp_flops;
93 
94 /* We must make the following structures available to access the event
95      activation flags in the PetscLogEventBegin/End() macros. These are not part of the PETSc public
96      API and are not intended to be used by other parts of PETSc or by users.
97 
98      The code that manipulates these structures is in src/sys/logging/utils.
99 */
100 typedef struct _n_PetscIntStack *PetscIntStack;
101 
102 /* -----------------------------------------------------------------------------------------------------*/
103 /*
104     PetscClassRegInfo, PetscClassPerfInfo - Each class has two data structures associated with it. The first has
105        static information about it, the second collects statistics on how many objects of the class are created,
106        how much memory they use, etc.
107 
108     PetscClassRegLog, PetscClassPerfLog - arrays of the PetscClassRegInfo and PetscClassPerfInfo for all classes.
109 */
110 typedef struct  {
111   char           *name;   /* The class name */
112   PetscClassId   classid; /* The integer identifying this class */
113 } PetscClassRegInfo;
114 
115 typedef struct {
116   PetscClassId   id;           /* The integer identifying this class */
117   int            creations;    /* The number of objects of this class created */
118   int            destructions; /* The number of objects of this class destroyed */
119   PetscLogDouble mem;          /* The total memory allocated by objects of this class */
120   PetscLogDouble descMem;      /* The total memory allocated by descendents of these objects */
121 } PetscClassPerfInfo;
122 
123 typedef struct _n_PetscClassRegLog *PetscClassRegLog;
124 struct _n_PetscClassRegLog {
125   int               numClasses; /* The number of classes registered */
126   int               maxClasses; /* The maximum number of classes */
127   PetscClassRegInfo *classInfo; /* The structure for class information (classids are monotonicly increasing) */
128 };
129 
130 typedef struct _n_PetscClassPerfLog *PetscClassPerfLog;
131 struct _n_PetscClassPerfLog {
132   int                numClasses; /* The number of logging classes */
133   int                maxClasses; /* The maximum number of classes */
134   PetscClassPerfInfo *classInfo; /* The structure for class information (classids are monotonicly increasing) */
135 };
136 /* -----------------------------------------------------------------------------------------------------*/
137 /*
138     PetscEventRegInfo, PetscEventPerfInfo - Each event has two data structures associated with it. The first has
139        static information about it, the second collects statistics on how many times the event is used, how
140        much time it takes, etc.
141 
142     PetscEventRegLog, PetscEventPerfLog - an array of all PetscEventRegInfo and PetscEventPerfInfo for all events. There is one
143       of these for each stage.
144 
145 */
146 typedef struct {
147   char         *name;         /* The name of this event */
148   PetscClassId classid;       /* The class the event is associated with */
149   PetscBool    collective;    /* Flag this event as collective */
150 #if defined (PETSC_HAVE_MPE)
151   int          mpe_id_begin;  /* MPE IDs that define the event */
152   int          mpe_id_end;
153 #endif
154 } PetscEventRegInfo;
155 
156 typedef struct {
157   int            id;            /* The integer identifying this event */
158   PetscBool      active;        /* The flag to activate logging */
159   PetscBool      visible;       /* The flag to print info in summary */
160   int            depth;         /* The nesting depth of the event call */
161   int            count;         /* The number of times this event was executed */
162   PetscLogDouble flops, flops2, flopsTmp; /* The flops and flops^2 used in this event */
163   PetscLogDouble time, time2, timeTmp;    /* The time and time^2 taken for this event */
164   PetscLogDouble syncTime;                /* The synchronization barrier time */
165   PetscLogDouble dof[8];        /* The number of degrees of freedom associated with this event */
166   PetscLogDouble errors[8];     /* The errors (user-defined) associated with this event */
167   PetscLogDouble numMessages;   /* The number of messages in this event */
168   PetscLogDouble messageLength; /* The total message lengths in this event */
169   PetscLogDouble numReductions; /* The number of reductions in this event */
170   PetscLogDouble memIncrease;   /* How much the resident memory has increased in this event */
171   PetscLogDouble mallocIncrease;/* How much the maximum malloced space has increased in this event */
172   PetscLogDouble mallocSpace;   /* How much the space was malloced and kept during this event */
173   PetscLogDouble mallocIncreaseEvent;  /* Maximum of the high water mark with in event minus memory available at the end of the event */
174   #if defined(PETSC_HAVE_DEVICE)
175   PetscLogDouble CpuToGpuCount; /* The total number of CPU to GPU copies */
176   PetscLogDouble GpuToCpuCount; /* The total number of GPU to CPU copies */
177   PetscLogDouble CpuToGpuSize;  /* The total size of CPU to GPU copies */
178   PetscLogDouble GpuToCpuSize;  /* The total size of GPU to CPU copies */
179   PetscLogDouble GpuFlops;      /* The flops done on a GPU in this event */
180   PetscLogDouble GpuTime;       /* The time spent on a GPU in this event */
181   #endif
182 } PetscEventPerfInfo;
183 
184 typedef struct _n_PetscEventRegLog *PetscEventRegLog;
185 struct _n_PetscEventRegLog {
186   int               numEvents;  /* The number of registered events */
187   int               maxEvents;  /* The maximum number of events */
188   PetscEventRegInfo *eventInfo; /* The registration information for each event */
189 };
190 
191 typedef struct _n_PetscEventPerfLog *PetscEventPerfLog;
192 struct _n_PetscEventPerfLog {
193   int                numEvents;  /* The number of logging events */
194   int                maxEvents;  /* The maximum number of events */
195   PetscEventPerfInfo *eventInfo; /* The performance information for each event */
196 };
197 /* ------------------------------------------------------------------------------------------------------------*/
198 /*
199    PetscStageInfo - Contains all the information about a particular stage.
200 
201    PetscStageLog - An array of PetscStageInfo for each registered stage. There is a single one of these in the code.
202 */
203 typedef struct _PetscStageInfo {
204   char               *name;     /* The stage name */
205   PetscBool          used;      /* The stage was pushed on this processor */
206   PetscEventPerfInfo perfInfo;  /* The stage performance information */
207   PetscEventPerfLog  eventLog;  /* The event information for this stage */
208   PetscClassPerfLog  classLog;  /* The class information for this stage */
209 } PetscStageInfo;
210 
211 typedef struct _n_PetscStageLog *PetscStageLog;
212 struct _n_PetscStageLog {
213   int              numStages;   /* The number of registered stages */
214   int              maxStages;   /* The maximum number of stages */
215   PetscIntStack    stack;       /* The stack for active stages */
216   int              curStage;    /* The current stage (only used in macros so we don't call PetscIntStackTop) */
217   PetscStageInfo   *stageInfo;  /* The information for each stage */
218   PetscEventRegLog eventLog;    /* The registered events */
219   PetscClassRegLog classLog;    /* The registered classes */
220 };
221 /* -----------------------------------------------------------------------------------------------------*/
222 
223 PETSC_EXTERN PetscErrorCode PetscLogObjectParent(PetscObject,PetscObject);
224 PETSC_EXTERN PetscErrorCode PetscLogObjectMemory(PetscObject,PetscLogDouble);
225 
226 #if defined(PETSC_USE_LOG)  /* --- Logging is turned on --------------------------------*/
227 PETSC_EXTERN PetscStageLog petsc_stageLog;
228 PETSC_EXTERN PetscErrorCode PetscLogGetStageLog(PetscStageLog*);
229 PETSC_EXTERN PetscErrorCode PetscStageLogGetCurrent(PetscStageLog,int*);
230 PETSC_EXTERN PetscErrorCode PetscStageLogGetEventPerfLog(PetscStageLog,int,PetscEventPerfLog*);
231 
232 /*
233    Flop counting:  We count each arithmetic operation (e.g., addition, multiplication) separately.
234 
235    For the complex numbers version, note that
236        1 complex addition = 2 flops
237        1 complex multiplication = 6 flops,
238    where we define 1 flop as that for a double precision scalar.  We roughly approximate
239    flop counting for complex numbers by multiplying the total flops by 4; this corresponds
240    to the assumption that we're counting mostly additions and multiplications -- and
241    roughly the same number of each.  More accurate counting could be done by distinguishing
242    among the various arithmetic operations.
243  */
244 
245 #if defined(PETSC_USE_COMPLEX)
246 #define PETSC_FLOPS_PER_OP 4.0
247 #else
248 #define PETSC_FLOPS_PER_OP 1.0
249 #endif
250 
251 /*@
252        PetscLogFlops - Log how many flops are performed in a calculation
253 
254    Input Paramters:
255     flops - the number of flops
256 
257    Notes:
258      To limit the chance of integer overflow when multiplying by a constant, represent the constant as a double,
259      not an integer. Use PetscLogFlops(4.0*n) not PetscLogFlops(4*n)
260 
261    Level: intermediate
262 
263 .seealso: PetscLogView(), PetscLogGpuFlops()
264 @*/
265 
266 PETSC_STATIC_INLINE PetscErrorCode PetscLogFlops(PetscLogDouble n)
267 {
268   PetscFunctionBegin;
269 #if defined(PETSC_USE_DEBUG)
270   if (n < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Cannot log negative flops");
271 #endif
272   petsc_TotalFlops += PETSC_FLOPS_PER_OP*n;
273   PetscFunctionReturn(0);
274 }
275 
276 #if defined(PETSC_HAVE_DEVICE)
277 /* Global GPU counters */
278 PETSC_EXTERN PetscLogDouble petsc_ctog_ct;
279 PETSC_EXTERN PetscLogDouble petsc_gtoc_ct;
280 PETSC_EXTERN PetscLogDouble petsc_ctog_sz;
281 PETSC_EXTERN PetscLogDouble petsc_gtoc_sz;
282 PETSC_EXTERN PetscLogDouble petsc_gflops;
283 PETSC_EXTERN PetscLogDouble petsc_gtime;
284 #if defined(PETSC_USE_DEBUG)
285 PETSC_EXTERN PetscBool      petsc_gtime_inuse;
286 #endif
287 
288 PETSC_STATIC_INLINE PetscErrorCode PetscLogCpuToGpu(PetscLogDouble size)
289 {
290   PetscFunctionBegin;
291   petsc_ctog_ct += 1;
292   petsc_ctog_sz += size;
293   PetscFunctionReturn(0);
294 }
295 
296 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuToCpu(PetscLogDouble size)
297 {
298   PetscFunctionBegin;
299   petsc_gtoc_ct += 1;
300   petsc_gtoc_sz += size;
301   PetscFunctionReturn(0);
302 }
303 
304 /*@
305        PetscLogGpuFlops - Log how many flops are performed in a calculation on the device
306 
307    Input Paramters:
308     flops - the number of flops
309 
310    Notes:
311      To limit the chance of integer overflow when multiplying by a constant, represent the constant as a double,
312      not an integer. Use PetscLogFlops(4.0*n) not PetscLogFlops(4*n)
313 
314    Level: intermediate
315 
316 .seealso: PetscLogView(), PetscLogFlops(), PetscLogGpuTimeBegin(), PetscLogGpuTimeEnd()
317 @*/
318 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuFlops(PetscLogDouble n)
319 {
320   PetscFunctionBegin;
321 #if defined(PETSC_USE_DEBUG)
322   if (n < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Cannot log negative flops");
323 #endif
324   petsc_TotalFlops += PETSC_FLOPS_PER_OP*n;
325   petsc_gflops += PETSC_FLOPS_PER_OP*n;
326   PetscFunctionReturn(0);
327 }
328 
329 /*@
330        PetscLogGpuTimeBegin - Start timer for device
331 
332    Level: intermediate
333 
334       Notes:
335         The timer is run on the CPU, it is a separate logging of time devoted to GPU computations (including kernel launch times).
336         This timer should NOT include times for data transfers between the GPU and CPU, nor setup actions such as allocating space.
337         The regular logging captures the time for data transfers and any CPU activites during the event
338         It is used to compute the flop rate on the GPU as it is actively engaged in running a kernel.
339 
340 
341 .seealso:  PetscLogView(), PetscLogGpuFlops(), PetscLogGpuTimeEnd()
342 @*/
343 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuTimeBegin()
344 {
345   PetscErrorCode ierr;
346   PetscFunctionBegin;
347 #if defined(PETSC_USE_DEBUG)
348   if (petsc_gtime_inuse) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Forgot to call PetscLogGpuTimeEnd()?");
349   petsc_gtime_inuse = PETSC_TRUE;
350 #endif
351   ierr = PetscTimeSubtract(&petsc_gtime);CHKERRQ(ierr);
352   PetscFunctionReturn(0);
353 }
354 /*@
355        PetscLogGpuTimeEnd - Stop timer for device
356 
357    Level: intermediate
358 
359 .seealso:  PetscLogView(), PetscLogGpuFlops(), PetscLogGpuTimeBegin()
360 @*/
361 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuTimeEnd()
362 {
363   PetscErrorCode ierr;
364   PetscFunctionBegin;
365 #if defined(PETSC_USE_DEBUG)
366   if (!petsc_gtime_inuse) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Forgot to call PetscLogGpuTimeBegin()?");
367   petsc_gtime_inuse = PETSC_FALSE;
368 #endif
369   ierr = PetscTimeAdd(&petsc_gtime);CHKERRQ(ierr);
370   PetscFunctionReturn(0);
371 }
372 
373 PETSC_STATIC_INLINE PetscErrorCode PetscLogGpuTimeAdd(PetscLogDouble t)
374 {
375   PetscFunctionBegin;
376   petsc_gtime += t;
377   PetscFunctionReturn(0);
378 }
379 #endif
380 
381 PETSC_EXTERN PetscErrorCode PetscGetFlops(PetscLogDouble *);
382 
383 #if defined (PETSC_HAVE_MPE)
384 PETSC_EXTERN PetscErrorCode PetscLogMPEBegin(void);
385 PETSC_EXTERN PetscErrorCode PetscLogMPEDump(const char[]);
386 #endif
387 
388 PETSC_EXTERN PetscErrorCode (*PetscLogPLB)(PetscLogEvent,int,PetscObject,PetscObject,PetscObject,PetscObject);
389 PETSC_EXTERN PetscErrorCode (*PetscLogPLE)(PetscLogEvent,int,PetscObject,PetscObject,PetscObject,PetscObject);
390 PETSC_EXTERN PetscErrorCode (*PetscLogPHC)(PetscObject);
391 PETSC_EXTERN PetscErrorCode (*PetscLogPHD)(PetscObject);
392 
393 #define PetscLogObjectParents(p,n,d)  0;do{int _i; for (_i=0; _i<(n); _i++) {ierr = PetscLogObjectParent((PetscObject)(p),(PetscObject)(d)[_i]);CHKERRQ(ierr);}}while (0)
394 #define PetscLogObjectCreate(h)      ((PetscLogPHC) ? (*PetscLogPHC)((PetscObject)(h)) : 0)
395 #define PetscLogObjectDestroy(h)     ((PetscLogPHD) ? (*PetscLogPHD)((PetscObject)(h)) : 0)
396 PETSC_EXTERN PetscErrorCode PetscLogObjectState(PetscObject, const char[], ...);
397 
398 /* Initialization functions */
399 PETSC_EXTERN PetscErrorCode PetscLogDefaultBegin(void);
400 PETSC_EXTERN PetscErrorCode PetscLogAllBegin(void);
401 PETSC_EXTERN PetscErrorCode PetscLogNestedBegin(void);
402 PETSC_EXTERN PetscErrorCode PetscLogTraceBegin(FILE *);
403 PETSC_EXTERN PetscErrorCode PetscLogActions(PetscBool);
404 PETSC_EXTERN PetscErrorCode PetscLogObjects(PetscBool);
405 PETSC_EXTERN PetscErrorCode PetscLogSetThreshold(PetscLogDouble,PetscLogDouble*);
406 PETSC_EXTERN PetscErrorCode PetscLogSet(PetscErrorCode (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject),
407                                         PetscErrorCode (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject));
408 
409 /* Output functions */
410 PETSC_EXTERN PetscErrorCode PetscLogView(PetscViewer);
411 PETSC_EXTERN PetscErrorCode PetscLogViewFromOptions(void);
412 PETSC_EXTERN PetscErrorCode PetscLogDump(const char[]);
413 
414 /* Stage functions */
415 PETSC_EXTERN PetscErrorCode PetscLogStageRegister(const char[],PetscLogStage*);
416 PETSC_EXTERN PetscErrorCode PetscLogStagePush(PetscLogStage);
417 PETSC_EXTERN PetscErrorCode PetscLogStagePop(void);
418 PETSC_EXTERN PetscErrorCode PetscLogStageSetActive(PetscLogStage,PetscBool);
419 PETSC_EXTERN PetscErrorCode PetscLogStageGetActive(PetscLogStage,PetscBool*);
420 PETSC_EXTERN PetscErrorCode PetscLogStageSetVisible(PetscLogStage,PetscBool);
421 PETSC_EXTERN PetscErrorCode PetscLogStageGetVisible(PetscLogStage,PetscBool*);
422 PETSC_EXTERN PetscErrorCode PetscLogStageGetId(const char[],PetscLogStage*);
423 
424 /* Event functions */
425 PETSC_EXTERN PetscErrorCode PetscLogEventRegister(const char[],PetscClassId,PetscLogEvent*);
426 PETSC_EXTERN PetscErrorCode PetscLogEventSetCollective(PetscLogEvent,PetscBool);
427 PETSC_EXTERN PetscErrorCode PetscLogEventIncludeClass(PetscClassId);
428 PETSC_EXTERN PetscErrorCode PetscLogEventExcludeClass(PetscClassId);
429 PETSC_EXTERN PetscErrorCode PetscLogEventActivate(PetscLogEvent);
430 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivate(PetscLogEvent);
431 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivatePush(PetscLogEvent);
432 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivatePop(PetscLogEvent);
433 PETSC_EXTERN PetscErrorCode PetscLogEventSetActiveAll(PetscLogEvent,PetscBool);
434 PETSC_EXTERN PetscErrorCode PetscLogEventActivateClass(PetscClassId);
435 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivateClass(PetscClassId);
436 PETSC_EXTERN PetscErrorCode PetscLogEventGetId(const char[],PetscLogEvent*);
437 PETSC_EXTERN PetscErrorCode PetscLogEventGetPerfInfo(int,PetscLogEvent,PetscEventPerfInfo*);
438 PETSC_EXTERN PetscErrorCode PetscLogEventSetDof(PetscLogEvent, PetscInt, PetscLogDouble);
439 PETSC_EXTERN PetscErrorCode PetscLogEventSetError(PetscLogEvent, PetscInt, PetscLogDouble);
440 
441 /* Global counters */
442 PETSC_EXTERN PetscLogDouble petsc_irecv_ct;
443 PETSC_EXTERN PetscLogDouble petsc_isend_ct;
444 PETSC_EXTERN PetscLogDouble petsc_recv_ct;
445 PETSC_EXTERN PetscLogDouble petsc_send_ct;
446 PETSC_EXTERN PetscLogDouble petsc_irecv_len;
447 PETSC_EXTERN PetscLogDouble petsc_isend_len;
448 PETSC_EXTERN PetscLogDouble petsc_recv_len;
449 PETSC_EXTERN PetscLogDouble petsc_send_len;
450 PETSC_EXTERN PetscLogDouble petsc_allreduce_ct;
451 PETSC_EXTERN PetscLogDouble petsc_gather_ct;
452 PETSC_EXTERN PetscLogDouble petsc_scatter_ct;
453 PETSC_EXTERN PetscLogDouble petsc_wait_ct;
454 PETSC_EXTERN PetscLogDouble petsc_wait_any_ct;
455 PETSC_EXTERN PetscLogDouble petsc_wait_all_ct;
456 PETSC_EXTERN PetscLogDouble petsc_sum_of_waits_ct;
457 
458 PETSC_EXTERN PetscBool      PetscLogMemory;
459 
460 PETSC_EXTERN PetscBool PetscLogSyncOn;  /* true if logging synchronization is enabled */
461 PETSC_EXTERN PetscErrorCode PetscLogEventSynchronize(PetscLogEvent, MPI_Comm);
462 
463 #define PetscLogEventSync(e,comm) \
464   (((PetscLogPLB && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? \
465     PetscLogEventSynchronize((e),(comm)) : 0))
466 
467 #define PetscLogEventBegin(e,o1,o2,o3,o4) \
468   (((PetscLogPLB && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? \
469     (*PetscLogPLB)((e),0,(PetscObject)(o1),(PetscObject)(o2),(PetscObject)(o3),(PetscObject)(o4)) : 0))
470 
471 #define PetscLogEventEnd(e,o1,o2,o3,o4) \
472   (((PetscLogPLE && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? \
473     (*PetscLogPLE)((e),0,(PetscObject)(o1),(PetscObject)(o2),(PetscObject)(o3),(PetscObject)(o4)) : 0))
474 
475 PETSC_EXTERN PetscErrorCode PetscLogEventGetFlops(PetscLogEvent,PetscLogDouble*);
476 PETSC_EXTERN PetscErrorCode PetscLogEventZeroFlops(PetscLogEvent);
477 
478 /*
479      These are used internally in the PETSc routines to keep a count of MPI messages and
480    their sizes.
481 
482      This does not work for MPI-Uni because our include/petsc/mpiuni/mpi.h file
483    uses macros to defined the MPI operations.
484 
485      It does not work correctly from HP-UX because it processes the
486    macros in a way that sometimes it double counts, hence
487    PETSC_HAVE_BROKEN_RECURSIVE_MACRO
488 
489      It does not work with Windows because winmpich lacks MPI_Type_size()
490 */
491 #if !defined(MPIUNI_H) && !defined(PETSC_HAVE_BROKEN_RECURSIVE_MACRO) && !defined (PETSC_HAVE_MPI_MISSING_TYPESIZE)
492 /*
493    Logging of MPI activities
494 */
495 PETSC_STATIC_INLINE PetscErrorCode PetscMPITypeSize(PetscInt count,MPI_Datatype type,PetscLogDouble *length)
496 {
497   PetscMPIInt    typesize;
498   PetscErrorCode ierr;
499   if (type == MPI_DATATYPE_NULL) return 0;
500   ierr     = MPI_Type_size(type,&typesize);CHKERRQ(ierr);
501   *length += (PetscLogDouble) (count*typesize);
502   return 0;
503 }
504 
505 PETSC_STATIC_INLINE PetscErrorCode PetscMPITypeSizeComm(MPI_Comm comm,const PetscMPIInt *counts,MPI_Datatype type,PetscLogDouble *length)
506 {
507   PetscMPIInt    typesize,size,p;
508   PetscErrorCode ierr;
509 
510   if (type == MPI_DATATYPE_NULL) return 0;
511   ierr = MPI_Comm_size(comm,&size);CHKERRMPI(ierr);
512   ierr = MPI_Type_size(type,&typesize);CHKERRMPI(ierr);
513   for (p=0; p<size; ++p) {
514     *length += (PetscLogDouble) (counts[p]*typesize);
515   }
516   return 0;
517 }
518 
519 PETSC_STATIC_INLINE PetscErrorCode PetscMPITypeSizeCount(PetscInt n,const PetscMPIInt *counts,MPI_Datatype type,PetscLogDouble *length)
520 {
521   PetscMPIInt    typesize,p;
522   PetscErrorCode ierr;
523 
524   if (type == MPI_DATATYPE_NULL) return 0;
525   ierr = MPI_Type_size(type,&typesize);CHKERRMPI(ierr);
526   for (p=0; p<n; ++p) {
527     *length += (PetscLogDouble) (counts[p]*typesize);
528   }
529   return 0;
530 }
531 
532 /*
533     Returns 1 if the communicator is parallel else zero
534 */
535 PETSC_STATIC_INLINE int PetscMPIParallelComm(MPI_Comm comm)
536 {
537   PetscMPIInt size; MPI_Comm_size(comm,&size); return size > 1;
538 }
539 
540 #define MPI_Irecv(buf,count,datatype,source,tag,comm,request) \
541   ((petsc_irecv_ct++,0) || PetscMPITypeSize((count),(datatype),&(petsc_irecv_len)) || MPI_Irecv((buf),(count),(datatype),(source),(tag),(comm),(request)))
542 
543 #define MPI_Isend(buf,count,datatype,dest,tag,comm,request) \
544   ((petsc_isend_ct++,0) || PetscMPITypeSize((count),(datatype),&(petsc_isend_len)) || MPI_Isend((buf),(count),(datatype),(dest),(tag),(comm),(request)))
545 
546 #define MPI_Startall_irecv(count,datatype,number,requests) \
547   ((petsc_irecv_ct += (PetscLogDouble)(number),0) || PetscMPITypeSize((count),(datatype),&(petsc_irecv_len)) || ((number) && MPI_Startall((number),(requests))))
548 
549 #define MPI_Startall_isend(count,datatype,number,requests) \
550   ((petsc_isend_ct += (PetscLogDouble)(number),0) || PetscMPITypeSize((count),(datatype),&(petsc_isend_len)) || ((number) && MPI_Startall((number),(requests))))
551 
552 #define MPI_Start_isend(count,datatype,requests) \
553   ((petsc_isend_ct++,0) || PetscMPITypeSize((count),(datatype),(&petsc_isend_len)) || MPI_Start((requests)))
554 
555 #define MPI_Recv(buf,count,datatype,source,tag,comm,status) \
556   ((petsc_recv_ct++,0) || PetscMPITypeSize((count),(datatype),(&petsc_recv_len)) || MPI_Recv((buf),(count),(datatype),(source),(tag),(comm),(status)))
557 
558 #define MPI_Send(buf,count,datatype,dest,tag,comm) \
559   ((petsc_send_ct++,0) || PetscMPITypeSize((count),(datatype),(&petsc_send_len)) || MPI_Send((buf),(count),(datatype),(dest),(tag),(comm)))
560 
561 #define MPI_Wait(request,status) \
562   ((petsc_wait_ct++,petsc_sum_of_waits_ct++,0) || MPI_Wait((request),(status)))
563 
564 #define MPI_Waitany(a,b,c,d) \
565   ((petsc_wait_any_ct++,petsc_sum_of_waits_ct++,0) || MPI_Waitany((a),(b),(c),(d)))
566 
567 #define MPI_Waitall(count,array_of_requests,array_of_statuses) \
568   ((petsc_wait_all_ct++,petsc_sum_of_waits_ct += (PetscLogDouble) (count),0) || MPI_Waitall((count),(array_of_requests),(array_of_statuses)))
569 
570 #define MPI_Allreduce(sendbuf,recvbuf,count,datatype,op,comm) \
571   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || MPI_Allreduce((sendbuf),(recvbuf),(count),(datatype),(op),(comm)))
572 
573 #define MPI_Bcast(buffer,count,datatype,root,comm) \
574   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || MPI_Bcast((buffer),(count),(datatype),(root),(comm)))
575 
576 #define MPI_Reduce_scatter_block(sendbuf,recvbuf,recvcount,datatype,op,comm) \
577   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || MPI_Reduce_scatter_block((sendbuf),(recvbuf),(recvcount),(datatype),(op),(comm)))
578 
579 #define MPI_Alltoall(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm) \
580   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Alltoall((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm)))
581 
582 #define MPI_Alltoallv(sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
583   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSizeComm((comm),(sendcnts),(sendtype),(&petsc_send_len)) || MPI_Alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm)))
584 
585 #define MPI_Allgather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm) \
586   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Allgather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm)))
587 
588 #define MPI_Allgatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,comm) \
589   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Allgatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(comm)))
590 
591 #define MPI_Gather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm) \
592   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Gather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm)))
593 
594 #define MPI_Gatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,root,comm) \
595   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Gatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(root),(comm)))
596 
597 #define MPI_Scatter(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm) \
598   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Scatter((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm)))
599 
600 #define MPI_Scatterv(sendbuf,sendcount,displs,sendtype,recvbuf,recvcount,recvtype,root,comm) \
601   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Scatterv((sendbuf),(sendcount),(displs),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm)))
602 
603 #define MPI_Ialltoall(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,request) \
604   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Ialltoall((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm),(request)))
605 
606 #define MPI_Ialltoallv(sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
607   ((petsc_allreduce_ct += PetscMPIParallelComm((comm)),0) || PetscMPITypeSizeComm((comm),(sendcnts),(sendtype),(&petsc_send_len)) || MPI_Ialltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request)))
608 
609 #define MPI_Iallgather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,comm,request) \
610   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Iallgather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(comm),(request)))
611 
612 #define MPI_Iallgatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,comm,request) \
613   ((petsc_gather_ct += PetscMPIParallelComm((comm)),0) || MPI_Iallgatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(comm),(request)))
614 
615 #define MPI_Igather(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm,request) \
616   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Igather((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm),(request)))
617 
618 #define MPI_Igatherv(sendbuf,sendcount,sendtype,recvbuf,recvcount,displs,recvtype,root,comm,request) \
619   ((petsc_gather_ct++,0) || PetscMPITypeSize((sendcount),(sendtype),(&petsc_send_len)) || MPI_Igatherv((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(displs),(recvtype),(root),(comm),(request)))
620 
621 #define MPI_Iscatter(sendbuf,sendcount,sendtype,recvbuf,recvcount,recvtype,root,comm,request) \
622   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Iscatter((sendbuf),(sendcount),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm),(request)))
623 
624 #define MPI_Iscatterv(sendbuf,sendcount,displs,sendtype,recvbuf,recvcount,recvtype,root,comm,request) \
625   ((petsc_scatter_ct++,0) || PetscMPITypeSize((recvcount),(recvtype),(&petsc_recv_len)) || MPI_Iscatterv((sendbuf),(sendcount),(displs),(sendtype),(recvbuf),(recvcount),(recvtype),(root),(comm),(request)))
626 
627 /* We treat MPI_Ineighbor_alltoallv as a set of isend/irecv instead of a traditional MPI collective.
628    OpenMPI-3.0 ran into error with outdegree = indegree = 0, so we use ((outdegree) || (indegree)) as a workaround.
629  */
630 #define MPI_Start_ineighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
631   ((petsc_isend_ct += (PetscLogDouble)(outdegree),0) || (petsc_irecv_ct += (PetscLogDouble)(indegree),0) || PetscMPITypeSizeCount((outdegree),(sendcnts),(sendtype),(&petsc_isend_len)) || PetscMPITypeSizeCount((indegree),(recvcnts),(recvtype),(&petsc_irecv_len)) || (((outdegree) || (indegree)) && MPI_Ineighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request))))
632 
633 #define MPI_Start_neighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
634   ((petsc_isend_ct += (PetscLogDouble)(outdegree),0) || (petsc_irecv_ct += (PetscLogDouble)(indegree),0) || PetscMPITypeSizeCount((outdegree),(sendcnts),(sendtype),(&petsc_isend_len)) || PetscMPITypeSizeCount((indegree),(recvcnts),(recvtype),(&petsc_irecv_len)) || (((outdegree) || (indegree)) && MPI_Neighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm))))
635 
636 #else
637 
638 #define MPI_Startall_irecv(count,datatype,number,requests) \
639   ((number) && MPI_Startall((number),(requests)))
640 
641 #define MPI_Startall_isend(count,datatype,number,requests) \
642   ((number) && MPI_Startall((number),(requests)))
643 
644 #define MPI_Start_isend(count,datatype,requests) \
645   (MPI_Start((requests)))
646 
647 #define MPI_Start_ineighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
648   (((outdegree) || (indegree)) && MPI_Ineighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request)))
649 
650 #define MPI_Start_neighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
651   (((outdegree) || (indegree)) && MPI_Neighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm)))
652 #endif /* !MPIUNI_H && ! PETSC_HAVE_BROKEN_RECURSIVE_MACRO */
653 
654 #else  /* ---Logging is turned off --------------------------------------------*/
655 
656 #define PetscLogMemory                     PETSC_FALSE
657 
658 #define PetscLogFlops(n)                   0
659 #define PetscGetFlops(a)                   (*(a) = 0.0,0)
660 
661 #define PetscLogStageRegister(a,b)         0
662 #define PetscLogStagePush(a)               0
663 #define PetscLogStagePop()                 0
664 #define PetscLogStageSetActive(a,b)        0
665 #define PetscLogStageGetActive(a,b)        0
666 #define PetscLogStageGetVisible(a,b)       0
667 #define PetscLogStageSetVisible(a,b)       0
668 #define PetscLogStageGetId(a,b)            (*(b)=0,0)
669 
670 #define PetscLogEventRegister(a,b,c)       0
671 #define PetscLogEventSetCollective(a,b)    0
672 #define PetscLogEventIncludeClass(a)       0
673 #define PetscLogEventExcludeClass(a)       0
674 #define PetscLogEventActivate(a)           0
675 #define PetscLogEventDeactivate(a)         0
676 #define PetscLogEventDeactivatePush(a)     0
677 #define PetscLogEventDeactivatePop(a)      0
678 #define PetscLogEventActivateClass(a)      0
679 #define PetscLogEventDeactivateClass(a)    0
680 #define PetscLogEventSetActiveAll(a,b)     0
681 #define PetscLogEventGetId(a,b)            (*(b)=0,0)
682 #define PetscLogEventGetPerfInfo(a,b,c)    0
683 #define PetscLogEventSetDof(a,b,c)         0
684 #define PetscLogEventSetError(a,b,c)       0
685 
686 #define PetscLogPLB                        0
687 #define PetscLogPLE                        0
688 #define PetscLogPHC                        0
689 #define PetscLogPHD                        0
690 
691 #define PetscLogObjectParents(p,n,c)       0
692 #define PetscLogObjectCreate(h)            0
693 #define PetscLogObjectDestroy(h)           0
694 PETSC_EXTERN PetscErrorCode PetscLogObjectState(PetscObject,const char[],...);
695 
696 #define PetscLogDefaultBegin()             0
697 #define PetscLogAllBegin()                 0
698 #define PetscLogNestedBegin()              0
699 #define PetscLogTraceBegin(file)           0
700 #define PetscLogActions(a)                 0
701 #define PetscLogObjects(a)                 0
702 #define PetscLogSetThreshold(a,b)          0
703 #define PetscLogSet(lb,le)                 0
704 
705 #define PetscLogView(viewer)               0
706 #define PetscLogViewFromOptions()          0
707 #define PetscLogDump(c)                    0
708 
709 #define PetscLogEventSync(e,comm)          0
710 #define PetscLogEventBegin(e,o1,o2,o3,o4)  0
711 #define PetscLogEventEnd(e,o1,o2,o3,o4)    0
712 
713 #if defined(PETSC_HAVE_DEVICE)
714 #define PetscLogCpuToGpu(a)                0
715 #define PetscLogGpuToCpu(a)                0
716 #define PetscLogGpuFlops(a)                0
717 #define PetscLogGpuTimeBegin()             0
718 #define PetscLogGpuTimeEnd()               0
719 #define PetscLogGpuTimeAdd(a)              0
720 #endif
721 
722 /* If PETSC_USE_LOG is NOT defined, these still need to be! */
723 #define MPI_Startall_irecv(count,datatype,number,requests) ((number) && MPI_Startall(number,requests))
724 #define MPI_Startall_isend(count,datatype,number,requests) ((number) && MPI_Startall(number,requests))
725 #define MPI_Start_isend(count,datatype,requests)           MPI_Start(requests)
726 #define MPI_Start_ineighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm,request) \
727   (((outdegree) || (indegree)) && MPI_Ineighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm),(request)))
728 #define MPI_Start_neighbor_alltoallv(outdegree,indegree,sendbuf,sendcnts,sdispls,sendtype,recvbuf,recvcnts,rdispls,recvtype,comm) \
729   (((outdegree) || (indegree)) && MPI_Neighbor_alltoallv((sendbuf),(sendcnts),(sdispls),(sendtype),(recvbuf),(recvcnts),(rdispls),(recvtype),(comm)))
730 
731 #endif   /* PETSC_USE_LOG */
732 
733 #define PetscPreLoadBegin(flag,name) \
734 do {\
735   PetscBool      PetscPreLoading = flag;\
736   int            PetscPreLoadMax,PetscPreLoadIt;\
737   PetscLogStage  _stageNum;\
738   PetscErrorCode _3_ierr; \
739   _3_ierr = PetscOptionsGetBool(NULL,NULL,"-preload",&PetscPreLoading,NULL);CHKERRQ(_3_ierr); \
740   PetscPreLoadMax = (int)(PetscPreLoading);\
741   PetscPreLoadingUsed = PetscPreLoading ? PETSC_TRUE : PetscPreLoadingUsed;\
742   for (PetscPreLoadIt=0; PetscPreLoadIt<=PetscPreLoadMax; PetscPreLoadIt++) {\
743     PetscPreLoadingOn = PetscPreLoading;\
744     _3_ierr = PetscBarrier(NULL);CHKERRQ(_3_ierr);\
745     if (PetscPreLoadIt>0) {\
746       _3_ierr = PetscLogStageGetId(name,&_stageNum);CHKERRQ(_3_ierr);\
747     } else {\
748       _3_ierr = PetscLogStageRegister(name,&_stageNum);CHKERRQ(_3_ierr); \
749     }\
750     _3_ierr = PetscLogStageSetActive(_stageNum,(PetscBool)(!PetscPreLoadMax || PetscPreLoadIt));\
751     _3_ierr = PetscLogStagePush(_stageNum);CHKERRQ(_3_ierr);
752 
753 #define PetscPreLoadEnd() \
754     _3_ierr = PetscLogStagePop();CHKERRQ(_3_ierr);\
755     PetscPreLoading = PETSC_FALSE;\
756   }\
757 } while (0)
758 
759 #define PetscPreLoadStage(name) do {                                         \
760     _3_ierr = PetscLogStagePop();CHKERRQ(_3_ierr);                      \
761     if (PetscPreLoadIt>0) {                                                  \
762       _3_ierr = PetscLogStageGetId(name,&_stageNum);CHKERRQ(_3_ierr);   \
763     } else {                                                            \
764       _3_ierr = PetscLogStageRegister(name,&_stageNum);CHKERRQ(_3_ierr); \
765     }                                                                   \
766     _3_ierr = PetscLogStageSetActive(_stageNum,(PetscBool)(!PetscPreLoadMax || PetscPreLoadIt)); \
767     _3_ierr = PetscLogStagePush(_stageNum);CHKERRQ(_3_ierr);            \
768   } while (0)
769 
770 /* some vars for logging */
771 PETSC_EXTERN PetscBool PetscPreLoadingUsed;       /* true if we are or have done preloading */
772 PETSC_EXTERN PetscBool PetscPreLoadingOn;         /* true if we are currently in a preloading calculation */
773 
774 #endif
775