1 /* 2 Defines profile/logging in PETSc. 3 */ 4 #if !defined(PETSCLOG_H) 5 #define PETSCLOG_H 6 7 #include <petscsys.h> 8 #include <petsctime.h> 9 10 /* SUBMANSEC = Sys */ 11 12 /* General logging of information; different from event logging */ 13 PETSC_EXTERN PetscErrorCode PetscInfo_Private(const char[], PetscObject, const char[], ...) PETSC_ATTRIBUTE_FORMAT(3, 4); 14 #if defined(PETSC_USE_INFO) 15 #define PetscInfo(A, ...) PetscInfo_Private(PETSC_FUNCTION_NAME, ((PetscObject)A), __VA_ARGS__) 16 #else 17 #define PetscInfo(A, ...) 0 18 #endif 19 20 #define PetscInfo1(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 21 #define PetscInfo2(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 22 #define PetscInfo3(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 23 #define PetscInfo4(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 24 #define PetscInfo5(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 25 #define PetscInfo6(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 26 #define PetscInfo7(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 27 #define PetscInfo8(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 28 #define PetscInfo9(...) PETSC_DEPRECATED_MACRO("GCC warning \"Use PetscInfo() (since version 3.17)\"") PetscInfo(__VA_ARGS__) 29 30 /*E 31 PetscInfoCommFlag - Describes the method by which to filter `PetscInfo()` by communicator size 32 33 Used as an input for `PetscInfoSetFilterCommSelf()` 34 35 $ `PETSC_INFO_COMM_ALL` - Default uninitialized value. `PetscInfo()` will not filter based on 36 communicator size (i.e. will print for all communicators) 37 $ `PETSC_INFO_COMM_NO_SELF` - `PetscInfo()` will NOT print for communicators with size = 1 (i.e. *_COMM_SELF) 38 $ `PETSC_INFO_COMM_ONLY_SELF` - `PetscInfo()` will ONLY print for communicators with size = 1 39 40 Level: intermediate 41 42 .seealso: `PetscInfo()`, `PetscInfoSetFromOptions()`, `PetscInfoSetFilterCommSelf()` 43 E*/ 44 typedef enum { 45 PETSC_INFO_COMM_ALL = -1, 46 PETSC_INFO_COMM_NO_SELF = 0, 47 PETSC_INFO_COMM_ONLY_SELF = 1 48 } PetscInfoCommFlag; 49 50 PETSC_EXTERN const char *const PetscInfoCommFlags[]; 51 PETSC_EXTERN PetscErrorCode PetscInfoDeactivateClass(PetscClassId); 52 PETSC_EXTERN PetscErrorCode PetscInfoActivateClass(PetscClassId); 53 PETSC_EXTERN PetscErrorCode PetscInfoEnabled(PetscClassId, PetscBool *); 54 PETSC_EXTERN PetscErrorCode PetscInfoAllow(PetscBool); 55 PETSC_EXTERN PetscErrorCode PetscInfoSetFile(const char[], const char[]); 56 PETSC_EXTERN PetscErrorCode PetscInfoGetFile(char **, FILE **); 57 PETSC_EXTERN PetscErrorCode PetscInfoSetClasses(PetscBool, PetscInt, const char *const *); 58 PETSC_EXTERN PetscErrorCode PetscInfoGetClass(const char *, PetscBool *); 59 PETSC_EXTERN PetscErrorCode PetscInfoGetInfo(PetscBool *, PetscBool *, PetscBool *, PetscBool *, PetscInfoCommFlag *); 60 PETSC_EXTERN PetscErrorCode PetscInfoProcessClass(const char[], PetscInt, const PetscClassId[]); 61 PETSC_EXTERN PetscErrorCode PetscInfoSetFilterCommSelf(PetscInfoCommFlag); 62 PETSC_EXTERN PetscErrorCode PetscInfoSetFromOptions(PetscOptions); 63 PETSC_EXTERN PetscErrorCode PetscInfoDestroy(void); 64 PETSC_EXTERN PetscBool PetscLogPrintInfo; /* if true, indicates PetscInfo() is turned on */ 65 66 /*MC 67 PetscLogEvent - id used to identify PETSc or user events which timed portions (blocks of executable) 68 code. 69 70 Level: intermediate 71 72 .seealso: `PetscLogEventRegister()`, `PetscLogEventBegin()`, `PetscLogEventEnd()`, `PetscLogStage` 73 M*/ 74 typedef int PetscLogEvent; 75 76 /*MC 77 PetscLogStage - id used to identify user stages (phases, sections) of runs - for logging 78 79 Level: intermediate 80 81 .seealso: `PetscLogStageRegister()`, `PetscLogStagePush()`, `PetscLogStagePop()`, `PetscLogEvent` 82 M*/ 83 typedef int PetscLogStage; 84 85 #define PETSC_EVENT 1311311 86 PETSC_EXTERN PetscLogEvent PETSC_LARGEST_EVENT; 87 88 /* Global flop counter */ 89 PETSC_EXTERN PetscLogDouble petsc_TotalFlops; 90 PETSC_EXTERN PetscLogDouble petsc_tmp_flops; 91 92 /* We must make the following structures available to access the event 93 activation flags in the PetscLogEventBegin/End() macros. These are not part of the PETSc public 94 API and are not intended to be used by other parts of PETSc or by users. 95 96 The code that manipulates these structures is in src/sys/logging/utils. 97 */ 98 typedef struct _n_PetscIntStack *PetscIntStack; 99 100 /* -----------------------------------------------------------------------------------------------------*/ 101 /* 102 PetscClassRegInfo, PetscClassPerfInfo - Each class has two data structures associated with it. The first has 103 static information about it, the second collects statistics on how many objects of the class are created, 104 how much memory they use, etc. 105 106 PetscClassRegLog, PetscClassPerfLog - arrays of the PetscClassRegInfo and PetscClassPerfInfo for all classes. 107 */ 108 typedef struct { 109 char *name; /* The class name */ 110 PetscClassId classid; /* The integer identifying this class */ 111 } PetscClassRegInfo; 112 113 typedef struct { 114 PetscClassId id; /* The integer identifying this class */ 115 int creations; /* The number of objects of this class created */ 116 int destructions; /* The number of objects of this class destroyed */ 117 PetscLogDouble mem; /* The total memory allocated by objects of this class; this is completely wrong and should possibly be removed */ 118 PetscLogDouble descMem; /* The total memory allocated by descendents of these objects; this is completely wrong and should possibly be removed */ 119 } PetscClassPerfInfo; 120 121 typedef struct _n_PetscClassRegLog *PetscClassRegLog; 122 struct _n_PetscClassRegLog { 123 int numClasses; /* The number of classes registered */ 124 int maxClasses; /* The maximum number of classes */ 125 PetscClassRegInfo *classInfo; /* The structure for class information (classids are monotonicly increasing) */ 126 }; 127 128 typedef struct _n_PetscClassPerfLog *PetscClassPerfLog; 129 struct _n_PetscClassPerfLog { 130 int numClasses; /* The number of logging classes */ 131 int maxClasses; /* The maximum number of classes */ 132 PetscClassPerfInfo *classInfo; /* The structure for class information (classids are monotonicly increasing) */ 133 }; 134 /* -----------------------------------------------------------------------------------------------------*/ 135 /* 136 PetscEventRegInfo, PetscEventPerfInfo - Each event has two data structures associated with it. The first has 137 static information about it, the second collects statistics on how many times the event is used, how 138 much time it takes, etc. 139 140 PetscEventRegLog, PetscEventPerfLog - an array of all PetscEventRegInfo and PetscEventPerfInfo for all events. There is one 141 of these for each stage. 142 143 */ 144 typedef struct { 145 char *name; /* The name of this event */ 146 PetscClassId classid; /* The class the event is associated with */ 147 PetscBool collective; /* Flag this event as collective */ 148 #if defined(PETSC_HAVE_MPE) 149 int mpe_id_begin; /* MPE IDs that define the event */ 150 int mpe_id_end; 151 #endif 152 } PetscEventRegInfo; 153 154 typedef struct { 155 int id; /* The integer identifying this event */ 156 PetscBool active; /* The flag to activate logging */ 157 PetscBool visible; /* The flag to print info in summary */ 158 int depth; /* The nesting depth of the event call */ 159 int count; /* The number of times this event was executed */ 160 PetscLogDouble flops, flops2, flopsTmp; /* The flops and flops^2 used in this event */ 161 PetscLogDouble time, time2, timeTmp; /* The time and time^2 taken for this event */ 162 PetscLogDouble syncTime; /* The synchronization barrier time */ 163 PetscLogDouble dof[8]; /* The number of degrees of freedom associated with this event */ 164 PetscLogDouble errors[8]; /* The errors (user-defined) associated with this event */ 165 PetscLogDouble numMessages; /* The number of messages in this event */ 166 PetscLogDouble messageLength; /* The total message lengths in this event */ 167 PetscLogDouble numReductions; /* The number of reductions in this event */ 168 PetscLogDouble memIncrease; /* How much the resident memory has increased in this event */ 169 PetscLogDouble mallocIncrease; /* How much the maximum malloced space has increased in this event */ 170 PetscLogDouble mallocSpace; /* How much the space was malloced and kept during this event */ 171 PetscLogDouble mallocIncreaseEvent; /* Maximum of the high water mark with in event minus memory available at the end of the event */ 172 #if defined(PETSC_HAVE_DEVICE) 173 PetscLogDouble CpuToGpuCount; /* The total number of CPU to GPU copies */ 174 PetscLogDouble GpuToCpuCount; /* The total number of GPU to CPU copies */ 175 PetscLogDouble CpuToGpuSize; /* The total size of CPU to GPU copies */ 176 PetscLogDouble GpuToCpuSize; /* The total size of GPU to CPU copies */ 177 PetscLogDouble GpuFlops; /* The flops done on a GPU in this event */ 178 PetscLogDouble GpuTime; /* The time spent on a GPU in this event */ 179 #endif 180 } PetscEventPerfInfo; 181 182 typedef struct _n_PetscEventRegLog *PetscEventRegLog; 183 struct _n_PetscEventRegLog { 184 int numEvents; /* The number of registered events */ 185 int maxEvents; /* The maximum number of events */ 186 PetscEventRegInfo *eventInfo; /* The registration information for each event */ 187 }; 188 189 typedef struct _n_PetscEventPerfLog *PetscEventPerfLog; 190 struct _n_PetscEventPerfLog { 191 int numEvents; /* The number of logging events */ 192 int maxEvents; /* The maximum number of events */ 193 PetscEventPerfInfo *eventInfo; /* The performance information for each event */ 194 }; 195 /* ------------------------------------------------------------------------------------------------------------*/ 196 /* 197 PetscStageInfo - Contains all the information about a particular stage. 198 199 PetscStageLog - An array of PetscStageInfo for each registered stage. There is a single one of these in the code. 200 */ 201 typedef struct _PetscStageInfo { 202 char *name; /* The stage name */ 203 PetscBool used; /* The stage was pushed on this processor */ 204 PetscEventPerfInfo perfInfo; /* The stage performance information */ 205 PetscEventPerfLog eventLog; /* The event information for this stage */ 206 PetscClassPerfLog classLog; /* The class information for this stage */ 207 } PetscStageInfo; 208 209 typedef struct _n_PetscStageLog *PetscStageLog; 210 struct _n_PetscStageLog { 211 int numStages; /* The number of registered stages */ 212 int maxStages; /* The maximum number of stages */ 213 PetscIntStack stack; /* The stack for active stages */ 214 int curStage; /* The current stage (only used in macros so we don't call PetscIntStackTop) */ 215 PetscStageInfo *stageInfo; /* The information for each stage */ 216 PetscEventRegLog eventLog; /* The registered events */ 217 PetscClassRegLog classLog; /* The registered classes */ 218 }; 219 /* -----------------------------------------------------------------------------------------------------*/ 220 221 PETSC_DEPRECATED_FUNCTION("PetscLogObjectParent() is deprecated (since version 3.18)") static inline PetscErrorCode PetscLogObjectParent(PetscObject o, PetscObject p) { 222 return 0; 223 } 224 225 PETSC_DEPRECATED_FUNCTION("PetscLogObjectMemory() is deprecated (since version 3.18)") static inline PetscErrorCode PetscLogObjectMemory(PetscObject o, PetscLogDouble m) { 226 return 0; 227 } 228 229 #if defined(PETSC_USE_LOG) /* --- Logging is turned on --------------------------------*/ 230 PETSC_EXTERN PetscStageLog petsc_stageLog; 231 PETSC_EXTERN PetscErrorCode PetscLogGetStageLog(PetscStageLog *); 232 PETSC_EXTERN PetscErrorCode PetscStageLogGetCurrent(PetscStageLog, int *); 233 PETSC_EXTERN PetscErrorCode PetscStageLogGetEventPerfLog(PetscStageLog, int, PetscEventPerfLog *); 234 235 /* 236 Flop counting: We count each arithmetic operation (e.g., addition, multiplication) separately. 237 238 For the complex numbers version, note that 239 1 complex addition = 2 flops 240 1 complex multiplication = 6 flops, 241 where we define 1 flop as that for a double precision scalar. We roughly approximate 242 flop counting for complex numbers by multiplying the total flops by 4; this corresponds 243 to the assumption that we're counting mostly additions and multiplications -- and 244 roughly the same number of each. More accurate counting could be done by distinguishing 245 among the various arithmetic operations. 246 */ 247 248 #if defined(PETSC_USE_COMPLEX) 249 #define PETSC_FLOPS_PER_OP 4.0 250 #else 251 #define PETSC_FLOPS_PER_OP 1.0 252 #endif 253 254 /*@C 255 PetscLogFlops - Log how many flops are performed in a calculation 256 257 Input Parameter: 258 . flops - the number of flops 259 260 Notes: 261 To limit the chance of integer overflow when multiplying by a constant, represent the constant as a double, 262 not an integer. Use PetscLogFlops(4.0*n) not PetscLogFlops(4*n) 263 264 Level: intermediate 265 266 .seealso: `PetscLogView()`, `PetscLogGpuFlops()` 267 @*/ 268 269 static inline PetscErrorCode PetscLogFlops(PetscLogDouble n) { 270 PetscFunctionBegin; 271 #if defined(PETSC_USE_DEBUG) 272 PetscCheck(n >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cannot log negative flops"); 273 #endif 274 petsc_TotalFlops += PETSC_FLOPS_PER_OP * n; 275 PetscFunctionReturn(0); 276 } 277 278 PETSC_EXTERN PetscErrorCode PetscGetFlops(PetscLogDouble *); 279 280 #if defined(PETSC_HAVE_MPE) 281 PETSC_EXTERN PetscErrorCode PetscLogMPEBegin(void); 282 PETSC_EXTERN PetscErrorCode PetscLogMPEDump(const char[]); 283 #endif 284 285 PETSC_EXTERN PetscErrorCode (*PetscLogPLB)(PetscLogEvent, int, PetscObject, PetscObject, PetscObject, PetscObject); 286 PETSC_EXTERN PetscErrorCode (*PetscLogPLE)(PetscLogEvent, int, PetscObject, PetscObject, PetscObject, PetscObject); 287 PETSC_EXTERN PetscErrorCode (*PetscLogPHC)(PetscObject); 288 PETSC_EXTERN PetscErrorCode (*PetscLogPHD)(PetscObject); 289 290 #define PetscLogObjectParents(p, n, d) PetscMacroReturnStandard(for (int _i = 0; _i < (n); ++_i) PetscCall(PetscLogObjectParent((PetscObject)(p), (PetscObject)(d)[_i]));) 291 #define PetscLogObjectCreate(h) ((PetscLogPHC) ? (*PetscLogPHC)((PetscObject)(h)) : 0) 292 #define PetscLogObjectDestroy(h) ((PetscLogPHD) ? (*PetscLogPHD)((PetscObject)(h)) : 0) 293 PETSC_EXTERN PetscErrorCode PetscLogObjectState(PetscObject, const char[], ...) PETSC_ATTRIBUTE_FORMAT(2, 3); 294 295 /* Initialization functions */ 296 PETSC_EXTERN PetscErrorCode PetscLogDefaultBegin(void); 297 PETSC_EXTERN PetscErrorCode PetscLogAllBegin(void); 298 PETSC_EXTERN PetscErrorCode PetscLogNestedBegin(void); 299 PETSC_EXTERN PetscErrorCode PetscLogTraceBegin(FILE *); 300 PETSC_EXTERN PetscErrorCode PetscLogActions(PetscBool); 301 PETSC_EXTERN PetscErrorCode PetscLogObjects(PetscBool); 302 PETSC_EXTERN PetscErrorCode PetscLogSetThreshold(PetscLogDouble, PetscLogDouble *); 303 PETSC_EXTERN PetscErrorCode PetscLogSet(PetscErrorCode (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject), PetscErrorCode (*)(int, int, PetscObject, PetscObject, PetscObject, PetscObject)); 304 305 /* Output functions */ 306 PETSC_EXTERN PetscErrorCode PetscLogView(PetscViewer); 307 PETSC_EXTERN PetscErrorCode PetscLogViewFromOptions(void); 308 PETSC_EXTERN PetscErrorCode PetscLogDump(const char[]); 309 310 /* Status checking functions */ 311 PETSC_EXTERN PetscErrorCode PetscLogIsActive(PetscBool *); 312 313 /* Stage functions */ 314 PETSC_EXTERN PetscErrorCode PetscLogStageRegister(const char[], PetscLogStage *); 315 PETSC_EXTERN PetscErrorCode PetscLogStagePush(PetscLogStage); 316 PETSC_EXTERN PetscErrorCode PetscLogStagePop(void); 317 PETSC_EXTERN PetscErrorCode PetscLogStageSetActive(PetscLogStage, PetscBool); 318 PETSC_EXTERN PetscErrorCode PetscLogStageGetActive(PetscLogStage, PetscBool *); 319 PETSC_EXTERN PetscErrorCode PetscLogStageSetVisible(PetscLogStage, PetscBool); 320 PETSC_EXTERN PetscErrorCode PetscLogStageGetVisible(PetscLogStage, PetscBool *); 321 PETSC_EXTERN PetscErrorCode PetscLogStageGetId(const char[], PetscLogStage *); 322 323 /* Event functions */ 324 PETSC_EXTERN PetscErrorCode PetscLogEventRegister(const char[], PetscClassId, PetscLogEvent *); 325 PETSC_EXTERN PetscErrorCode PetscLogEventSetCollective(PetscLogEvent, PetscBool); 326 PETSC_EXTERN PetscErrorCode PetscLogEventIncludeClass(PetscClassId); 327 PETSC_EXTERN PetscErrorCode PetscLogEventExcludeClass(PetscClassId); 328 PETSC_EXTERN PetscErrorCode PetscLogEventActivate(PetscLogEvent); 329 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivate(PetscLogEvent); 330 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivatePush(PetscLogEvent); 331 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivatePop(PetscLogEvent); 332 PETSC_EXTERN PetscErrorCode PetscLogEventSetActiveAll(PetscLogEvent, PetscBool); 333 PETSC_EXTERN PetscErrorCode PetscLogEventActivateClass(PetscClassId); 334 PETSC_EXTERN PetscErrorCode PetscLogEventDeactivateClass(PetscClassId); 335 PETSC_EXTERN PetscErrorCode PetscLogEventGetId(const char[], PetscLogEvent *); 336 PETSC_EXTERN PetscErrorCode PetscLogEventGetPerfInfo(int, PetscLogEvent, PetscEventPerfInfo *); 337 PETSC_EXTERN PetscErrorCode PetscLogEventSetDof(PetscLogEvent, PetscInt, PetscLogDouble); 338 PETSC_EXTERN PetscErrorCode PetscLogEventSetError(PetscLogEvent, PetscInt, PetscLogDouble); 339 PETSC_EXTERN PetscErrorCode PetscLogPushCurrentEvent_Internal(PetscLogEvent); 340 PETSC_EXTERN PetscErrorCode PetscLogPopCurrentEvent_Internal(void); 341 342 /* Global counters */ 343 PETSC_EXTERN PetscLogDouble petsc_irecv_ct; 344 PETSC_EXTERN PetscLogDouble petsc_isend_ct; 345 PETSC_EXTERN PetscLogDouble petsc_recv_ct; 346 PETSC_EXTERN PetscLogDouble petsc_send_ct; 347 PETSC_EXTERN PetscLogDouble petsc_irecv_len; 348 PETSC_EXTERN PetscLogDouble petsc_isend_len; 349 PETSC_EXTERN PetscLogDouble petsc_recv_len; 350 PETSC_EXTERN PetscLogDouble petsc_send_len; 351 PETSC_EXTERN PetscLogDouble petsc_allreduce_ct; 352 PETSC_EXTERN PetscLogDouble petsc_gather_ct; 353 PETSC_EXTERN PetscLogDouble petsc_scatter_ct; 354 PETSC_EXTERN PetscLogDouble petsc_wait_ct; 355 PETSC_EXTERN PetscLogDouble petsc_wait_any_ct; 356 PETSC_EXTERN PetscLogDouble petsc_wait_all_ct; 357 PETSC_EXTERN PetscLogDouble petsc_sum_of_waits_ct; 358 359 PETSC_EXTERN PetscBool PetscLogMemory; 360 361 PETSC_EXTERN PetscBool PetscLogSyncOn; /* true if logging synchronization is enabled */ 362 PETSC_EXTERN PetscErrorCode PetscLogEventSynchronize(PetscLogEvent, MPI_Comm); 363 364 #define PetscLogEventSync(e, comm) \ 365 (((PetscLogPLB && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? PetscLogEventSynchronize((e), (comm)) : 0)) 366 367 #define PetscLogEventBegin(e, o1, o2, o3, o4) \ 368 ((PetscLogPLB && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? (((*PetscLogPLB)((e), 0, (PetscObject)(o1), (PetscObject)(o2), (PetscObject)(o3), (PetscObject)(o4))) || PetscLogPushCurrentEvent_Internal(e)) : 0) 369 370 #define PetscLogEventEnd(e, o1, o2, o3, o4) \ 371 ((PetscLogPLE && petsc_stageLog->stageInfo[petsc_stageLog->curStage].perfInfo.active && petsc_stageLog->stageInfo[petsc_stageLog->curStage].eventLog->eventInfo[e].active) ? (((*PetscLogPLE)((e), 0, (PetscObject)(o1), (PetscObject)(o2), (PetscObject)(o3), (PetscObject)(o4))) || PetscLogPopCurrentEvent_Internal()) : 0) 372 373 PETSC_EXTERN PetscErrorCode PetscLogEventGetFlops(PetscLogEvent, PetscLogDouble *); 374 PETSC_EXTERN PetscErrorCode PetscLogEventZeroFlops(PetscLogEvent); 375 376 /* 377 These are used internally in the PETSc routines to keep a count of MPI messages and 378 their sizes. 379 380 This does not work for MPI-Uni because our include/petsc/mpiuni/mpi.h file 381 uses macros to defined the MPI operations. 382 383 It does not work correctly from HP-UX because it processes the 384 macros in a way that sometimes it double counts, hence 385 PETSC_HAVE_BROKEN_RECURSIVE_MACRO 386 387 It does not work with Windows because winmpich lacks MPI_Type_size() 388 */ 389 #if !defined(MPIUNI_H) && !defined(PETSC_HAVE_BROKEN_RECURSIVE_MACRO) && !defined(PETSC_HAVE_MPI_MISSING_TYPESIZE) 390 /* 391 Logging of MPI activities 392 */ 393 static inline PetscErrorCode PetscMPITypeSize(PetscInt count, MPI_Datatype type, PetscLogDouble *length) { 394 PetscMPIInt typesize; 395 396 if (type == MPI_DATATYPE_NULL) return 0; 397 PetscCallMPI(MPI_Type_size(type, &typesize)); 398 *length += (PetscLogDouble)(count * typesize); 399 return 0; 400 } 401 402 static inline PetscErrorCode PetscMPITypeSizeComm(MPI_Comm comm, const PetscMPIInt *counts, MPI_Datatype type, PetscLogDouble *length) { 403 PetscMPIInt typesize, size, p; 404 405 if (type == MPI_DATATYPE_NULL) return 0; 406 PetscCallMPI(MPI_Comm_size(comm, &size)); 407 PetscCallMPI(MPI_Type_size(type, &typesize)); 408 for (p = 0; p < size; ++p) *length += (PetscLogDouble)(counts[p] * typesize); 409 return 0; 410 } 411 412 static inline PetscErrorCode PetscMPITypeSizeCount(PetscInt n, const PetscMPIInt *counts, MPI_Datatype type, PetscLogDouble *length) { 413 PetscMPIInt typesize, p; 414 415 if (type == MPI_DATATYPE_NULL) return 0; 416 PetscCallMPI(MPI_Type_size(type, &typesize)); 417 for (p = 0; p < n; ++p) *length += (PetscLogDouble)(counts[p] * typesize); 418 return 0; 419 } 420 421 /* 422 Returns 1 if the communicator is parallel else zero 423 */ 424 static inline int PetscMPIParallelComm(MPI_Comm comm) { 425 PetscMPIInt size; 426 MPI_Comm_size(comm, &size); 427 return size > 1; 428 } 429 430 #define MPI_Irecv(buf, count, datatype, source, tag, comm, request) ((petsc_irecv_ct++, 0) || PetscMPITypeSize((count), (datatype), &(petsc_irecv_len)) || MPI_Irecv((buf), (count), (datatype), (source), (tag), (comm), (request))) 431 432 #define MPI_Irecv_c(buf, count, datatype, source, tag, comm, request) ((petsc_irecv_ct++, 0) || PetscMPITypeSize((count), (datatype), &(petsc_irecv_len)) || MPI_Irecv_c((buf), (count), (datatype), (source), (tag), (comm), (request))) 433 434 #define MPI_Isend(buf, count, datatype, dest, tag, comm, request) ((petsc_isend_ct++, 0) || PetscMPITypeSize((count), (datatype), &(petsc_isend_len)) || MPI_Isend((buf), (count), (datatype), (dest), (tag), (comm), (request))) 435 436 #define MPI_Isend_c(buf, count, datatype, dest, tag, comm, request) ((petsc_isend_ct++, 0) || PetscMPITypeSize((count), (datatype), &(petsc_isend_len)) || MPI_Isend_c((buf), (count), (datatype), (dest), (tag), (comm), (request))) 437 438 #define MPI_Startall_irecv(count, datatype, number, requests) ((petsc_irecv_ct += (PetscLogDouble)(number), 0) || PetscMPITypeSize((count), (datatype), &(petsc_irecv_len)) || ((number) && MPI_Startall((number), (requests)))) 439 440 #define MPI_Startall_isend(count, datatype, number, requests) ((petsc_isend_ct += (PetscLogDouble)(number), 0) || PetscMPITypeSize((count), (datatype), &(petsc_isend_len)) || ((number) && MPI_Startall((number), (requests)))) 441 442 #define MPI_Start_isend(count, datatype, requests) ((petsc_isend_ct++, 0) || PetscMPITypeSize((count), (datatype), (&petsc_isend_len)) || MPI_Start((requests))) 443 444 #define MPI_Recv(buf, count, datatype, source, tag, comm, status) ((petsc_recv_ct++, 0) || PetscMPITypeSize((count), (datatype), (&petsc_recv_len)) || MPI_Recv((buf), (count), (datatype), (source), (tag), (comm), (status))) 445 446 #define MPI_Recv_c(buf, count, datatype, source, tag, comm, status) ((petsc_recv_ct++, 0) || PetscMPITypeSize((count), (datatype), (&petsc_recv_len)) || MPI_Recv_c((buf), (count), (datatype), (source), (tag), (comm), (status))) 447 448 #define MPI_Send(buf, count, datatype, dest, tag, comm) ((petsc_send_ct++, 0) || PetscMPITypeSize((count), (datatype), (&petsc_send_len)) || MPI_Send((buf), (count), (datatype), (dest), (tag), (comm))) 449 450 #define MPI_Send_c(buf, count, datatype, dest, tag, comm) ((petsc_send_ct++, 0) || PetscMPITypeSize((count), (datatype), (&petsc_send_len)) || MPI_Send_c((buf), (count), (datatype), (dest), (tag), (comm))) 451 452 #define MPI_Wait(request, status) ((petsc_wait_ct++, petsc_sum_of_waits_ct++, 0) || MPI_Wait((request), (status))) 453 454 #define MPI_Waitany(a, b, c, d) ((petsc_wait_any_ct++, petsc_sum_of_waits_ct++, 0) || MPI_Waitany((a), (b), (c), (d))) 455 456 #define MPI_Waitall(count, array_of_requests, array_of_statuses) ((petsc_wait_all_ct++, petsc_sum_of_waits_ct += (PetscLogDouble)(count), 0) || MPI_Waitall((count), (array_of_requests), (array_of_statuses))) 457 458 #define MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm) (petsc_allreduce_ct += PetscMPIParallelComm((comm)), MPI_Allreduce((sendbuf), (recvbuf), (count), (datatype), (op), (comm))) 459 460 #define MPI_Bcast(buffer, count, datatype, root, comm) ((petsc_allreduce_ct += PetscMPIParallelComm((comm)), 0) || MPI_Bcast((buffer), (count), (datatype), (root), (comm))) 461 462 #define MPI_Reduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm) ((petsc_allreduce_ct += PetscMPIParallelComm((comm)), 0) || MPI_Reduce_scatter_block((sendbuf), (recvbuf), (recvcount), (datatype), (op), (comm))) 463 464 #define MPI_Alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm) \ 465 ((petsc_allreduce_ct += PetscMPIParallelComm((comm)), 0) || PetscMPITypeSize((sendcount), (sendtype), (&petsc_send_len)) || MPI_Alltoall((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (comm))) 466 467 #define MPI_Alltoallv(sendbuf, sendcnts, sdispls, sendtype, recvbuf, recvcnts, rdispls, recvtype, comm) \ 468 ((petsc_allreduce_ct += PetscMPIParallelComm((comm)), 0) || PetscMPITypeSizeComm((comm), (sendcnts), (sendtype), (&petsc_send_len)) || MPI_Alltoallv((sendbuf), (sendcnts), (sdispls), (sendtype), (recvbuf), (recvcnts), (rdispls), (recvtype), (comm))) 469 470 #define MPI_Allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm) ((petsc_gather_ct += PetscMPIParallelComm((comm)), 0) || MPI_Allgather((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (comm))) 471 472 #define MPI_Allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcount, displs, recvtype, comm) \ 473 ((petsc_gather_ct += PetscMPIParallelComm((comm)), 0) || MPI_Allgatherv((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (displs), (recvtype), (comm))) 474 475 #define MPI_Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm) \ 476 ((petsc_gather_ct++, 0) || PetscMPITypeSize((sendcount), (sendtype), (&petsc_send_len)) || MPI_Gather((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (root), (comm))) 477 478 #define MPI_Gatherv(sendbuf, sendcount, sendtype, recvbuf, recvcount, displs, recvtype, root, comm) \ 479 ((petsc_gather_ct++, 0) || PetscMPITypeSize((sendcount), (sendtype), (&petsc_send_len)) || MPI_Gatherv((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (displs), (recvtype), (root), (comm))) 480 481 #define MPI_Scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm) \ 482 ((petsc_scatter_ct++, 0) || PetscMPITypeSize((recvcount), (recvtype), (&petsc_recv_len)) || MPI_Scatter((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (root), (comm))) 483 484 #define MPI_Scatterv(sendbuf, sendcount, displs, sendtype, recvbuf, recvcount, recvtype, root, comm) \ 485 ((petsc_scatter_ct++, 0) || PetscMPITypeSize((recvcount), (recvtype), (&petsc_recv_len)) || MPI_Scatterv((sendbuf), (sendcount), (displs), (sendtype), (recvbuf), (recvcount), (recvtype), (root), (comm))) 486 487 #define MPI_Ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request) \ 488 ((petsc_allreduce_ct += PetscMPIParallelComm((comm)), 0) || PetscMPITypeSize((sendcount), (sendtype), (&petsc_send_len)) || MPI_Ialltoall((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (comm), (request))) 489 490 #define MPI_Ialltoallv(sendbuf, sendcnts, sdispls, sendtype, recvbuf, recvcnts, rdispls, recvtype, comm, request) \ 491 ((petsc_allreduce_ct += PetscMPIParallelComm((comm)), 0) || PetscMPITypeSizeComm((comm), (sendcnts), (sendtype), (&petsc_send_len)) || MPI_Ialltoallv((sendbuf), (sendcnts), (sdispls), (sendtype), (recvbuf), (recvcnts), (rdispls), (recvtype), (comm), (request))) 492 493 #define MPI_Iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request) \ 494 ((petsc_gather_ct += PetscMPIParallelComm((comm)), 0) || MPI_Iallgather((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (comm), (request))) 495 496 #define MPI_Iallgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcount, displs, recvtype, comm, request) \ 497 ((petsc_gather_ct += PetscMPIParallelComm((comm)), 0) || MPI_Iallgatherv((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (displs), (recvtype), (comm), (request))) 498 499 #define MPI_Igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request) \ 500 ((petsc_gather_ct++, 0) || PetscMPITypeSize((sendcount), (sendtype), (&petsc_send_len)) || MPI_Igather((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (root), (comm), (request))) 501 502 #define MPI_Igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcount, displs, recvtype, root, comm, request) \ 503 ((petsc_gather_ct++, 0) || PetscMPITypeSize((sendcount), (sendtype), (&petsc_send_len)) || MPI_Igatherv((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (displs), (recvtype), (root), (comm), (request))) 504 505 #define MPI_Iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request) \ 506 ((petsc_scatter_ct++, 0) || PetscMPITypeSize((recvcount), (recvtype), (&petsc_recv_len)) || MPI_Iscatter((sendbuf), (sendcount), (sendtype), (recvbuf), (recvcount), (recvtype), (root), (comm), (request))) 507 508 #define MPI_Iscatterv(sendbuf, sendcount, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request) \ 509 ((petsc_scatter_ct++, 0) || PetscMPITypeSize((recvcount), (recvtype), (&petsc_recv_len)) || MPI_Iscatterv((sendbuf), (sendcount), (displs), (sendtype), (recvbuf), (recvcount), (recvtype), (root), (comm), (request))) 510 511 #else 512 513 #define MPI_Startall_irecv(count, datatype, number, requests) ((number) && MPI_Startall((number), (requests))) 514 515 #define MPI_Startall_isend(count, datatype, number, requests) ((number) && MPI_Startall((number), (requests))) 516 517 #define MPI_Start_isend(count, datatype, requests) (MPI_Start((requests))) 518 519 #endif /* !MPIUNI_H && ! PETSC_HAVE_BROKEN_RECURSIVE_MACRO */ 520 521 #else /* ---Logging is turned off --------------------------------------------*/ 522 523 #define PetscLogMemory PETSC_FALSE 524 525 #define PetscLogFlops(n) 0 526 #define PetscGetFlops(a) (*(a) = 0.0, 0) 527 528 #define PetscLogStageRegister(a, b) 0 529 #define PetscLogStagePush(a) 0 530 #define PetscLogStagePop() 0 531 #define PetscLogStageSetActive(a, b) 0 532 #define PetscLogStageGetActive(a, b) 0 533 #define PetscLogStageGetVisible(a, b) 0 534 #define PetscLogStageSetVisible(a, b) 0 535 #define PetscLogStageGetId(a, b) (*(b) = 0, 0) 536 537 #define PetscLogEventRegister(a, b, c) 0 538 #define PetscLogEventSetCollective(a, b) 0 539 #define PetscLogEventIncludeClass(a) 0 540 #define PetscLogEventExcludeClass(a) 0 541 #define PetscLogEventActivate(a) 0 542 #define PetscLogEventDeactivate(a) 0 543 #define PetscLogEventDeactivatePush(a) 0 544 #define PetscLogEventDeactivatePop(a) 0 545 #define PetscLogEventActivateClass(a) 0 546 #define PetscLogEventDeactivateClass(a) 0 547 #define PetscLogEventSetActiveAll(a, b) 0 548 #define PetscLogEventGetId(a, b) (*(b) = 0, 0) 549 #define PetscLogEventGetPerfInfo(a, b, c) 0 550 #define PetscLogEventSetDof(a, b, c) 0 551 #define PetscLogEventSetError(a, b, c) 0 552 553 #define PetscLogPLB 0 554 #define PetscLogPLE 0 555 #define PetscLogPHC 0 556 #define PetscLogPHD 0 557 558 #define PetscLogObjectParents(p, n, c) 0 559 #define PetscLogObjectCreate(h) 0 560 #define PetscLogObjectDestroy(h) 0 561 PETSC_EXTERN PetscErrorCode PetscLogObjectState(PetscObject, const char[], ...) PETSC_ATTRIBUTE_FORMAT(2, 3); 562 563 #define PetscLogDefaultBegin() 0 564 #define PetscLogAllBegin() 0 565 #define PetscLogNestedBegin() 0 566 #define PetscLogTraceBegin(file) 0 567 #define PetscLogActions(a) 0 568 #define PetscLogObjects(a) 0 569 #define PetscLogSetThreshold(a, b) 0 570 #define PetscLogSet(lb, le) 0 571 #define PetscLogIsActive(flag) (*(flag) = PETSC_FALSE, 0) 572 573 #define PetscLogView(viewer) 0 574 #define PetscLogViewFromOptions() 0 575 #define PetscLogDump(c) 0 576 577 #define PetscLogEventSync(e, comm) 0 578 #define PetscLogEventBegin(e, o1, o2, o3, o4) 0 579 #define PetscLogEventEnd(e, o1, o2, o3, o4) 0 580 581 /* If PETSC_USE_LOG is NOT defined, these still need to be! */ 582 #define MPI_Startall_irecv(count, datatype, number, requests) ((number) && MPI_Startall(number, requests)) 583 #define MPI_Startall_isend(count, datatype, number, requests) ((number) && MPI_Startall(number, requests)) 584 #define MPI_Start_isend(count, datatype, requests) MPI_Start(requests) 585 586 #endif /* PETSC_USE_LOG */ 587 588 #if defined(PETSC_USE_LOG) && defined(PETSC_HAVE_DEVICE) 589 590 /* Global GPU counters */ 591 PETSC_EXTERN PetscLogDouble petsc_ctog_ct; 592 PETSC_EXTERN PetscLogDouble petsc_gtoc_ct; 593 PETSC_EXTERN PetscLogDouble petsc_ctog_sz; 594 PETSC_EXTERN PetscLogDouble petsc_gtoc_sz; 595 PETSC_EXTERN PetscLogDouble petsc_ctog_ct_scalar; 596 PETSC_EXTERN PetscLogDouble petsc_gtoc_ct_scalar; 597 PETSC_EXTERN PetscLogDouble petsc_ctog_sz_scalar; 598 PETSC_EXTERN PetscLogDouble petsc_gtoc_sz_scalar; 599 PETSC_EXTERN PetscLogDouble petsc_gflops; 600 PETSC_EXTERN PetscLogDouble petsc_gtime; 601 602 static inline PetscErrorCode PetscLogCpuToGpu(PetscLogDouble size) { 603 PetscFunctionBegin; 604 petsc_ctog_ct += 1; 605 petsc_ctog_sz += size; 606 PetscFunctionReturn(0); 607 } 608 609 static inline PetscErrorCode PetscLogGpuToCpu(PetscLogDouble size) { 610 PetscFunctionBegin; 611 petsc_gtoc_ct += 1; 612 petsc_gtoc_sz += size; 613 PetscFunctionReturn(0); 614 } 615 616 static inline PetscErrorCode PetscLogCpuToGpuScalar(PetscLogDouble size) { 617 PetscFunctionBegin; 618 petsc_ctog_ct_scalar += 1; 619 petsc_ctog_sz_scalar += size; 620 PetscFunctionReturn(0); 621 } 622 623 static inline PetscErrorCode PetscLogGpuToCpuScalar(PetscLogDouble size) { 624 PetscFunctionBegin; 625 petsc_gtoc_ct_scalar += 1; 626 petsc_gtoc_sz_scalar += size; 627 PetscFunctionReturn(0); 628 } 629 630 /*@C 631 PetscLogGpuFlops - Log how many flops are performed in a calculation on the device 632 633 Input Parameter: 634 . flops - the number of flops 635 636 Notes: 637 To limit the chance of integer overflow when multiplying by a constant, represent the constant as a double, 638 not an integer. Use PetscLogFlops(4.0*n) not PetscLogFlops(4*n) 639 640 The values are also added to the total flop count for the MPI rank that is set with `PetscLogFlops()`; hence the number of flops 641 just on the CPU would be the value from set from `PetscLogFlops()` minus the value set from `PetscLogGpuFlops()` 642 643 Level: intermediate 644 645 .seealso: `PetscLogView()`, `PetscLogFlops()`, `PetscLogGpuTimeBegin()`, `PetscLogGpuTimeEnd()` 646 @*/ 647 static inline PetscErrorCode PetscLogGpuFlops(PetscLogDouble n) { 648 PetscFunctionBegin; 649 PetscCheck(n >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cannot log negative flops"); 650 petsc_TotalFlops += PETSC_FLOPS_PER_OP * n; 651 petsc_gflops += PETSC_FLOPS_PER_OP * n; 652 PetscFunctionReturn(0); 653 } 654 655 static inline PetscErrorCode PetscLogGpuTimeAdd(PetscLogDouble t) { 656 PetscFunctionBegin; 657 petsc_gtime += t; 658 PetscFunctionReturn(0); 659 } 660 661 PETSC_EXTERN PetscErrorCode PetscLogGpuTime(void); 662 PETSC_EXTERN PetscErrorCode PetscLogGpuTimeBegin(void); 663 PETSC_EXTERN PetscErrorCode PetscLogGpuTimeEnd(void); 664 665 #else 666 667 #define PetscLogCpuToGpu(a) 0 668 #define PetscLogGpuToCpu(a) 0 669 #define PetscLogCpuToGpuScalar(a) 0 670 #define PetscLogGpuToCpuScalar(a) 0 671 #define PetscLogGpuFlops(a) 0 672 #define PetscLogGpuTime() 0 673 #define PetscLogGpuTimeAdd(a) 0 674 #define PetscLogGpuTimeBegin() 0 675 #define PetscLogGpuTimeEnd() 0 676 677 #endif /* PETSC_USE_LOG && PETSC_HAVE_DEVICE */ 678 679 #define PetscPreLoadBegin(flag, name) \ 680 do { \ 681 PetscBool PetscPreLoading = flag; \ 682 int PetscPreLoadMax, PetscPreLoadIt; \ 683 PetscLogStage _stageNum; \ 684 PetscCall(PetscOptionsGetBool(NULL, NULL, "-preload", &PetscPreLoading, NULL)); \ 685 PetscPreLoadMax = (int)(PetscPreLoading); \ 686 PetscPreLoadingUsed = PetscPreLoading ? PETSC_TRUE : PetscPreLoadingUsed; \ 687 for (PetscPreLoadIt = 0; PetscPreLoadIt <= PetscPreLoadMax; PetscPreLoadIt++) { \ 688 PetscPreLoadingOn = PetscPreLoading; \ 689 PetscCall(PetscBarrier(NULL)); \ 690 if (PetscPreLoadIt > 0) PetscCall(PetscLogStageGetId(name, &_stageNum)); \ 691 else PetscCall(PetscLogStageRegister(name, &_stageNum)); \ 692 PetscCall(PetscLogStageSetActive(_stageNum, (PetscBool)(!PetscPreLoadMax || PetscPreLoadIt))); \ 693 PetscCall(PetscLogStagePush(_stageNum)); 694 695 #define PetscPreLoadEnd() \ 696 PetscCall(PetscLogStagePop()); \ 697 PetscPreLoading = PETSC_FALSE; \ 698 } \ 699 } \ 700 while (0) 701 702 #define PetscPreLoadStage(name) \ 703 do { \ 704 PetscCall(PetscLogStagePop()); \ 705 if (PetscPreLoadIt > 0) PetscCall(PetscLogStageGetId(name, &_stageNum)); \ 706 else PetscCall(PetscLogStageRegister(name, &_stageNum)); \ 707 PetscCall(PetscLogStageSetActive(_stageNum, (PetscBool)(!PetscPreLoadMax || PetscPreLoadIt))); \ 708 PetscCall(PetscLogStagePush(_stageNum)); \ 709 } while (0) 710 711 /* some vars for logging */ 712 PETSC_EXTERN PetscBool PetscPreLoadingUsed; /* true if we are or have done preloading */ 713 PETSC_EXTERN PetscBool PetscPreLoadingOn; /* true if we are currently in a preloading calculation */ 714 715 #endif 716