1 #include <petscviewer.h> 2 #include "lognested.h" 3 #include "xmlviewer.h" 4 5 PETSC_INTERN PetscErrorCode PetscLogHandlerNestedSetThreshold(PetscLogHandler h, PetscLogDouble newThresh, PetscLogDouble *oldThresh) 6 { 7 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 8 9 PetscFunctionBegin; 10 if (oldThresh) *oldThresh = nested->threshold; 11 if (newThresh == (PetscLogDouble)PETSC_DECIDE) newThresh = 0.01; 12 if (newThresh == (PetscLogDouble)PETSC_DEFAULT) newThresh = 0.01; 13 nested->threshold = PetscMax(newThresh, 0.0); 14 PetscFunctionReturn(PETSC_SUCCESS); 15 } 16 17 static PetscErrorCode PetscLogEventGetNestedEvent(PetscLogHandler h, PetscLogEvent e, PetscLogEvent *nested_event) 18 { 19 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 20 NestedIdPair key; 21 PetscHashIter iter; 22 PetscBool missing; 23 PetscLogState state; 24 25 PetscFunctionBegin; 26 PetscCall(PetscLogHandlerGetState(h, &state)); 27 PetscCall(PetscIntStackTop(nested->nested_stack, &key.root)); 28 key.leaf = NestedIdFromEvent(e); 29 PetscCall(PetscNestedHashPut(nested->pair_map, key, &iter, &missing)); 30 if (missing) { 31 // register a new nested event 32 char name[BUFSIZ]; 33 PetscLogEventInfo event_info; 34 PetscLogEventInfo nested_event_info; 35 36 PetscCall(PetscLogStateEventGetInfo(state, e, &event_info)); 37 PetscCall(PetscLogStateEventGetInfo(nested->state, key.root, &nested_event_info)); 38 PetscCall(PetscSNPrintf(name, sizeof(name) - 1, "%s;%s", nested_event_info.name, event_info.name)); 39 PetscCall(PetscLogStateEventRegister(nested->state, name, event_info.classid, nested_event)); 40 PetscCall(PetscLogStateEventSetCollective(nested->state, *nested_event, event_info.collective)); 41 PetscCall(PetscNestedHashIterSet(nested->pair_map, iter, *nested_event)); 42 } else { 43 PetscCall(PetscNestedHashIterGet(nested->pair_map, iter, nested_event)); 44 } 45 PetscFunctionReturn(PETSC_SUCCESS); 46 } 47 48 static PetscErrorCode PetscLogStageGetNestedEvent(PetscLogHandler h, PetscLogStage stage, PetscLogEvent *nested_event) 49 { 50 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 51 NestedIdPair key; 52 PetscHashIter iter; 53 PetscBool missing; 54 PetscLogState state; 55 56 PetscFunctionBegin; 57 PetscCall(PetscLogHandlerGetState(h, &state)); 58 PetscCall(PetscIntStackTop(nested->nested_stack, &key.root)); 59 key.leaf = NestedIdFromStage(stage); 60 PetscCall(PetscNestedHashPut(nested->pair_map, key, &iter, &missing)); 61 if (missing) { 62 PetscLogStageInfo stage_info; 63 char name[BUFSIZ]; 64 PetscBool collective = PETSC_TRUE; 65 66 PetscCall(PetscLogStateStageGetInfo(state, stage, &stage_info)); 67 if (key.root >= 0) { 68 PetscLogEventInfo nested_event_info; 69 70 PetscCall(PetscLogStateEventGetInfo(nested->state, key.root, &nested_event_info)); 71 PetscCall(PetscSNPrintf(name, sizeof(name) - 1, "%s;%s", nested_event_info.name, stage_info.name)); 72 collective = nested_event_info.collective; 73 } else { 74 PetscCall(PetscSNPrintf(name, sizeof(name) - 1, "%s", stage_info.name)); 75 } 76 PetscCall(PetscLogStateEventRegister(nested->state, name, nested->nested_stage_id, nested_event)); 77 PetscCall(PetscLogStateEventSetCollective(nested->state, *nested_event, collective)); 78 PetscCall(PetscNestedHashIterSet(nested->pair_map, iter, *nested_event)); 79 } else { 80 PetscCall(PetscNestedHashIterGet(nested->pair_map, iter, nested_event)); 81 } 82 PetscFunctionReturn(PETSC_SUCCESS); 83 } 84 85 static PetscErrorCode PetscLogNestedFindNestedId(PetscLogHandler h, NestedId orig_id, PetscInt *pop_count) 86 { 87 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 88 PetscInt count, i; 89 90 PetscFunctionBegin; 91 // stop before zero cause there is a null event at the bottom of the stack 92 for (i = nested->orig_stack->top, count = 0; i > 0; i--) { 93 count++; 94 if (nested->orig_stack->stack[i] == orig_id) break; 95 } 96 *pop_count = count; 97 if (count == 1) PetscFunctionReturn(PETSC_SUCCESS); // Normal function, just the top of the stack is being popped. 98 if (orig_id > 0) { 99 PetscLogEvent event_id = NestedIdToEvent(orig_id); 100 PetscLogState state; 101 PetscLogEventInfo event_info; 102 103 PetscCall(PetscLogHandlerGetState(h, &state)); 104 PetscCall(PetscLogStateEventGetInfo(state, event_id, &event_info)); 105 PetscCheck(i > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Tried to end event %s, but it is not in the event stack", event_info.name); 106 } else { 107 PetscLogStage stage_id = NestedIdToStage(orig_id); 108 PetscLogState state; 109 PetscLogStageInfo stage_info; 110 111 PetscCall(PetscLogHandlerGetState(h, &state)); 112 PetscCall(PetscLogStateStageGetInfo(state, stage_id, &stage_info)); 113 PetscCheck(i > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Tried to pop stage %s, but it is not in the stage stack", stage_info.name); 114 } 115 PetscFunctionReturn(PETSC_SUCCESS); 116 } 117 118 static PetscErrorCode PetscLogNestedCheckNested(PetscLogHandler h, NestedId leaf, PetscLogEvent nested_event) 119 { 120 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 121 NestedIdPair key; 122 NestedId val; 123 124 PetscFunctionBegin; 125 PetscCall(PetscIntStackTop(nested->nested_stack, &key.root)); 126 key.leaf = leaf; 127 PetscCall(PetscNestedHashGet(nested->pair_map, key, &val)); 128 PetscCheck(val == nested_event, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Logging events and stages are not nested, nested logging cannot be used"); 129 PetscFunctionReturn(PETSC_SUCCESS); 130 } 131 132 static PetscErrorCode PetscLogHandlerEventBegin_Nested(PetscLogHandler h, PetscLogEvent e, PetscObject o1, PetscObject o2, PetscObject o3, PetscObject o4) 133 { 134 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 135 PetscLogEvent nested_event; 136 137 PetscFunctionBegin; 138 PetscCall(PetscLogEventGetNestedEvent(h, e, &nested_event)); 139 PetscCall(PetscLogHandlerEventBegin(nested->handler, nested_event, o1, o2, o3, o4)); 140 PetscCall(PetscIntStackPush(nested->nested_stack, nested_event)); 141 PetscCall(PetscIntStackPush(nested->orig_stack, NestedIdFromEvent(e))); 142 PetscFunctionReturn(PETSC_SUCCESS); 143 } 144 145 static PetscErrorCode PetscLogHandlerNestedEventEnd(PetscLogHandler h, NestedId id, PetscObject o1, PetscObject o2, PetscObject o3, PetscObject o4) 146 { 147 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 148 PetscInt pop_count; 149 150 PetscFunctionBegin; 151 PetscCall(PetscLogNestedFindNestedId(h, id, &pop_count)); 152 for (PetscInt c = 0; c < pop_count; c++) { 153 PetscLogEvent nested_event; 154 PetscLogEvent nested_id; 155 156 PetscCall(PetscIntStackPop(nested->nested_stack, &nested_event)); 157 PetscCall(PetscIntStackPop(nested->orig_stack, &nested_id)); 158 if (PetscDefined(USE_DEBUG)) PetscCall(PetscLogNestedCheckNested(h, nested_id, nested_event)); 159 if ((pop_count > 1) && (c + 1 < pop_count)) { 160 if (nested_id > 0) { 161 PetscLogEvent event_id = NestedIdToEvent(nested_id); 162 PetscLogState state; 163 PetscLogEventInfo event_info; 164 165 PetscCall(PetscLogHandlerGetState(h, &state)); 166 PetscCall(PetscLogStateEventGetInfo(state, event_id, &event_info)); 167 PetscCall(PetscInfo(h, "Log event %s wasn't ended, ending it to maintain stack property for nested log handler\n", event_info.name)); 168 } 169 } 170 PetscCall(PetscLogHandlerEventEnd(nested->handler, nested_event, o1, o2, o3, o4)); 171 } 172 PetscFunctionReturn(PETSC_SUCCESS); 173 } 174 175 static PetscErrorCode PetscLogHandlerEventEnd_Nested(PetscLogHandler h, PetscLogEvent e, PetscObject o1, PetscObject o2, PetscObject o3, PetscObject o4) 176 { 177 PetscFunctionBegin; 178 PetscCall(PetscLogHandlerNestedEventEnd(h, NestedIdFromEvent(e), o1, o2, o3, o4)); 179 PetscFunctionReturn(PETSC_SUCCESS); 180 } 181 182 static PetscErrorCode PetscLogHandlerEventSync_Nested(PetscLogHandler h, PetscLogEvent e, MPI_Comm comm) 183 { 184 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 185 PetscLogEvent nested_event; 186 187 PetscFunctionBegin; 188 PetscCall(PetscLogEventGetNestedEvent(h, e, &nested_event)); 189 PetscCall(PetscLogHandlerEventSync(nested->handler, nested_event, comm)); 190 PetscFunctionReturn(PETSC_SUCCESS); 191 } 192 193 static PetscErrorCode PetscLogHandlerStagePush_Nested(PetscLogHandler h, PetscLogStage stage) 194 { 195 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 196 PetscLogEvent nested_event; 197 198 PetscFunctionBegin; 199 if (nested->nested_stage_id == -1) PetscCall(PetscClassIdRegister("LogNestedStage", &nested->nested_stage_id)); 200 PetscCall(PetscLogStageGetNestedEvent(h, stage, &nested_event)); 201 PetscCall(PetscLogHandlerEventBegin(nested->handler, nested_event, NULL, NULL, NULL, NULL)); 202 PetscCall(PetscIntStackPush(nested->nested_stack, nested_event)); 203 PetscCall(PetscIntStackPush(nested->orig_stack, NestedIdFromStage(stage))); 204 PetscFunctionReturn(PETSC_SUCCESS); 205 } 206 207 static PetscErrorCode PetscLogHandlerStagePop_Nested(PetscLogHandler h, PetscLogStage stage) 208 { 209 PetscFunctionBegin; 210 PetscCall(PetscLogHandlerNestedEventEnd(h, NestedIdFromStage(stage), NULL, NULL, NULL, NULL)); 211 PetscFunctionReturn(PETSC_SUCCESS); 212 } 213 214 static PetscErrorCode PetscLogHandlerContextCreate_Nested(MPI_Comm comm, PetscLogHandler_Nested *nested_p) 215 { 216 PetscLogStage root_stage; 217 PetscLogHandler_Nested nested; 218 219 PetscFunctionBegin; 220 PetscCall(PetscNew(nested_p)); 221 nested = *nested_p; 222 PetscCall(PetscLogStateCreate(&nested->state)); 223 PetscCall(PetscIntStackCreate(&nested->nested_stack)); 224 PetscCall(PetscIntStackCreate(&nested->orig_stack)); 225 nested->nested_stage_id = -1; 226 nested->threshold = 0.01; 227 PetscCall(PetscNestedHashCreate(&nested->pair_map)); 228 PetscCall(PetscLogHandlerCreate(comm, &nested->handler)); 229 PetscCall(PetscLogHandlerSetType(nested->handler, PETSCLOGHANDLERDEFAULT)); 230 PetscCall(PetscLogHandlerSetState(nested->handler, nested->state)); 231 PetscCall(PetscLogStateStageRegister(nested->state, "", &root_stage)); 232 PetscAssert(root_stage == 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "root stage not zero"); 233 PetscCall(PetscLogHandlerStagePush(nested->handler, root_stage)); 234 PetscCall(PetscLogStateStagePush(nested->state, root_stage)); 235 PetscCall(PetscIntStackPush(nested->nested_stack, -1)); 236 PetscCall(PetscIntStackPush(nested->orig_stack, -1)); 237 PetscFunctionReturn(PETSC_SUCCESS); 238 } 239 240 static PetscErrorCode PetscLogHandlerObjectCreate_Nested(PetscLogHandler h, PetscObject obj) 241 { 242 PetscClassId classid; 243 PetscInt num_registered, num_nested_registered; 244 PetscLogState state; 245 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 246 247 PetscFunctionBegin; 248 // register missing objects 249 PetscCall(PetscObjectGetClassId(obj, &classid)); 250 PetscCall(PetscLogHandlerGetState(h, &state)); 251 PetscCall(PetscLogStateGetNumClasses(nested->state, &num_nested_registered)); 252 PetscCall(PetscLogStateGetNumClasses(state, &num_registered)); 253 for (PetscLogClass c = num_nested_registered; c < num_registered; c++) { 254 PetscLogClassInfo class_info; 255 PetscLogClass nested_c; 256 257 PetscCall(PetscLogStateClassGetInfo(state, c, &class_info)); 258 PetscCall(PetscLogStateClassRegister(nested->state, class_info.name, class_info.classid, &nested_c)); 259 } 260 PetscCall(PetscLogHandlerObjectCreate(nested->handler, obj)); 261 PetscFunctionReturn(PETSC_SUCCESS); 262 } 263 264 static PetscErrorCode PetscLogHandlerObjectDestroy_Nested(PetscLogHandler h, PetscObject obj) 265 { 266 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 267 268 PetscFunctionBegin; 269 PetscCall(PetscLogHandlerObjectDestroy(nested->handler, obj)); 270 PetscFunctionReturn(PETSC_SUCCESS); 271 } 272 273 static PetscErrorCode PetscLogHandlerDestroy_Nested(PetscLogHandler h) 274 { 275 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)h->data; 276 277 PetscFunctionBegin; 278 PetscCall(PetscLogStateStagePop(nested->state)); 279 PetscCall(PetscLogHandlerStagePop(nested->handler, 0)); 280 PetscCall(PetscLogStateDestroy(&nested->state)); 281 PetscCall(PetscIntStackDestroy(nested->nested_stack)); 282 PetscCall(PetscIntStackDestroy(nested->orig_stack)); 283 PetscCall(PetscNestedHashDestroy(&nested->pair_map)); 284 PetscCall(PetscLogHandlerDestroy(&nested->handler)); 285 PetscCall(PetscFree(nested)); 286 PetscFunctionReturn(PETSC_SUCCESS); 287 } 288 289 static PetscErrorCode PetscLogNestedEventNodesOrderDepthFirst(PetscInt num_nodes, PetscInt parent, PetscNestedEventNode tree[], PetscInt *num_descendants) 290 { 291 PetscInt node, start_loc; 292 293 PetscFunctionBegin; 294 node = 0; 295 start_loc = 0; 296 while (node < num_nodes) { 297 if (tree[node].parent == parent) { 298 PetscInt num_this_descendants = 0; 299 PetscNestedEventNode tmp = tree[start_loc]; 300 tree[start_loc] = tree[node]; 301 tree[node] = tmp; 302 PetscCall(PetscLogNestedEventNodesOrderDepthFirst(num_nodes - start_loc - 1, tree[start_loc].id, &tree[start_loc + 1], &num_this_descendants)); 303 tree[start_loc].num_descendants = num_this_descendants; 304 *num_descendants += 1 + num_this_descendants; 305 start_loc += 1 + num_this_descendants; 306 node = start_loc; 307 } else { 308 node++; 309 } 310 } 311 PetscFunctionReturn(PETSC_SUCCESS); 312 } 313 314 static PetscErrorCode PetscLogNestedCreatePerfNodes(MPI_Comm comm, PetscLogHandler_Nested nested, PetscLogGlobalNames global_events, PetscNestedEventNode **tree_p, PetscEventPerfInfo **perf_p) 315 { 316 PetscMPIInt size; 317 PetscInt num_nodes; 318 PetscInt num_map_entries; 319 PetscEventPerfInfo *perf; 320 NestedIdPair *keys; 321 NestedId *vals; 322 PetscInt offset; 323 PetscInt num_descendants; 324 PetscNestedEventNode *tree; 325 326 PetscFunctionBegin; 327 PetscCall(PetscLogGlobalNamesGetSize(global_events, NULL, &num_nodes)); 328 PetscCall(PetscCalloc1(num_nodes, &tree)); 329 for (PetscInt node = 0; node < num_nodes; node++) { 330 tree[node].id = node; 331 PetscCall(PetscLogGlobalNamesGlobalGetName(global_events, node, &tree[node].name)); 332 tree[node].parent = -1; 333 } 334 PetscCall(PetscNestedHashGetSize(nested->pair_map, &num_map_entries)); 335 PetscCall(PetscMalloc2(num_map_entries, &keys, num_map_entries, &vals)); 336 offset = 0; 337 PetscCall(PetscNestedHashGetPairs(nested->pair_map, &offset, keys, vals)); 338 for (PetscInt k = 0; k < num_map_entries; k++) { 339 NestedId root_local = keys[k].root; 340 NestedId leaf_local = vals[k]; 341 PetscInt root_global; 342 PetscInt leaf_global; 343 344 PetscCall(PetscLogGlobalNamesLocalGetGlobal(global_events, leaf_local, &leaf_global)); 345 if (root_local >= 0) { 346 PetscCall(PetscLogGlobalNamesLocalGetGlobal(global_events, root_local, &root_global)); 347 tree[leaf_global].parent = root_global; 348 } 349 } 350 PetscCall(PetscFree2(keys, vals)); 351 PetscCallMPI(MPI_Comm_size(comm, &size)); 352 if (size > 1) { // get missing parents from other processes 353 PetscInt *parents; 354 355 PetscCall(PetscMalloc1(num_nodes, &parents)); 356 for (PetscInt node = 0; node < num_nodes; node++) parents[node] = tree[node].parent; 357 PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, parents, num_nodes, MPIU_INT, MPI_MAX, comm)); 358 for (PetscInt node = 0; node < num_nodes; node++) tree[node].parent = parents[node]; 359 PetscCall(PetscFree(parents)); 360 } 361 362 num_descendants = 0; 363 PetscCall(PetscLogNestedEventNodesOrderDepthFirst(num_nodes, -1, tree, &num_descendants)); 364 PetscAssert(num_descendants == num_nodes, comm, PETSC_ERR_PLIB, "Failed tree ordering invariant"); 365 366 PetscCall(PetscCalloc1(num_nodes, &perf)); 367 for (PetscInt tree_node = 0; tree_node < num_nodes; tree_node++) { 368 PetscInt global_id = tree[tree_node].id; 369 PetscInt event_id; 370 371 PetscCall(PetscLogGlobalNamesGlobalGetLocal(global_events, global_id, &event_id)); 372 if (event_id >= 0) { 373 PetscEventPerfInfo *event_info; 374 375 PetscCall(PetscLogHandlerGetEventPerfInfo(nested->handler, 0, event_id, &event_info)); 376 perf[tree_node] = *event_info; 377 } else { 378 PetscCall(PetscArrayzero(&perf[tree_node], 1)); 379 } 380 } 381 *tree_p = tree; 382 *perf_p = perf; 383 PetscFunctionReturn(PETSC_SUCCESS); 384 } 385 386 static PetscErrorCode PetscLogHandlerView_Nested(PetscLogHandler handler, PetscViewer viewer) 387 { 388 PetscLogHandler_Nested nested = (PetscLogHandler_Nested)handler->data; 389 PetscNestedEventNode *nodes; 390 PetscEventPerfInfo *perf; 391 PetscLogGlobalNames global_events; 392 PetscNestedEventTree tree; 393 PetscViewerFormat format; 394 MPI_Comm comm = PetscObjectComm((PetscObject)viewer); 395 396 PetscFunctionBegin; 397 PetscCall(PetscLogRegistryCreateGlobalEventNames(comm, nested->state->registry, &global_events)); 398 PetscCall(PetscLogNestedCreatePerfNodes(comm, nested, global_events, &nodes, &perf)); 399 tree.comm = comm; 400 tree.global_events = global_events; 401 tree.perf = perf; 402 tree.nodes = nodes; 403 PetscCall(PetscViewerGetFormat(viewer, &format)); 404 if (format == PETSC_VIEWER_ASCII_XML) { 405 PetscCall(PetscLogHandlerView_Nested_XML(nested, &tree, viewer)); 406 } else if (format == PETSC_VIEWER_ASCII_FLAMEGRAPH) { 407 PetscCall(PetscLogHandlerView_Nested_Flamegraph(nested, &tree, viewer)); 408 } else SETERRQ(comm, PETSC_ERR_ARG_INCOMP, "No nested viewer for this format"); 409 PetscCall(PetscLogGlobalNamesDestroy(&global_events)); 410 PetscCall(PetscFree(tree.nodes)); 411 PetscCall(PetscFree(tree.perf)); 412 PetscFunctionReturn(PETSC_SUCCESS); 413 } 414 415 /*MC 416 PETSCLOGHANDLERNESTED - PETSCLOGHANDLERNESTED = "nested" - A `PetscLogHandler` that collects data for PETSc's 417 XML and flamegraph profiling log viewers. A log handler of this type is created and started by 418 by `PetscLogNestedBegin()`. 419 420 Level: developer 421 422 .seealso: [](ch_profiling), `PetscLogHandler` 423 M*/ 424 425 PETSC_INTERN PetscErrorCode PetscLogHandlerCreate_Nested(PetscLogHandler handler) 426 { 427 PetscFunctionBegin; 428 PetscCall(PetscLogHandlerContextCreate_Nested(PetscObjectComm((PetscObject)handler), (PetscLogHandler_Nested *)&handler->data)); 429 handler->ops->destroy = PetscLogHandlerDestroy_Nested; 430 handler->ops->stagepush = PetscLogHandlerStagePush_Nested; 431 handler->ops->stagepop = PetscLogHandlerStagePop_Nested; 432 handler->ops->eventbegin = PetscLogHandlerEventBegin_Nested; 433 handler->ops->eventend = PetscLogHandlerEventEnd_Nested; 434 handler->ops->eventsync = PetscLogHandlerEventSync_Nested; 435 handler->ops->objectcreate = PetscLogHandlerObjectCreate_Nested; 436 handler->ops->objectdestroy = PetscLogHandlerObjectDestroy_Nested; 437 handler->ops->view = PetscLogHandlerView_Nested; 438 PetscFunctionReturn(PETSC_SUCCESS); 439 } 440