1 2 /* 3 4 This file defines part of the initialization of PETSc 5 6 This file uses regular malloc and free because it cannot know 7 what malloc is being used until it has already processed the input. 8 */ 9 10 #include <petscsys.h> /*I "petscsys.h" I*/ 11 #include <pthread.h> 12 #if defined(PETSC_HAVE_STDLIB_H) 13 #include <stdlib.h> 14 #endif 15 #if defined(PETSC_HAVE_MALLOC_H) 16 #include <malloc.h> 17 #endif 18 #if defined(PETSC_HAVE_VALGRIND) 19 #include <valgrind/valgrind.h> 20 #endif 21 22 /* ------------------------Nasty global variables -------------------------------*/ 23 /* 24 Indicates if PETSc started up MPI, or it was 25 already started before PETSc was initialized. 26 */ 27 PetscBool PetscBeganMPI = PETSC_FALSE; 28 PetscBool PetscInitializeCalled = PETSC_FALSE; 29 PetscBool PetscFinalizeCalled = PETSC_FALSE; 30 PetscBool PetscUseThreadPool = PETSC_FALSE; 31 PetscBool PetscThreadGo = PETSC_TRUE; 32 PetscMPIInt PetscGlobalRank = -1; 33 PetscMPIInt PetscGlobalSize = -1; 34 PetscMPIInt PetscMaxThreads = 2; 35 pthread_t* PetscThreadPoint; 36 pthread_barrier_t* BarrPoint; 37 38 typedef struct { 39 pthread_mutex_t mutex; 40 pthread_cond_t cond; 41 void* (*pfunc)(void*); 42 void** pdata; 43 pthread_barrier_t* pbarr; 44 int iNumJobThreads; 45 int iNumReadyThreads; 46 PetscBool startJob; 47 } sjob; 48 sjob job = {PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER,NULL,NULL,NULL,0,0,PETSC_FALSE}; 49 50 pthread_cond_t main_cond = PTHREAD_COND_INITIALIZER; 51 52 void* PetscThreadFunc(void*); 53 void* PetscThreadInitialize(PetscInt); 54 PetscErrorCode PetscThreadFinalize(void); 55 void MainWait(void); 56 void MainJob(void* (*pFunc)(void*),void**,pthread_barrier_t*,PetscInt); 57 void* FuncFinish(void*); 58 59 #if defined(PETSC_USE_COMPLEX) 60 #if defined(PETSC_COMPLEX_INSTANTIATE) 61 template <> class std::complex<double>; /* instantiate complex template class */ 62 #endif 63 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 64 MPI_Datatype MPI_C_DOUBLE_COMPLEX; 65 MPI_Datatype MPI_C_COMPLEX; 66 #endif 67 PetscScalar PETSC_i; 68 #else 69 PetscScalar PETSC_i = 0.0; 70 #endif 71 #if defined(PETSC_USE_REAL___FLOAT128) 72 MPI_Datatype MPIU___FLOAT128 = 0; 73 #endif 74 MPI_Datatype MPIU_2SCALAR = 0; 75 MPI_Datatype MPIU_2INT = 0; 76 77 /* 78 These are needed by petscbt.h 79 */ 80 #include <petscbt.h> 81 char _BT_mask = ' '; 82 char _BT_c = ' '; 83 PetscInt _BT_idx = 0; 84 85 /* 86 Function that is called to display all error messages 87 */ 88 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 89 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 90 #if defined(PETSC_HAVE_MATLAB_ENGINE) 91 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintf_Matlab; 92 #else 93 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 94 #endif 95 /* 96 This is needed to turn on/off cusp synchronization */ 97 PetscBool synchronizeCUSP = PETSC_FALSE; 98 99 /* ------------------------------------------------------------------------------*/ 100 /* 101 Optional file where all PETSc output from various prints is saved 102 */ 103 FILE *petsc_history = PETSC_NULL; 104 105 #undef __FUNCT__ 106 #define __FUNCT__ "PetscOpenHistoryFile" 107 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 108 { 109 PetscErrorCode ierr; 110 PetscMPIInt rank,size; 111 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 112 char version[256]; 113 114 PetscFunctionBegin; 115 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 116 if (!rank) { 117 char arch[10]; 118 int err; 119 PetscViewer viewer; 120 121 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 122 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 123 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 124 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 125 if (filename) { 126 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 127 } else { 128 ierr = PetscGetHomeDirectory(pfile,240);CHKERRQ(ierr); 129 ierr = PetscStrcat(pfile,"/.petschistory");CHKERRQ(ierr); 130 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 131 } 132 133 *fd = fopen(fname,"a"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 134 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 135 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 136 ierr = PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr); 137 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 138 ierr = PetscViewerASCIIOpenWithFILE(PETSC_COMM_WORLD,*fd,&viewer);CHKERRQ(ierr); 139 ierr = PetscOptionsView(viewer);CHKERRQ(ierr); 140 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 141 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 142 err = fflush(*fd); 143 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 144 } 145 PetscFunctionReturn(0); 146 } 147 148 #undef __FUNCT__ 149 #define __FUNCT__ "PetscCloseHistoryFile" 150 PetscErrorCode PetscCloseHistoryFile(FILE **fd) 151 { 152 PetscErrorCode ierr; 153 PetscMPIInt rank; 154 char date[64]; 155 int err; 156 157 PetscFunctionBegin; 158 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 159 if (!rank) { 160 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 161 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 162 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 164 err = fflush(*fd); 165 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 166 err = fclose(*fd); 167 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 168 } 169 PetscFunctionReturn(0); 170 } 171 172 /* ------------------------------------------------------------------------------*/ 173 174 /* 175 This is ugly and probably belongs somewhere else, but I want to 176 be able to put a true MPI abort error handler with command line args. 177 178 This is so MPI errors in the debugger will leave all the stack 179 frames. The default MP_Abort() cleans up and exits thus providing no useful information 180 in the debugger hence we call abort() instead of MPI_Abort(). 181 */ 182 183 #undef __FUNCT__ 184 #define __FUNCT__ "Petsc_MPI_AbortOnError" 185 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag) 186 { 187 PetscFunctionBegin; 188 (*PetscErrorPrintf)("MPI error %d\n",*flag); 189 abort(); 190 } 191 192 #undef __FUNCT__ 193 #define __FUNCT__ "Petsc_MPI_DebuggerOnError" 194 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag) 195 { 196 PetscErrorCode ierr; 197 198 PetscFunctionBegin; 199 (*PetscErrorPrintf)("MPI error %d\n",*flag); 200 ierr = PetscAttachDebugger(); 201 if (ierr) { /* hopeless so get out */ 202 MPI_Abort(*comm,*flag); 203 } 204 } 205 206 #undef __FUNCT__ 207 #define __FUNCT__ "PetscEnd" 208 /*@C 209 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 210 wishes a clean exit somewhere deep in the program. 211 212 Collective on PETSC_COMM_WORLD 213 214 Options Database Keys are the same as for PetscFinalize() 215 216 Level: advanced 217 218 Note: 219 See PetscInitialize() for more general runtime options. 220 221 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 222 @*/ 223 PetscErrorCode PetscEnd(void) 224 { 225 PetscFunctionBegin; 226 PetscFinalize(); 227 exit(0); 228 return 0; 229 } 230 231 PetscBool PetscOptionsPublish = PETSC_FALSE; 232 extern PetscErrorCode PetscSetUseTrMalloc_Private(void); 233 extern PetscBool petscsetmallocvisited; 234 static char emacsmachinename[256]; 235 236 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0; 237 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0; 238 239 #undef __FUNCT__ 240 #define __FUNCT__ "PetscSetHelpVersionFunctions" 241 /*@C 242 PetscSetHelpVersionFunctions - Sets functions that print help and version information 243 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 244 This routine enables a "higher-level" package that uses PETSc to print its messages first. 245 246 Input Parameter: 247 + help - the help function (may be PETSC_NULL) 248 - version - the version function (may be PETSC_NULL) 249 250 Level: developer 251 252 Concepts: package help message 253 254 @*/ 255 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 256 { 257 PetscFunctionBegin; 258 PetscExternalHelpFunction = help; 259 PetscExternalVersionFunction = version; 260 PetscFunctionReturn(0); 261 } 262 263 #undef __FUNCT__ 264 #define __FUNCT__ "PetscOptionsCheckInitial_Private" 265 PetscErrorCode PetscOptionsCheckInitial_Private(void) 266 { 267 char string[64],mname[PETSC_MAX_PATH_LEN],*f; 268 MPI_Comm comm = PETSC_COMM_WORLD; 269 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flg4 = PETSC_FALSE,flag,flgz,flgzout; 270 PetscErrorCode ierr; 271 PetscReal si; 272 int i; 273 PetscMPIInt rank; 274 char version[256]; 275 276 PetscFunctionBegin; 277 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 278 279 /* 280 Setup the memory management; support for tracing malloc() usage 281 */ 282 ierr = PetscOptionsHasName(PETSC_NULL,"-malloc_log",&flg3);CHKERRQ(ierr); 283 #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD) 284 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg1,&flg2);CHKERRQ(ierr); 285 if ((!flg2 || flg1) && !petscsetmallocvisited) { 286 #if defined(PETSC_HAVE_VALGRIND) 287 if (flg2 || !(RUNNING_ON_VALGRIND)) { 288 /* turn off default -malloc if valgrind is being used */ 289 #endif 290 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 291 #if defined(PETSC_HAVE_VALGRIND) 292 } 293 #endif 294 } 295 #else 296 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);CHKERRQ(ierr); 297 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg2,PETSC_NULL);CHKERRQ(ierr); 298 if (flg1 || flg2 || flg3) {ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr);} 299 #endif 300 if (flg3) { 301 ierr = PetscMallocSetDumpLog();CHKERRQ(ierr); 302 } 303 flg1 = PETSC_FALSE; 304 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc_debug",&flg1,PETSC_NULL);CHKERRQ(ierr); 305 if (flg1) { 306 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 307 ierr = PetscMallocDebug(PETSC_TRUE);CHKERRQ(ierr); 308 } 309 310 flg1 = PETSC_FALSE; 311 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg1,PETSC_NULL);CHKERRQ(ierr); 312 if (!flg1) { 313 flg1 = PETSC_FALSE; 314 ierr = PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg1,PETSC_NULL);CHKERRQ(ierr); 315 } 316 if (flg1) { 317 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 318 } 319 320 /* 321 Set the display variable for graphics 322 */ 323 ierr = PetscSetDisplay();CHKERRQ(ierr); 324 325 /* 326 Determine whether user specified maximum number of threads 327 */ 328 ierr = PetscOptionsHasName(PETSC_NULL,"-thread_max",&flg1);CHKERRQ(ierr); 329 if(flg1) { 330 ierr = PetscOptionsGetInt(PETSC_NULL,"-thread_max",&PetscMaxThreads,PETSC_NULL);CHKERRQ(ierr); 331 } 332 333 /* 334 Determine whether to use thread pool 335 */ 336 ierr = PetscOptionsHasName(PETSC_NULL,"-use_thread_pool",&flg1);CHKERRQ(ierr); 337 if(flg1) { 338 PetscUseThreadPool = PETSC_TRUE; 339 PetscThreadInitialize(PetscMaxThreads); 340 } 341 342 /* 343 Print the PETSc version information 344 */ 345 ierr = PetscOptionsHasName(PETSC_NULL,"-v",&flg1);CHKERRQ(ierr); 346 ierr = PetscOptionsHasName(PETSC_NULL,"-version",&flg2);CHKERRQ(ierr); 347 ierr = PetscOptionsHasName(PETSC_NULL,"-help",&flg3);CHKERRQ(ierr); 348 if (flg1 || flg2 || flg3){ 349 350 /* 351 Print "higher-level" package version message 352 */ 353 if (PetscExternalVersionFunction) { 354 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 355 } 356 357 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 358 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------\ 359 ------------------------------\n");CHKERRQ(ierr); 360 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 361 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 362 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 363 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 364 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 365 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 366 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------\ 367 ------------------------------\n");CHKERRQ(ierr); 368 } 369 370 /* 371 Print "higher-level" package help message 372 */ 373 if (flg3){ 374 if (PetscExternalHelpFunction) { 375 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 376 } 377 } 378 379 /* 380 Setup the error handling 381 */ 382 flg1 = PETSC_FALSE; 383 ierr = PetscOptionsGetBool(PETSC_NULL,"-on_error_abort",&flg1,PETSC_NULL);CHKERRQ(ierr); 384 if (flg1) { ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr);} 385 flg1 = PETSC_FALSE; 386 ierr = PetscOptionsGetBool(PETSC_NULL,"-on_error_mpiabort",&flg1,PETSC_NULL);CHKERRQ(ierr); 387 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);CHKERRQ(ierr);} 388 flg1 = PETSC_FALSE; 389 ierr = PetscOptionsGetBool(PETSC_NULL,"-mpi_return_on_error",&flg1,PETSC_NULL);CHKERRQ(ierr); 390 if (flg1) { 391 ierr = MPI_Errhandler_set(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 392 } 393 flg1 = PETSC_FALSE; 394 ierr = PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);CHKERRQ(ierr); 395 if (!flg1) {ierr = PetscPushSignalHandler(PetscDefaultSignalHandler,(void*)0);CHKERRQ(ierr);} 396 flg1 = PETSC_FALSE; 397 ierr = PetscOptionsGetBool(PETSC_NULL,"-fp_trap",&flg1,PETSC_NULL);CHKERRQ(ierr); 398 if (flg1) {ierr = PetscSetFPTrap(PETSC_FP_TRAP_ON);CHKERRQ(ierr);} 399 400 /* 401 Setup debugger information 402 */ 403 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 404 ierr = PetscOptionsGetString(PETSC_NULL,"-on_error_attach_debugger",string,64,&flg1);CHKERRQ(ierr); 405 if (flg1) { 406 MPI_Errhandler err_handler; 407 408 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 409 ierr = MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 410 ierr = MPI_Errhandler_set(comm,err_handler);CHKERRQ(ierr); 411 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);CHKERRQ(ierr); 412 } 413 ierr = PetscOptionsGetString(PETSC_NULL,"-debug_terminal",string,64,&flg1);CHKERRQ(ierr); 414 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 415 ierr = PetscOptionsGetString(PETSC_NULL,"-start_in_debugger",string,64,&flg1);CHKERRQ(ierr); 416 ierr = PetscOptionsGetString(PETSC_NULL,"-stop_for_debugger",string,64,&flg2);CHKERRQ(ierr); 417 if (flg1 || flg2) { 418 PetscMPIInt size; 419 PetscInt lsize,*nodes; 420 MPI_Errhandler err_handler; 421 /* 422 we have to make sure that all processors have opened 423 connections to all other processors, otherwise once the 424 debugger has stated it is likely to receive a SIGUSR1 425 and kill the program. 426 */ 427 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 428 if (size > 2) { 429 PetscMPIInt dummy = 0; 430 MPI_Status status; 431 for (i=0; i<size; i++) { 432 if (rank != i) { 433 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);CHKERRQ(ierr); 434 } 435 } 436 for (i=0; i<size; i++) { 437 if (rank != i) { 438 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);CHKERRQ(ierr); 439 } 440 } 441 } 442 /* check if this processor node should be in debugger */ 443 ierr = PetscMalloc(size*sizeof(PetscInt),&nodes);CHKERRQ(ierr); 444 lsize = size; 445 ierr = PetscOptionsGetIntArray(PETSC_NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 446 if (flag) { 447 for (i=0; i<lsize; i++) { 448 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 449 } 450 } 451 if (!flag) { 452 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 453 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr); 454 if (flg1) { 455 ierr = PetscAttachDebugger();CHKERRQ(ierr); 456 } else { 457 ierr = PetscStopForDebugger();CHKERRQ(ierr); 458 } 459 ierr = MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 460 ierr = MPI_Errhandler_set(comm,err_handler);CHKERRQ(ierr); 461 } 462 ierr = PetscFree(nodes);CHKERRQ(ierr); 463 } 464 465 ierr = PetscOptionsGetString(PETSC_NULL,"-on_error_emacs",emacsmachinename,128,&flg1);CHKERRQ(ierr); 466 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 467 468 #if defined(PETSC_USE_SOCKET_VIEWER) 469 /* 470 Activates new sockets for zope if needed 471 */ 472 ierr = PetscOptionsHasName(PETSC_NULL,"-zope", &flgz);CHKERRQ(ierr); 473 ierr = PetscOptionsHasName(PETSC_NULL,"-nostdout", &flgzout);CHKERRQ(ierr); 474 if (flgz){ 475 int sockfd; 476 char hostname[256]; 477 char username[256]; 478 int remoteport = 9999; 479 480 ierr = PetscOptionsGetString(PETSC_NULL, "-zope", hostname, 256, &flgz);CHKERRQ(ierr); 481 if (!hostname[0]){ 482 ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr); 483 } 484 ierr = PetscOpenSocket(hostname, remoteport, &sockfd);CHKERRQ(ierr); 485 ierr = PetscGetUserName(username, 256);CHKERRQ(ierr); 486 PETSC_ZOPEFD = fdopen(sockfd, "w"); 487 if (flgzout){ 488 PETSC_STDOUT = PETSC_ZOPEFD; 489 fprintf(PETSC_STDOUT, "<<<user>>> %s\n",username); 490 fprintf(PETSC_STDOUT, "<<<start>>>"); 491 } else { 492 fprintf(PETSC_ZOPEFD, "<<<user>>> %s\n",username); 493 fprintf(PETSC_ZOPEFD, "<<<start>>>"); 494 } 495 } 496 #endif 497 #if defined(PETSC_USE_SERVER) 498 ierr = PetscOptionsHasName(PETSC_NULL,"-server", &flgz);CHKERRQ(ierr); 499 if (flgz){ 500 PetscInt port = PETSC_DECIDE; 501 ierr = PetscOptionsGetInt(PETSC_NULL,"-server",&port,PETSC_NULL);CHKERRQ(ierr); 502 ierr = PetscWebServe(PETSC_COMM_WORLD,(int)port);CHKERRQ(ierr); 503 } 504 #endif 505 506 /* 507 Setup profiling and logging 508 */ 509 #if defined (PETSC_USE_INFO) 510 { 511 char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0; 512 ierr = PetscOptionsGetString(PETSC_NULL,"-info",logname,250,&flg1);CHKERRQ(ierr); 513 if (flg1 && logname[0]) { 514 ierr = PetscInfoAllow(PETSC_TRUE,logname);CHKERRQ(ierr); 515 } else if (flg1) { 516 ierr = PetscInfoAllow(PETSC_TRUE,PETSC_NULL);CHKERRQ(ierr); 517 } 518 } 519 #endif 520 #if defined(PETSC_USE_LOG) 521 mname[0] = 0; 522 ierr = PetscOptionsGetString(PETSC_NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 523 if (flg1) { 524 if (mname[0]) { 525 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 526 } else { 527 ierr = PetscOpenHistoryFile(0,&petsc_history);CHKERRQ(ierr); 528 } 529 } 530 #if defined(PETSC_HAVE_MPE) 531 flg1 = PETSC_FALSE; 532 ierr = PetscOptionsHasName(PETSC_NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 533 if (flg1) PetscLogMPEBegin(); 534 #endif 535 flg1 = PETSC_FALSE; 536 flg2 = PETSC_FALSE; 537 flg3 = PETSC_FALSE; 538 ierr = PetscOptionsGetBool(PETSC_NULL,"-log_all",&flg1,PETSC_NULL);CHKERRQ(ierr); 539 ierr = PetscOptionsGetBool(PETSC_NULL,"-log",&flg2,PETSC_NULL);CHKERRQ(ierr); 540 ierr = PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);CHKERRQ(ierr); 541 ierr = PetscOptionsHasName(PETSC_NULL,"-log_summary_python",&flg4);CHKERRQ(ierr); 542 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 543 else if (flg2 || flg3 || flg4) { ierr = PetscLogBegin();CHKERRQ(ierr);} 544 545 ierr = PetscOptionsGetString(PETSC_NULL,"-log_trace",mname,250,&flg1);CHKERRQ(ierr); 546 if (flg1) { 547 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 548 FILE *file; 549 if (mname[0]) { 550 sprintf(name,"%s.%d",mname,rank); 551 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 552 file = fopen(fname,"w"); 553 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 554 } else { 555 file = PETSC_STDOUT; 556 } 557 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 558 } 559 #endif 560 561 /* 562 Setup building of stack frames for all function calls 563 */ 564 #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD) 565 ierr = PetscStackCreate();CHKERRQ(ierr); 566 #endif 567 568 ierr = PetscOptionsGetBool(PETSC_NULL,"-options_gui",&PetscOptionsPublish,PETSC_NULL);CHKERRQ(ierr); 569 570 /* 571 Print basic help message 572 */ 573 ierr = PetscOptionsHasName(PETSC_NULL,"-help",&flg1);CHKERRQ(ierr); 574 if (flg1) { 575 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 576 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 577 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 578 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 579 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 580 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 581 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 582 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 583 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 584 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 585 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 586 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 587 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 588 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 589 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 590 ierr = (*PetscHelpPrintf)(comm," -display display: Location where graphics and debuggers are displayed\n");CHKERRQ(ierr); 591 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 592 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 593 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 594 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 595 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 596 ierr = (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");CHKERRQ(ierr); 597 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");CHKERRQ(ierr); 598 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 599 ierr = (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");CHKERRQ(ierr); 600 ierr = (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");CHKERRQ(ierr); 601 ierr = (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");CHKERRQ(ierr); 602 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 603 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 604 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 605 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 606 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 607 ierr = (*PetscHelpPrintf)(comm," -memory_info: print memory usage at end of run\n");CHKERRQ(ierr); 608 #if defined(PETSC_USE_LOG) 609 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 610 ierr = (*PetscHelpPrintf)(comm," -log[_all _summary]: logging objects and events\n");CHKERRQ(ierr); 611 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 612 #if defined(PETSC_HAVE_MPE) 613 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through upshot\n");CHKERRQ(ierr); 614 #endif 615 ierr = (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");CHKERRQ(ierr); 616 #endif 617 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 618 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 619 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 620 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 621 } 622 623 ierr = PetscOptionsGetReal(PETSC_NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 624 if (flg1) { 625 ierr = PetscSleep(si);CHKERRQ(ierr); 626 } 627 628 ierr = PetscOptionsGetString(PETSC_NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 629 ierr = PetscStrstr(mname,"null",&f);CHKERRQ(ierr); 630 if (f) { 631 ierr = PetscInfoDeactivateClass(PETSC_NULL);CHKERRQ(ierr); 632 } 633 634 #if defined(PETSC_HAVE_CUSP) 635 ierr = PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);CHKERRQ(ierr); 636 if (flg3) flg1 = PETSC_TRUE; 637 else flg1 = PETSC_FALSE; 638 ierr = PetscOptionsGetBool(PETSC_NULL,"-cusp_synchronize",&flg1,PETSC_NULL);CHKERRQ(ierr); 639 if (flg1) synchronizeCUSP = PETSC_TRUE; 640 #endif 641 642 PetscFunctionReturn(0); 643 } 644 645 void* PetscThreadFunc(void* arg) { 646 int ierr; 647 int* pId = (int*)arg; 648 int ThreadId = *pId; 649 650 ierr = pthread_mutex_lock(&job.mutex); 651 job.iNumReadyThreads++; 652 if(job.iNumReadyThreads==PetscMaxThreads) { 653 ierr = pthread_cond_signal(&main_cond); 654 } 655 //the while loop needs to have an exit 656 //the 'main' thread can terminate all the threads by performing a broacast 657 //and calling FuncFinish 658 while(PetscThreadGo) { 659 //need to check the condition to ensure we don't have to wait 660 //waiting when you don't have to causes problems 661 //also need to wait if another thread sneaks in and messes with the predicate 662 while(job.startJob==PETSC_FALSE&&job.iNumJobThreads==0) { 663 //upon entry, automically releases the lock and blocks 664 //upon return, has the lock 665 printf("Thread Blocking!\n"); 666 ierr = pthread_cond_wait(&job.cond,&job.mutex); 667 } 668 job.startJob = PETSC_FALSE; 669 job.iNumJobThreads--; 670 job.iNumReadyThreads--; 671 printf("before JOB PDATA address 0 = %p\n",job.pdata[0]); 672 printf("before JOB PDATA address 1 = %p\n",job.pdata[1]); 673 if(job.pdata==NULL) { 674 job.pfunc(job.pdata); 675 } 676 else { 677 job.pfunc(job.pdata[PetscMaxThreads-job.iNumReadyThreads-1]); 678 } 679 printf("after JOB PDATA address 0 = %p\n",job.pdata[0]); 680 printf("after JOB PDATA address 1 = %p\n",job.pdata[1]); 681 pthread_mutex_unlock(&job.mutex); 682 pthread_barrier_wait(job.pbarr); //ensures all threads are finished 683 printf("Thread %d Got Past The Barrier!\n",ThreadId); 684 //do collection of results? 685 //reset job 686 if(PetscThreadGo) { 687 pthread_mutex_lock(&job.mutex); 688 job.iNumReadyThreads++; 689 if(job.iNumReadyThreads==PetscMaxThreads) { 690 //signal the 'main' thread that the job is done! 691 ierr = pthread_cond_signal(&main_cond); 692 } 693 } 694 } 695 printf("Thread Signing Off!\n"); 696 return NULL; 697 } 698 699 void* PetscThreadInitialize(PetscInt N) { 700 PetscInt i; 701 int status; 702 int* p = &i; 703 printf("In Thread Initialize Function\n"); 704 //allocate memory in the heap for the thread structure 705 PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t)); 706 BarrPoint = (pthread_barrier_t*)malloc((N+1)*sizeof(pthread_barrier_t)); //BarrPoint[0] makes no sense, don't use it! 707 for(i=0; i<N; i++) { 708 status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,p); 709 //error check to ensure proper thread creation 710 status = pthread_barrier_init(&BarrPoint[i+1],NULL,i+1); 711 //error check 712 } 713 return NULL; 714 } 715 716 PetscErrorCode PetscThreadFinalize() { 717 int i,ierr; 718 void* jstatus; 719 720 PetscFunctionBegin; 721 printf("In Thread Finalize Function\n"); 722 MainWait(); //guarantee that all threads are ready to go 723 printf("Main Thread Done Waiting!\n"); 724 MainJob(FuncFinish,NULL,&BarrPoint[PetscMaxThreads],PetscMaxThreads); //set up job and broadcast work 725 printf("Main Thread Done With Finish Function!\n"); 726 //join the threads 727 for(i=0; i<PetscMaxThreads; i++) { 728 ierr = pthread_join(PetscThreadPoint[i],&jstatus); 729 //do error checking 730 } 731 printf("Main Thread Done With Joins!\n"); 732 free(BarrPoint); 733 free(PetscThreadPoint); 734 PetscFunctionReturn(0); 735 } 736 737 void MainWait() { 738 int ierr; 739 740 ierr = pthread_mutex_lock(&job.mutex); 741 while(job.iNumReadyThreads<PetscMaxThreads||job.startJob==PETSC_TRUE) { 742 ierr = pthread_cond_wait(&main_cond,&job.mutex); 743 } 744 //ierr = pthread_mutex_unlock(&job.mutex); 745 } 746 747 void MainJob(void* (*pFunc)(void*),void** data,pthread_barrier_t* barr,PetscInt n) { 748 int ierr; 749 750 //ierr = pthread_mutex_lock(&job.mutex); 751 printf("In Job, before, data[0] = %p\n",data[0]); 752 printf("In Job, before, data[1] = %p\n",data[1]); 753 job.pfunc = pFunc; 754 job.pdata = data; 755 printf("In Job, after, job.pdata[0] = %p\n",job.pdata[0]); 756 printf("In Job, after, job.pdata[1] = %p\n",job.pdata[1]); 757 job.pbarr = barr; 758 job.iNumJobThreads = n; 759 job.startJob = PETSC_TRUE; 760 ierr = pthread_cond_broadcast(&job.cond); 761 ierr = pthread_mutex_unlock(&job.mutex); 762 } 763 764 void* FuncFinish(void* arg) { 765 printf("In Thread Finish Function\n"); 766 PetscThreadGo = PETSC_FALSE; 767 return NULL; 768 } 769