1 2 /* 3 4 This file defines part of the initialization of PETSc 5 6 This file uses regular malloc and free because it cannot know 7 what malloc is being used until it has already processed the input. 8 */ 9 10 #include <petscsys.h> /*I "petscsys.h" I*/ 11 #include <pthread.h> 12 #if defined(PETSC_HAVE_STDLIB_H) 13 #include <stdlib.h> 14 #endif 15 #if defined(PETSC_HAVE_MALLOC_H) 16 #include <malloc.h> 17 #endif 18 #if defined(PETSC_HAVE_VALGRIND) 19 #include <valgrind/valgrind.h> 20 #endif 21 22 /* ------------------------Nasty global variables -------------------------------*/ 23 /* 24 Indicates if PETSc started up MPI, or it was 25 already started before PETSc was initialized. 26 */ 27 PetscBool PetscBeganMPI = PETSC_FALSE; 28 PetscBool PetscInitializeCalled = PETSC_FALSE; 29 PetscBool PetscFinalizeCalled = PETSC_FALSE; 30 PetscBool PetscUseThreadPool = PETSC_FALSE; 31 PetscBool PetscThreadGo = PETSC_TRUE; 32 PetscMPIInt PetscGlobalRank = -1; 33 PetscMPIInt PetscGlobalSize = -1; 34 PetscMPIInt PetscMaxThreads = 2; 35 pthread_t* PetscThreadPoint; 36 pthread_barrier_t* BarrPoint; 37 int* pVal; 38 39 typedef struct { 40 pthread_mutex_t mutex; 41 pthread_cond_t cond; 42 void* (*pfunc)(void*); 43 void** pdata; 44 pthread_barrier_t* pbarr; 45 int iNumJobThreads; 46 int iNumReadyThreads; 47 PetscBool startJob; 48 } sjob; 49 sjob job = {PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER,NULL,NULL,NULL,0,0,PETSC_FALSE}; 50 51 pthread_cond_t main_cond = PTHREAD_COND_INITIALIZER; 52 53 void* PetscThreadFunc(void*); 54 void* PetscThreadInitialize(PetscInt); 55 PetscErrorCode PetscThreadFinalize(void); 56 void MainWait(void); 57 void MainJob(void* (*pFunc)(void*),void**,pthread_barrier_t*,PetscInt); 58 void* FuncFinish(void*); 59 60 #if defined(PETSC_USE_COMPLEX) 61 #if defined(PETSC_COMPLEX_INSTANTIATE) 62 template <> class std::complex<double>; /* instantiate complex template class */ 63 #endif 64 #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX) 65 MPI_Datatype MPI_C_DOUBLE_COMPLEX; 66 MPI_Datatype MPI_C_COMPLEX; 67 #endif 68 PetscScalar PETSC_i; 69 #else 70 PetscScalar PETSC_i = 0.0; 71 #endif 72 #if defined(PETSC_USE_REAL___FLOAT128) 73 MPI_Datatype MPIU___FLOAT128 = 0; 74 #endif 75 MPI_Datatype MPIU_2SCALAR = 0; 76 MPI_Datatype MPIU_2INT = 0; 77 78 /* 79 These are needed by petscbt.h 80 */ 81 #include <petscbt.h> 82 char _BT_mask = ' '; 83 char _BT_c = ' '; 84 PetscInt _BT_idx = 0; 85 86 /* 87 Function that is called to display all error messages 88 */ 89 PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault; 90 PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault; 91 #if defined(PETSC_HAVE_MATLAB_ENGINE) 92 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintf_Matlab; 93 #else 94 PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault; 95 #endif 96 /* 97 This is needed to turn on/off cusp synchronization */ 98 PetscBool synchronizeCUSP = PETSC_FALSE; 99 100 /* ------------------------------------------------------------------------------*/ 101 /* 102 Optional file where all PETSc output from various prints is saved 103 */ 104 FILE *petsc_history = PETSC_NULL; 105 106 #undef __FUNCT__ 107 #define __FUNCT__ "PetscOpenHistoryFile" 108 PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd) 109 { 110 PetscErrorCode ierr; 111 PetscMPIInt rank,size; 112 char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64]; 113 char version[256]; 114 115 PetscFunctionBegin; 116 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 117 if (!rank) { 118 char arch[10]; 119 int err; 120 PetscViewer viewer; 121 122 ierr = PetscGetArchType(arch,10);CHKERRQ(ierr); 123 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 124 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 125 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 126 if (filename) { 127 ierr = PetscFixFilename(filename,fname);CHKERRQ(ierr); 128 } else { 129 ierr = PetscGetHomeDirectory(pfile,240);CHKERRQ(ierr); 130 ierr = PetscStrcat(pfile,"/.petschistory");CHKERRQ(ierr); 131 ierr = PetscFixFilename(pfile,fname);CHKERRQ(ierr); 132 } 133 134 *fd = fopen(fname,"a"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname); 135 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 136 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);CHKERRQ(ierr); 137 ierr = PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr); 138 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);CHKERRQ(ierr); 139 ierr = PetscViewerASCIIOpenWithFILE(PETSC_COMM_WORLD,*fd,&viewer);CHKERRQ(ierr); 140 ierr = PetscOptionsView(viewer);CHKERRQ(ierr); 141 ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); 142 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 143 err = fflush(*fd); 144 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 145 } 146 PetscFunctionReturn(0); 147 } 148 149 #undef __FUNCT__ 150 #define __FUNCT__ "PetscCloseHistoryFile" 151 PetscErrorCode PetscCloseHistoryFile(FILE **fd) 152 { 153 PetscErrorCode ierr; 154 PetscMPIInt rank; 155 char date[64]; 156 int err; 157 158 PetscFunctionBegin; 159 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 160 if (!rank) { 161 ierr = PetscGetDate(date,64);CHKERRQ(ierr); 162 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 163 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);CHKERRQ(ierr); 164 ierr = PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");CHKERRQ(ierr); 165 err = fflush(*fd); 166 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file"); 167 err = fclose(*fd); 168 if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file"); 169 } 170 PetscFunctionReturn(0); 171 } 172 173 /* ------------------------------------------------------------------------------*/ 174 175 /* 176 This is ugly and probably belongs somewhere else, but I want to 177 be able to put a true MPI abort error handler with command line args. 178 179 This is so MPI errors in the debugger will leave all the stack 180 frames. The default MP_Abort() cleans up and exits thus providing no useful information 181 in the debugger hence we call abort() instead of MPI_Abort(). 182 */ 183 184 #undef __FUNCT__ 185 #define __FUNCT__ "Petsc_MPI_AbortOnError" 186 void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag) 187 { 188 PetscFunctionBegin; 189 (*PetscErrorPrintf)("MPI error %d\n",*flag); 190 abort(); 191 } 192 193 #undef __FUNCT__ 194 #define __FUNCT__ "Petsc_MPI_DebuggerOnError" 195 void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag) 196 { 197 PetscErrorCode ierr; 198 199 PetscFunctionBegin; 200 (*PetscErrorPrintf)("MPI error %d\n",*flag); 201 ierr = PetscAttachDebugger(); 202 if (ierr) { /* hopeless so get out */ 203 MPI_Abort(*comm,*flag); 204 } 205 } 206 207 #undef __FUNCT__ 208 #define __FUNCT__ "PetscEnd" 209 /*@C 210 PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 211 wishes a clean exit somewhere deep in the program. 212 213 Collective on PETSC_COMM_WORLD 214 215 Options Database Keys are the same as for PetscFinalize() 216 217 Level: advanced 218 219 Note: 220 See PetscInitialize() for more general runtime options. 221 222 .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize() 223 @*/ 224 PetscErrorCode PetscEnd(void) 225 { 226 PetscFunctionBegin; 227 PetscFinalize(); 228 exit(0); 229 return 0; 230 } 231 232 PetscBool PetscOptionsPublish = PETSC_FALSE; 233 extern PetscErrorCode PetscSetUseTrMalloc_Private(void); 234 extern PetscBool petscsetmallocvisited; 235 static char emacsmachinename[256]; 236 237 PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0; 238 PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0; 239 240 #undef __FUNCT__ 241 #define __FUNCT__ "PetscSetHelpVersionFunctions" 242 /*@C 243 PetscSetHelpVersionFunctions - Sets functions that print help and version information 244 before the PETSc help and version information is printed. Must call BEFORE PetscInitialize(). 245 This routine enables a "higher-level" package that uses PETSc to print its messages first. 246 247 Input Parameter: 248 + help - the help function (may be PETSC_NULL) 249 - version - the version function (may be PETSC_NULL) 250 251 Level: developer 252 253 Concepts: package help message 254 255 @*/ 256 PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm)) 257 { 258 PetscFunctionBegin; 259 PetscExternalHelpFunction = help; 260 PetscExternalVersionFunction = version; 261 PetscFunctionReturn(0); 262 } 263 264 #undef __FUNCT__ 265 #define __FUNCT__ "PetscOptionsCheckInitial_Private" 266 PetscErrorCode PetscOptionsCheckInitial_Private(void) 267 { 268 char string[64],mname[PETSC_MAX_PATH_LEN],*f; 269 MPI_Comm comm = PETSC_COMM_WORLD; 270 PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flg4 = PETSC_FALSE,flag,flgz,flgzout; 271 PetscErrorCode ierr; 272 PetscReal si; 273 int i; 274 PetscMPIInt rank; 275 char version[256]; 276 277 PetscFunctionBegin; 278 ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); 279 280 /* 281 Setup the memory management; support for tracing malloc() usage 282 */ 283 ierr = PetscOptionsHasName(PETSC_NULL,"-malloc_log",&flg3);CHKERRQ(ierr); 284 #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD) 285 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg1,&flg2);CHKERRQ(ierr); 286 if ((!flg2 || flg1) && !petscsetmallocvisited) { 287 #if defined(PETSC_HAVE_VALGRIND) 288 if (flg2 || !(RUNNING_ON_VALGRIND)) { 289 /* turn off default -malloc if valgrind is being used */ 290 #endif 291 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 292 #if defined(PETSC_HAVE_VALGRIND) 293 } 294 #endif 295 } 296 #else 297 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);CHKERRQ(ierr); 298 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg2,PETSC_NULL);CHKERRQ(ierr); 299 if (flg1 || flg2 || flg3) {ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr);} 300 #endif 301 if (flg3) { 302 ierr = PetscMallocSetDumpLog();CHKERRQ(ierr); 303 } 304 flg1 = PETSC_FALSE; 305 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc_debug",&flg1,PETSC_NULL);CHKERRQ(ierr); 306 if (flg1) { 307 ierr = PetscSetUseTrMalloc_Private();CHKERRQ(ierr); 308 ierr = PetscMallocDebug(PETSC_TRUE);CHKERRQ(ierr); 309 } 310 311 flg1 = PETSC_FALSE; 312 ierr = PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg1,PETSC_NULL);CHKERRQ(ierr); 313 if (!flg1) { 314 flg1 = PETSC_FALSE; 315 ierr = PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg1,PETSC_NULL);CHKERRQ(ierr); 316 } 317 if (flg1) { 318 ierr = PetscMemorySetGetMaximumUsage();CHKERRQ(ierr); 319 } 320 321 /* 322 Set the display variable for graphics 323 */ 324 ierr = PetscSetDisplay();CHKERRQ(ierr); 325 326 /* 327 Determine whether user specified maximum number of threads 328 */ 329 ierr = PetscOptionsHasName(PETSC_NULL,"-thread_max",&flg1);CHKERRQ(ierr); 330 if(flg1) { 331 ierr = PetscOptionsGetInt(PETSC_NULL,"-thread_max",&PetscMaxThreads,PETSC_NULL);CHKERRQ(ierr); 332 } 333 334 /* 335 Determine whether to use thread pool 336 */ 337 ierr = PetscOptionsHasName(PETSC_NULL,"-use_thread_pool",&flg1);CHKERRQ(ierr); 338 if(flg1) { 339 PetscUseThreadPool = PETSC_TRUE; 340 PetscThreadInitialize(PetscMaxThreads); 341 } 342 343 /* 344 Print the PETSc version information 345 */ 346 ierr = PetscOptionsHasName(PETSC_NULL,"-v",&flg1);CHKERRQ(ierr); 347 ierr = PetscOptionsHasName(PETSC_NULL,"-version",&flg2);CHKERRQ(ierr); 348 ierr = PetscOptionsHasName(PETSC_NULL,"-help",&flg3);CHKERRQ(ierr); 349 if (flg1 || flg2 || flg3){ 350 351 /* 352 Print "higher-level" package version message 353 */ 354 if (PetscExternalVersionFunction) { 355 ierr = (*PetscExternalVersionFunction)(comm);CHKERRQ(ierr); 356 } 357 358 ierr = PetscGetVersion(version,256);CHKERRQ(ierr); 359 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------\ 360 ------------------------------\n");CHKERRQ(ierr); 361 ierr = (*PetscHelpPrintf)(comm,"%s\n",version);CHKERRQ(ierr); 362 ierr = (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);CHKERRQ(ierr); 363 ierr = (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");CHKERRQ(ierr); 364 ierr = (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");CHKERRQ(ierr); 365 ierr = (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");CHKERRQ(ierr); 366 ierr = (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);CHKERRQ(ierr); 367 ierr = (*PetscHelpPrintf)(comm,"--------------------------------------------\ 368 ------------------------------\n");CHKERRQ(ierr); 369 } 370 371 /* 372 Print "higher-level" package help message 373 */ 374 if (flg3){ 375 if (PetscExternalHelpFunction) { 376 ierr = (*PetscExternalHelpFunction)(comm);CHKERRQ(ierr); 377 } 378 } 379 380 /* 381 Setup the error handling 382 */ 383 flg1 = PETSC_FALSE; 384 ierr = PetscOptionsGetBool(PETSC_NULL,"-on_error_abort",&flg1,PETSC_NULL);CHKERRQ(ierr); 385 if (flg1) { ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr);} 386 flg1 = PETSC_FALSE; 387 ierr = PetscOptionsGetBool(PETSC_NULL,"-on_error_mpiabort",&flg1,PETSC_NULL);CHKERRQ(ierr); 388 if (flg1) { ierr = PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);CHKERRQ(ierr);} 389 flg1 = PETSC_FALSE; 390 ierr = PetscOptionsGetBool(PETSC_NULL,"-mpi_return_on_error",&flg1,PETSC_NULL);CHKERRQ(ierr); 391 if (flg1) { 392 ierr = MPI_Errhandler_set(comm,MPI_ERRORS_RETURN);CHKERRQ(ierr); 393 } 394 flg1 = PETSC_FALSE; 395 ierr = PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);CHKERRQ(ierr); 396 if (!flg1) {ierr = PetscPushSignalHandler(PetscDefaultSignalHandler,(void*)0);CHKERRQ(ierr);} 397 flg1 = PETSC_FALSE; 398 ierr = PetscOptionsGetBool(PETSC_NULL,"-fp_trap",&flg1,PETSC_NULL);CHKERRQ(ierr); 399 if (flg1) {ierr = PetscSetFPTrap(PETSC_FP_TRAP_ON);CHKERRQ(ierr);} 400 401 /* 402 Setup debugger information 403 */ 404 ierr = PetscSetDefaultDebugger();CHKERRQ(ierr); 405 ierr = PetscOptionsGetString(PETSC_NULL,"-on_error_attach_debugger",string,64,&flg1);CHKERRQ(ierr); 406 if (flg1) { 407 MPI_Errhandler err_handler; 408 409 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 410 ierr = MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);CHKERRQ(ierr); 411 ierr = MPI_Errhandler_set(comm,err_handler);CHKERRQ(ierr); 412 ierr = PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);CHKERRQ(ierr); 413 } 414 ierr = PetscOptionsGetString(PETSC_NULL,"-debug_terminal",string,64,&flg1);CHKERRQ(ierr); 415 if (flg1) { ierr = PetscSetDebugTerminal(string);CHKERRQ(ierr); } 416 ierr = PetscOptionsGetString(PETSC_NULL,"-start_in_debugger",string,64,&flg1);CHKERRQ(ierr); 417 ierr = PetscOptionsGetString(PETSC_NULL,"-stop_for_debugger",string,64,&flg2);CHKERRQ(ierr); 418 if (flg1 || flg2) { 419 PetscMPIInt size; 420 PetscInt lsize,*nodes; 421 MPI_Errhandler err_handler; 422 /* 423 we have to make sure that all processors have opened 424 connections to all other processors, otherwise once the 425 debugger has stated it is likely to receive a SIGUSR1 426 and kill the program. 427 */ 428 ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); 429 if (size > 2) { 430 PetscMPIInt dummy = 0; 431 MPI_Status status; 432 for (i=0; i<size; i++) { 433 if (rank != i) { 434 ierr = MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);CHKERRQ(ierr); 435 } 436 } 437 for (i=0; i<size; i++) { 438 if (rank != i) { 439 ierr = MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);CHKERRQ(ierr); 440 } 441 } 442 } 443 /* check if this processor node should be in debugger */ 444 ierr = PetscMalloc(size*sizeof(PetscInt),&nodes);CHKERRQ(ierr); 445 lsize = size; 446 ierr = PetscOptionsGetIntArray(PETSC_NULL,"-debugger_nodes",nodes,&lsize,&flag);CHKERRQ(ierr); 447 if (flag) { 448 for (i=0; i<lsize; i++) { 449 if (nodes[i] == rank) { flag = PETSC_FALSE; break; } 450 } 451 } 452 if (!flag) { 453 ierr = PetscSetDebuggerFromString(string);CHKERRQ(ierr); 454 ierr = PetscPushErrorHandler(PetscAbortErrorHandler,0);CHKERRQ(ierr); 455 if (flg1) { 456 ierr = PetscAttachDebugger();CHKERRQ(ierr); 457 } else { 458 ierr = PetscStopForDebugger();CHKERRQ(ierr); 459 } 460 ierr = MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);CHKERRQ(ierr); 461 ierr = MPI_Errhandler_set(comm,err_handler);CHKERRQ(ierr); 462 } 463 ierr = PetscFree(nodes);CHKERRQ(ierr); 464 } 465 466 ierr = PetscOptionsGetString(PETSC_NULL,"-on_error_emacs",emacsmachinename,128,&flg1);CHKERRQ(ierr); 467 if (flg1 && !rank) {ierr = PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);CHKERRQ(ierr);} 468 469 #if defined(PETSC_USE_SOCKET_VIEWER) 470 /* 471 Activates new sockets for zope if needed 472 */ 473 ierr = PetscOptionsHasName(PETSC_NULL,"-zope", &flgz);CHKERRQ(ierr); 474 ierr = PetscOptionsHasName(PETSC_NULL,"-nostdout", &flgzout);CHKERRQ(ierr); 475 if (flgz){ 476 int sockfd; 477 char hostname[256]; 478 char username[256]; 479 int remoteport = 9999; 480 481 ierr = PetscOptionsGetString(PETSC_NULL, "-zope", hostname, 256, &flgz);CHKERRQ(ierr); 482 if (!hostname[0]){ 483 ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr); 484 } 485 ierr = PetscOpenSocket(hostname, remoteport, &sockfd);CHKERRQ(ierr); 486 ierr = PetscGetUserName(username, 256);CHKERRQ(ierr); 487 PETSC_ZOPEFD = fdopen(sockfd, "w"); 488 if (flgzout){ 489 PETSC_STDOUT = PETSC_ZOPEFD; 490 fprintf(PETSC_STDOUT, "<<<user>>> %s\n",username); 491 fprintf(PETSC_STDOUT, "<<<start>>>"); 492 } else { 493 fprintf(PETSC_ZOPEFD, "<<<user>>> %s\n",username); 494 fprintf(PETSC_ZOPEFD, "<<<start>>>"); 495 } 496 } 497 #endif 498 #if defined(PETSC_USE_SERVER) 499 ierr = PetscOptionsHasName(PETSC_NULL,"-server", &flgz);CHKERRQ(ierr); 500 if (flgz){ 501 PetscInt port = PETSC_DECIDE; 502 ierr = PetscOptionsGetInt(PETSC_NULL,"-server",&port,PETSC_NULL);CHKERRQ(ierr); 503 ierr = PetscWebServe(PETSC_COMM_WORLD,(int)port);CHKERRQ(ierr); 504 } 505 #endif 506 507 /* 508 Setup profiling and logging 509 */ 510 #if defined (PETSC_USE_INFO) 511 { 512 char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0; 513 ierr = PetscOptionsGetString(PETSC_NULL,"-info",logname,250,&flg1);CHKERRQ(ierr); 514 if (flg1 && logname[0]) { 515 ierr = PetscInfoAllow(PETSC_TRUE,logname);CHKERRQ(ierr); 516 } else if (flg1) { 517 ierr = PetscInfoAllow(PETSC_TRUE,PETSC_NULL);CHKERRQ(ierr); 518 } 519 } 520 #endif 521 #if defined(PETSC_USE_LOG) 522 mname[0] = 0; 523 ierr = PetscOptionsGetString(PETSC_NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 524 if (flg1) { 525 if (mname[0]) { 526 ierr = PetscOpenHistoryFile(mname,&petsc_history);CHKERRQ(ierr); 527 } else { 528 ierr = PetscOpenHistoryFile(0,&petsc_history);CHKERRQ(ierr); 529 } 530 } 531 #if defined(PETSC_HAVE_MPE) 532 flg1 = PETSC_FALSE; 533 ierr = PetscOptionsHasName(PETSC_NULL,"-log_mpe",&flg1);CHKERRQ(ierr); 534 if (flg1) PetscLogMPEBegin(); 535 #endif 536 flg1 = PETSC_FALSE; 537 flg2 = PETSC_FALSE; 538 flg3 = PETSC_FALSE; 539 ierr = PetscOptionsGetBool(PETSC_NULL,"-log_all",&flg1,PETSC_NULL);CHKERRQ(ierr); 540 ierr = PetscOptionsGetBool(PETSC_NULL,"-log",&flg2,PETSC_NULL);CHKERRQ(ierr); 541 ierr = PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);CHKERRQ(ierr); 542 ierr = PetscOptionsHasName(PETSC_NULL,"-log_summary_python",&flg4);CHKERRQ(ierr); 543 if (flg1) { ierr = PetscLogAllBegin();CHKERRQ(ierr); } 544 else if (flg2 || flg3 || flg4) { ierr = PetscLogBegin();CHKERRQ(ierr);} 545 546 ierr = PetscOptionsGetString(PETSC_NULL,"-log_trace",mname,250,&flg1);CHKERRQ(ierr); 547 if (flg1) { 548 char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN]; 549 FILE *file; 550 if (mname[0]) { 551 sprintf(name,"%s.%d",mname,rank); 552 ierr = PetscFixFilename(name,fname);CHKERRQ(ierr); 553 file = fopen(fname,"w"); 554 if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname); 555 } else { 556 file = PETSC_STDOUT; 557 } 558 ierr = PetscLogTraceBegin(file);CHKERRQ(ierr); 559 } 560 #endif 561 562 /* 563 Setup building of stack frames for all function calls 564 */ 565 #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD) 566 ierr = PetscStackCreate();CHKERRQ(ierr); 567 #endif 568 569 ierr = PetscOptionsGetBool(PETSC_NULL,"-options_gui",&PetscOptionsPublish,PETSC_NULL);CHKERRQ(ierr); 570 571 /* 572 Print basic help message 573 */ 574 ierr = PetscOptionsHasName(PETSC_NULL,"-help",&flg1);CHKERRQ(ierr); 575 if (flg1) { 576 ierr = (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");CHKERRQ(ierr); 577 ierr = (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");CHKERRQ(ierr); 578 ierr = (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");CHKERRQ(ierr); 579 ierr = (*PetscHelpPrintf)(comm," only when run in the debugger\n");CHKERRQ(ierr); 580 ierr = (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 581 ierr = (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");CHKERRQ(ierr); 582 ierr = (*PetscHelpPrintf)(comm," unless noxterm is given\n");CHKERRQ(ierr); 583 ierr = (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");CHKERRQ(ierr); 584 ierr = (*PetscHelpPrintf)(comm," start all processes in the debugger\n");CHKERRQ(ierr); 585 ierr = (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");CHKERRQ(ierr); 586 ierr = (*PetscHelpPrintf)(comm," emacs jumps to error file\n");CHKERRQ(ierr); 587 ierr = (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");CHKERRQ(ierr); 588 ierr = (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");CHKERRQ(ierr); 589 ierr = (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");CHKERRQ(ierr); 590 ierr = (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");CHKERRQ(ierr); 591 ierr = (*PetscHelpPrintf)(comm," -display display: Location where graphics and debuggers are displayed\n");CHKERRQ(ierr); 592 ierr = (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");CHKERRQ(ierr); 593 ierr = (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");CHKERRQ(ierr); 594 ierr = (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");CHKERRQ(ierr); 595 ierr = (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");CHKERRQ(ierr); 596 ierr = (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");CHKERRQ(ierr); 597 ierr = (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");CHKERRQ(ierr); 598 ierr = (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");CHKERRQ(ierr); 599 ierr = (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");CHKERRQ(ierr); 600 ierr = (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");CHKERRQ(ierr); 601 ierr = (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");CHKERRQ(ierr); 602 ierr = (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");CHKERRQ(ierr); 603 ierr = (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");CHKERRQ(ierr); 604 ierr = (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");CHKERRQ(ierr); 605 ierr = (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");CHKERRQ(ierr); 606 ierr = (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");CHKERRQ(ierr); 607 ierr = (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");CHKERRQ(ierr); 608 ierr = (*PetscHelpPrintf)(comm," -memory_info: print memory usage at end of run\n");CHKERRQ(ierr); 609 #if defined(PETSC_USE_LOG) 610 ierr = (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");CHKERRQ(ierr); 611 ierr = (*PetscHelpPrintf)(comm," -log[_all _summary]: logging objects and events\n");CHKERRQ(ierr); 612 ierr = (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");CHKERRQ(ierr); 613 #if defined(PETSC_HAVE_MPE) 614 ierr = (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through upshot\n");CHKERRQ(ierr); 615 #endif 616 ierr = (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");CHKERRQ(ierr); 617 #endif 618 ierr = (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");CHKERRQ(ierr); 619 ierr = (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");CHKERRQ(ierr); 620 ierr = (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");CHKERRQ(ierr); 621 ierr = (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");CHKERRQ(ierr); 622 } 623 624 ierr = PetscOptionsGetReal(PETSC_NULL,"-petsc_sleep",&si,&flg1);CHKERRQ(ierr); 625 if (flg1) { 626 ierr = PetscSleep(si);CHKERRQ(ierr); 627 } 628 629 ierr = PetscOptionsGetString(PETSC_NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);CHKERRQ(ierr); 630 ierr = PetscStrstr(mname,"null",&f);CHKERRQ(ierr); 631 if (f) { 632 ierr = PetscInfoDeactivateClass(PETSC_NULL);CHKERRQ(ierr); 633 } 634 635 #if defined(PETSC_HAVE_CUSP) 636 ierr = PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);CHKERRQ(ierr); 637 if (flg3) flg1 = PETSC_TRUE; 638 else flg1 = PETSC_FALSE; 639 ierr = PetscOptionsGetBool(PETSC_NULL,"-cusp_synchronize",&flg1,PETSC_NULL);CHKERRQ(ierr); 640 if (flg1) synchronizeCUSP = PETSC_TRUE; 641 #endif 642 643 PetscFunctionReturn(0); 644 } 645 646 void* PetscThreadFunc(void* arg) { 647 int ierr; 648 int* pId = (int*)arg; 649 int ThreadId = *pId; 650 651 ierr = pthread_mutex_lock(&job.mutex); 652 job.iNumReadyThreads++; 653 if(job.iNumReadyThreads==PetscMaxThreads) { 654 ierr = pthread_cond_signal(&main_cond); 655 } 656 //the while loop needs to have an exit 657 //the 'main' thread can terminate all the threads by performing a broacast 658 //and calling FuncFinish 659 while(PetscThreadGo) { 660 //need to check the condition to ensure we don't have to wait 661 //waiting when you don't have to causes problems 662 //also need to wait if another thread sneaks in and messes with the predicate 663 while(job.startJob==PETSC_FALSE&&job.iNumJobThreads==0) { 664 //upon entry, automically releases the lock and blocks 665 //upon return, has the lock 666 printf("Thread Blocking!\n"); 667 ierr = pthread_cond_wait(&job.cond,&job.mutex); 668 } 669 job.startJob = PETSC_FALSE; 670 job.iNumJobThreads--; 671 job.iNumReadyThreads--; 672 if(job.pdata==NULL) { 673 job.pfunc(job.pdata); 674 } 675 else { 676 job.pfunc(job.pdata[PetscMaxThreads-job.iNumReadyThreads-1]); 677 } 678 pthread_mutex_unlock(&job.mutex); 679 pthread_barrier_wait(job.pbarr); //ensures all threads are finished 680 printf("Thread %d Got Past The Barrier!\n",ThreadId); 681 //do collection of results? 682 //reset job 683 if(PetscThreadGo) { 684 pthread_mutex_lock(&job.mutex); 685 job.iNumReadyThreads++; 686 if(job.iNumReadyThreads==PetscMaxThreads) { 687 //signal the 'main' thread that the job is done! 688 ierr = pthread_cond_signal(&main_cond); 689 } 690 } 691 } 692 printf("Thread Signing Off!\n"); 693 return NULL; 694 } 695 696 #undef __FUNCT__ 697 #define __FUNCT__ "PetscThreadInitialize" 698 void* PetscThreadInitialize(PetscInt N) { 699 PetscInt i; 700 int status; 701 pVal = (int*)malloc(N*sizeof(int)); 702 printf("In Thread Initialize Function\n"); 703 //allocate memory in the heap for the thread structure 704 PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t)); 705 BarrPoint = (pthread_barrier_t*)malloc((N+1)*sizeof(pthread_barrier_t)); //BarrPoint[0] makes no sense, don't use it! 706 job.pdata = (void**)malloc(N*sizeof(void*)); 707 for(i=0; i<N; i++) { 708 pVal[i] = i; 709 status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]); 710 //error check to ensure proper thread creation 711 status = pthread_barrier_init(&BarrPoint[i+1],NULL,i+1); 712 //error check 713 } 714 return NULL; 715 } 716 717 718 #undef __FUNCT__ 719 #define __FUNCT__ "PetscThreadFinalize" 720 PetscErrorCode PetscThreadFinalize() { 721 int i,ierr; 722 void* jstatus; 723 724 PetscFunctionBegin; 725 printf("In Thread Finalize Function\n"); 726 MainWait(); //guarantee that all threads are ready to go 727 printf("Main Thread Done Waiting!\n"); 728 MainJob(FuncFinish,NULL,&BarrPoint[PetscMaxThreads],PetscMaxThreads); //set up job and broadcast work 729 printf("Main Thread Done With Finish Function!\n"); 730 //join the threads 731 for(i=0; i<PetscMaxThreads; i++) { 732 ierr = pthread_join(PetscThreadPoint[i],&jstatus); 733 //do error checking 734 } 735 printf("Main Thread Done With Joins!\n"); 736 free(BarrPoint); 737 free(PetscThreadPoint); 738 PetscFunctionReturn(0); 739 } 740 741 #undef __FUNCT__ 742 #define __FUNCT__ "MainWait" 743 void MainWait() { 744 int ierr; 745 746 ierr = pthread_mutex_lock(&job.mutex); 747 while(job.iNumReadyThreads<PetscMaxThreads||job.startJob==PETSC_TRUE) { 748 ierr = pthread_cond_wait(&main_cond,&job.mutex); 749 } 750 ierr = pthread_mutex_unlock(&job.mutex); 751 } 752 753 #undef __FUNCT__ 754 #define __FUNCT__ "MainJob" 755 void MainJob(void* (*pFunc)(void*),void** data,pthread_barrier_t* barr,PetscInt n) { 756 int ierr; 757 758 ierr = pthread_mutex_lock(&job.mutex); 759 job.pfunc = pFunc; 760 job.pdata = data; 761 job.pbarr = barr; 762 job.iNumJobThreads = n; 763 job.startJob = PETSC_TRUE; 764 ierr = pthread_cond_broadcast(&job.cond); 765 ierr = pthread_mutex_unlock(&job.mutex); 766 } 767 768 void* FuncFinish(void* arg) { 769 printf("In Thread Finish Function\n"); 770 PetscThreadGo = PETSC_FALSE; 771 return NULL; 772 } 773