1 2 /* 3 Code that allows a user to dictate what malloc() PETSc uses. 4 */ 5 #include <petscsys.h> /*I "petscsys.h" I*/ 6 #if defined(PETSC_HAVE_MALLOC_H) 7 #include <malloc.h> 8 #endif 9 #if defined(PETSC_HAVE_MEMKIND) 10 #include <memkind.h> 11 typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType; 12 PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED; 13 PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED; 14 #endif 15 /* 16 We want to make sure that all mallocs of double or complex numbers are complex aligned. 17 1) on systems with memalign() we call that routine to get an aligned memory location 18 2) on systems without memalign() we 19 - allocate one sizeof(PetscScalar) extra space 20 - we shift the pointer up slightly if needed to get PetscScalar aligned 21 - if shifted we store at ptr[-1] the amount of shift (plus a classid) 22 */ 23 #define SHIFT_CLASSID 456123 24 25 PetscErrorCode PetscMallocAlign(size_t mem,int line,const char func[],const char file[],void **result) 26 { 27 if (!mem) { *result = NULL; return 0; } 28 #if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8) 29 # if defined(PETSC_HAVE_MEMKIND) 30 { 31 int ierr; 32 if (!currentmktype) ierr = memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem); 33 else ierr = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem); 34 if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memory requested with memkind %.0f",(PetscLogDouble)mem); 35 } 36 # else 37 *result = malloc(mem); 38 # endif 39 #elif defined(PETSC_HAVE_MEMALIGN) 40 # if defined(PETSC_HAVE_MEMKIND) 41 { 42 int ierr; 43 if (!currentmktype) ierr = memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem); 44 else ierr = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem); 45 if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memory requested with memkind %.0f",(PetscLogDouble)mem); 46 } 47 # else 48 *result = memalign(PETSC_MEMALIGN,mem); 49 # endif 50 #else 51 { 52 /* 53 malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned 54 */ 55 # if defined(PETSC_HAVE_MEMKIND) 56 int *ptr,ierr; 57 if (!currentmktype) ierr = memkind_posix_memalign(MEMKIND_DEFAULT,&ptr,PETSC_MEMALIGN,mem+2*PETSC_MEMALIGN); 58 else ierr = memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&ptr,PETSC_MEMALIGN,mem+2*PETSC_MEMALIGN); 59 if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memory requested with memkind %.0f",(PetscLogDouble)(mem+2*PETSC_MEMALIGN)); 60 # else 61 int *ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN); 62 # endif 63 if (ptr) { 64 int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN); 65 shift = (2*PETSC_MEMALIGN - shift)/sizeof(int); 66 ptr[shift-1] = shift + SHIFT_CLASSID; 67 ptr += shift; 68 *result = (void*)ptr; 69 } else { 70 *result = NULL; 71 } 72 } 73 #endif 74 if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem); 75 return 0; 76 } 77 78 PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[]) 79 { 80 if (!ptr) return 0; 81 #if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN)) 82 { 83 /* 84 Previous int tells us how many ints the pointer has been shifted from 85 the original address provided by the system malloc(). 86 */ 87 int shift = *(((int*)ptr)-1) - SHIFT_CLASSID; 88 if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap"); 89 if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap"); 90 ptr = (void*)(((int*)ptr) - shift); 91 } 92 #endif 93 94 #if defined(PETSC_HAVE_FREE_RETURN_INT) 95 int err = free(ptr); 96 if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err); 97 #elif defined(PETSC_HAVE_MEMKIND) 98 memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */ 99 #else 100 free(ptr); 101 #endif 102 return 0; 103 } 104 105 PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result) 106 { 107 PetscErrorCode ierr; 108 109 if (!mem) { 110 ierr = PetscFreeAlign(*result, line, func, file); 111 if (ierr) return ierr; 112 *result = NULL; 113 return 0; 114 } 115 #if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN)) 116 { 117 /* 118 Previous int tells us how many ints the pointer has been shifted from 119 the original address provided by the system malloc(). 120 */ 121 int shift = *(((int*)*result)-1) - SHIFT_CLASSID; 122 if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap"); 123 if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap"); 124 *result = (void*)(((int*)*result) - shift); 125 } 126 #endif 127 128 #if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN) 129 # if defined(PETSC_HAVE_MEMKIND) 130 if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem); 131 else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem); 132 # else 133 *result = realloc(*result, mem); 134 # endif 135 #else 136 { 137 /* 138 malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned 139 */ 140 # if defined(PETSC_HAVE_MEMKIND) 141 int *ptr; 142 if (!currentmktype) ptr = (int*)memkind_realloc(MEMKIND_DEFAULT,*result,mem+2*PETSC_MEMALIGN); 143 else ptr = (int*)memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem+2*PETSC_MEMALIGN); 144 # else 145 int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN); 146 # endif 147 if (ptr) { 148 int shift = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN); 149 shift = (2*PETSC_MEMALIGN - shift)/sizeof(int); 150 ptr[shift-1] = shift + SHIFT_CLASSID; 151 ptr += shift; 152 *result = (void*)ptr; 153 } else { 154 *result = NULL; 155 } 156 } 157 #endif 158 if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem); 159 #if defined(PETSC_HAVE_MEMALIGN) 160 /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to 161 * realloc and, if the alignment is wrong, malloc/copy/free. */ 162 if (((size_t) (*result)) % PETSC_MEMALIGN) { 163 void *newResult; 164 165 newResult = memalign(PETSC_MEMALIGN,mem); 166 if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem); 167 ierr = PetscMemcpy(newResult,*result,mem); 168 if (ierr) return ierr; 169 # if defined(PETSC_HAVE_FREE_RETURN_INT) 170 { 171 int err = free(*result); 172 if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err); 173 } 174 # else 175 # if defined(PETSC_HAVE_MEMKIND) 176 memkind_free(0,*result); 177 # else 178 free(*result); 179 # endif 180 # endif 181 *result = newResult; 182 } 183 #endif 184 return 0; 185 } 186 187 PetscErrorCode (*PetscTrMalloc)(size_t,int,const char[],const char[],void**) = PetscMallocAlign; 188 PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[]) = PetscFreeAlign; 189 PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**) = PetscReallocAlign; 190 191 PetscBool petscsetmallocvisited = PETSC_FALSE; 192 193 /*@C 194 PetscMallocSet - Sets the routines used to do mallocs and frees. 195 This routine MUST be called before PetscInitialize() and may be 196 called only once. 197 198 Not Collective 199 200 Input Parameters: 201 + malloc - the malloc routine 202 - free - the free routine 203 204 Level: developer 205 206 Concepts: malloc 207 Concepts: memory^allocation 208 209 @*/ 210 PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,int,const char[],const char[],void**), 211 PetscErrorCode (*ifree)(void*,int,const char[],const char[])) 212 { 213 PetscFunctionBegin; 214 if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times"); 215 PetscTrMalloc = imalloc; 216 PetscTrFree = ifree; 217 petscsetmallocvisited = PETSC_TRUE; 218 PetscFunctionReturn(0); 219 } 220 221 /*@C 222 PetscMallocClear - Resets the routines used to do mallocs and frees to the 223 defaults. 224 225 Not Collective 226 227 Level: developer 228 229 Notes: 230 In general one should never run a PETSc program with different malloc() and 231 free() settings for different parts; this is because one NEVER wants to 232 free() an address that was malloced by a different memory management system 233 234 @*/ 235 PetscErrorCode PetscMallocClear(void) 236 { 237 PetscFunctionBegin; 238 PetscTrMalloc = PetscMallocAlign; 239 PetscTrFree = PetscFreeAlign; 240 petscsetmallocvisited = PETSC_FALSE; 241 PetscFunctionReturn(0); 242 } 243 244 PetscErrorCode PetscMemoryTrace(const char label[]) 245 { 246 PetscErrorCode ierr; 247 PetscLogDouble mem,mal; 248 static PetscLogDouble oldmem = 0,oldmal = 0; 249 250 PetscFunctionBegin; 251 ierr = PetscMemoryGetCurrentUsage(&mem);CHKERRQ(ierr); 252 ierr = PetscMallocGetCurrentUsage(&mal);CHKERRQ(ierr); 253 254 ierr = PetscPrintf(PETSC_COMM_WORLD,"%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);CHKERRQ(ierr); 255 oldmem = mem; 256 oldmal = mal; 257 PetscFunctionReturn(0); 258 } 259 260 static PetscErrorCode (*PetscTrMallocOld)(size_t,int,const char[],const char[],void**) = PetscMallocAlign; 261 static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[]) = PetscFreeAlign; 262 263 /*@C 264 PetscMallocSetDRAM - Set PetscMalloc to use DRAM. 265 If memkind is available, change the memkind type. Otherwise, switch the 266 current malloc and free routines to the PetscMallocAlign and 267 PetscFreeAlign (PETSc default). 268 269 Not Collective 270 271 Level: developer 272 273 Notes: 274 This provides a way to do the allocation on DRAM temporarily. One 275 can switch back to the previous choice by calling PetscMallocReset(). 276 277 .seealso: PetscMallocReset() 278 @*/ 279 PetscErrorCode PetscMallocSetDRAM(void) 280 { 281 PetscFunctionBegin; 282 if (PetscTrMalloc == PetscMallocAlign) { 283 #if defined(PETSC_HAVE_MEMKIND) 284 previousmktype = currentmktype; 285 currentmktype = PETSC_MK_DEFAULT; 286 #endif 287 } else { 288 /* Save the previous choice */ 289 PetscTrMallocOld = PetscTrMalloc; 290 PetscTrFreeOld = PetscTrFree; 291 PetscTrMalloc = PetscMallocAlign; 292 PetscTrFree = PetscFreeAlign; 293 } 294 PetscFunctionReturn(0); 295 } 296 297 /*@C 298 PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM 299 300 Not Collective 301 302 Level: developer 303 304 .seealso: PetscMallocSetDRAM() 305 @*/ 306 PetscErrorCode PetscMallocResetDRAM(void) 307 { 308 PetscFunctionBegin; 309 if (PetscTrMalloc == PetscMallocAlign) { 310 #if defined(PETSC_HAVE_MEMKIND) 311 currentmktype = previousmktype; 312 #endif 313 } else { 314 /* Reset to the previous choice */ 315 PetscTrMalloc = PetscTrMallocOld; 316 PetscTrFree = PetscTrFreeOld; 317 } 318 PetscFunctionReturn(0); 319 } 320