From ee937b70d93a77a409eb347e6fd778f0c6a34494 Mon Sep 17 00:00:00 2001 From: Misko Date: Mon, 30 Jan 2023 10:33:24 -0800 Subject: [PATCH 01/32] more debug --- bigmaac.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bigmaac.c b/bigmaac.c index 038537e..f662bb8 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -645,6 +645,11 @@ void *malloc(size_t size) } if (size>min_size_fry) { + fprintf(stderr,"BigMaac: Malloc %lu\n",size); + fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac); void * p=create_chunk(size); if (p==NULL) { OOM(); return NULL; @@ -714,6 +719,11 @@ void *realloc(void * ptr, size_t size) } pthread_mutex_unlock(&lock); + fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); + fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac); //allocated memory is big enough if (n->size>=size) { return ptr; From cd899d80c1e30022b3fd8a84e7f5c0c8ec8e219d Mon Sep 17 00:00:00 2001 From: Misko Date: Tue, 31 Jan 2023 15:54:20 -0800 Subject: [PATCH 02/32] add in signal sending --- bigmaac.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bigmaac.c b/bigmaac.c index f662bb8..0fd3f1f 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -13,6 +13,11 @@ #include #include +#define BIGMAAC_SIGNAL +#ifdef BIGMAAC_SIGNAL +#include +#endif + #include "bigmaac.h" #define OOM() fprintf(stderr,"BigMaac : Failed to find available space\n"); errno=ENOMEM; @@ -645,6 +650,9 @@ void *malloc(size_t size) } if (size>min_size_fry) { +#ifdef BIGMAAC_SIGNAL + kill(getpid(), SIGUSR1); +#endif fprintf(stderr,"BigMaac: Malloc %lu\n",size); fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", active_mmaps, @@ -676,6 +684,9 @@ void *calloc(size_t count, size_t size) //library is loaded and count/size are reasonable if (size>min_size_fry) { +#ifdef BIGMAAC_SIGNAL + kill(getpid(), SIGUSR1); +#endif void * p=create_chunk(size); if (p==NULL) { OOM(); return NULL; @@ -719,6 +730,9 @@ void *realloc(void * ptr, size_t size) } pthread_mutex_unlock(&lock); +#ifdef BIGMAAC_SIGNAL + kill(getpid(), SIGUSR1); +#endif fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", active_mmaps, From f9fb81cdfe30089654f18d9a71b74130fa1d43ef Mon Sep 17 00:00:00 2001 From: Misko Date: Tue, 31 Jan 2023 21:46:15 -0800 Subject: [PATCH 03/32] this one should be MAP_SHARED, although the behaviour might not be what the programmer expects --- bigmaac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index 0fd3f1f..4870962 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -546,7 +546,7 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); return -1; } - void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, fd, 0); + void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); if (ret_ptr==MAP_FAILED) { fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", active_mmaps, From 86197f47c07e442050b118af397888f1ce754705 Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 13:26:04 -0800 Subject: [PATCH 04/32] add 2x size allocation to mmaps --- bigmaac.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 4870962..7acc5fd 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -35,7 +35,7 @@ tmp->next->previous=tmp->previous; \ tmp->previous->next=tmp->next; \ } - +#define BIGMAAC_EXPECTED(size) (size*2) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, NOT_LOADED=0, @@ -658,7 +658,7 @@ void *malloc(size_t size) active_mmaps, 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac); - void * p=create_chunk(size); + void * p=create_chunk(BIGMAAC_EXPECTED(size)); if (p==NULL) { OOM(); return NULL; } @@ -746,7 +746,7 @@ void *realloc(void * ptr, size_t size) //existing chunk is not big enough void *p = NULL; if (size>min_size_fry) { - p=create_chunk(size); + p=create_chunk(BIGMAAC_EXPECTED(size)); if (p==NULL) { OOM(); //set errno } @@ -776,7 +776,7 @@ void *realloc(void * ptr, size_t size) return NULL; //errno already set } - void * p=create_chunk(size); + void * p=create_chunk(BIGMAAC_EXPECTED(size)); if (p!=NULL) { memcpy(p,mallocd_p,size); real_free((size_t)mallocd_p); From 6ae0e6c7803226c3c5326edbc58d673fbd322d09 Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 13:33:29 -0800 Subject: [PATCH 05/32] add 1.25x size allocation to mmaps --- bigmaac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index 7acc5fd..2eec2ce 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -35,7 +35,7 @@ tmp->next->previous=tmp->previous; \ tmp->previous->next=tmp->next; \ } -#define BIGMAAC_EXPECTED(size) (size*2) +#define BIGMAAC_EXPECTED(size) ((unsigned int)(size*1.25)) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, NOT_LOADED=0, From 0072acd914b373a00628ee2b010548e2a1350167 Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 13:33:59 -0800 Subject: [PATCH 06/32] add 1.25x size allocation to mmaps --- bigmaac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index 2eec2ce..8ab7ee6 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -435,7 +435,7 @@ static void bigmaac_init(void) fprintf(stderr,"Already init %d\n",load_state); return; } - fprintf(stderr,"Loading Bigmaac Heap X! PID:%d PPID:%d\n",getpid(),getppid()); + fprintf(stderr,"Loading Bigmaac Heap X2! PID:%d PPID:%d\n",getpid(),getppid()); load_state=LOADING_MEM_FUNCS; real_malloc = dlsym(RTLD_NEXT, "malloc"); real_free = dlsym(RTLD_NEXT, "free"); From 3d58006d53f36bf4ea66939aabbb85756d17bece Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 14:22:24 -0800 Subject: [PATCH 07/32] adding realloc proper --- bigmaac.c | 52 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 8ab7ee6..7ab65f0 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -316,7 +316,7 @@ static node * heap_pop_split(node * const head, const size_t size) { node * free_node = node_array[0]; if (free_node->sizesize==size) { + if (free_node->size==size) { //free node is exactly good size wise! heap_remove_idx(heap, free_node->heap_idx); free_node->in_use=IN_USE; verify_memory(head,1); @@ -727,8 +727,12 @@ void *realloc(void * ptr, size_t size) fprintf(stderr,"BigMaac: Cannot find node in BigMaac\n"); pthread_mutex_unlock(&lock); return NULL; - } - pthread_mutex_unlock(&lock); + } + + if (n->size>=size) { + pthread_mutex_unlock(&lock); + return ptr; + } #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); @@ -738,10 +742,44 @@ void *realloc(void * ptr, size_t size) active_mmaps, 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac); + + node * head = ptrnext!=NULL && n->in_use==FREE && (n->size+n->next->size)>=size) { + // check for equality + if ((n->size+n->next->size)==size) { + //remove the node and swallow it + n->size+=n->next->size; + heap_remove_idx(n->heap, n->next->heap_idx); + UNLINK(n->next); + real_free((size_t)n->next); + verify_memory(head,1); + } + + n->next->size-=(size-n->size); + heapify_down(heap,free_node->heap_idx); + verify_memory(head,1); + + //heapify from this node down + *used_node = (node){ + .size = size, + .ptr = free_node->ptr, + .next = free_node, + .previous = free_node->previous, + .in_use = IN_USE, + .heap_idx = -1 + }; + + free_node->size-=size; // need to now heapify this node + free_node->ptr=free_node->ptr+size; + + free_node->previous->next=used_node; + free_node->previous=used_node; + + } + pthread_mutex_unlock(&lock); + //allocated memory is big enough - if (n->size>=size) { - return ptr; - } //existing chunk is not big enough void *p = NULL; From 3c4649dc358b18187b615e11e70d03a652fb7289 Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 15:57:12 -0800 Subject: [PATCH 08/32] realloc might work --- bigmaac.c | 113 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 51 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 7ab65f0..5725fe9 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -58,6 +58,7 @@ typedef struct node { char * ptr; size_t size; heap * heap; + int fd; } node; //heap operations @@ -352,7 +353,8 @@ static node * heap_pop_split(node * const head, const size_t size) { .next = free_node, .previous = free_node->previous, .in_use = IN_USE, - .heap_idx = -1 + .heap_idx = -1, + .fd = -1 }; free_node->size-=size; // need to now heapify this node @@ -394,7 +396,8 @@ static node * ll_new(void * const ptr, const size_t size) { .next = head+1, .previous = NULL, .in_use = IN_USE, - .heap_idx = -1 + .heap_idx = -1, + .fd = -1 }; head[1] = (node){ .size = size, @@ -402,7 +405,8 @@ static node * ll_new(void * const ptr, const size_t size) { .next = NULL, .previous = head, .in_use = FREE, - .heap_idx = 0 + .heap_idx = 0, + .fd = -1 }; head->heap = (heap*)real_malloc(sizeof(heap)); @@ -495,13 +499,13 @@ static void bigmaac_init(void) } active_mmaps++; - const int ret = mmap_tmpfile(base_fries,size_fries); //allocate fries right away - if (ret<0) { + const int fd = mmap_tmpfile(base_fries,size_fries); //allocate fries right away + if (fd<0) { fprintf(stderr,"BigMaac: Failed to initialize library\n"); load_state=LIBRARY_FAIL; pthread_mutex_unlock(&lock); return; - } + } end_fries=((char*)base_fries)+size_fries; @@ -557,13 +561,13 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { } active_mmaps++; - ret = close(fd);//mmap keeps the fd open now + /*ret = close(fd);//mmap keeps the fd open now if (ret==-1) { fprintf(stderr,"BigMaac: close fd failed! %s\n", strerror(errno)); return -1; - } + }*/ - return 0; + return fd; } static void * create_chunk(size_t size) { @@ -586,10 +590,11 @@ static void * create_chunk(size_t size) { } if (head==_head_bigmaacs) { - int ret = mmap_tmpfile(heap_chunk->ptr,size); - if (ret<0) { + int fd = mmap_tmpfile(heap_chunk->ptr,size); + if (fd<0) { return NULL; } + heap_chunk->fd=fd; } return heap_chunk->ptr; @@ -619,6 +624,8 @@ static int remove_chunk_with_ptr(void * const ptr, void * const new_ptr, const s pthread_mutex_unlock(&lock); return 0; } + close(n->fd); + n->fd=-1; active_mmaps--; used_bigmaacs-=n->size; } else { @@ -730,56 +737,60 @@ void *realloc(void * ptr, size_t size) } if (n->size>=size) { - pthread_mutex_unlock(&lock); - return ptr; - } + pthread_mutex_unlock(&lock); + return ptr; + } #ifdef BIGMAAC_SIGNAL - kill(getpid(), SIGUSR1); + kill(getpid(), SIGUSR1); #endif fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); - fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac); + fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac); - node * head = ptrnext!=NULL && n->in_use==FREE && (n->size+n->next->size)>=size) { - // check for equality - if ((n->size+n->next->size)==size) { - //remove the node and swallow it - n->size+=n->next->size; - heap_remove_idx(n->heap, n->next->heap_idx); - UNLINK(n->next); - real_free((size_t)n->next); - verify_memory(head,1); - } - - n->next->size-=(size-n->size); - heapify_down(heap,free_node->heap_idx); - verify_memory(head,1); - - //heapify from this node down - *used_node = (node){ - .size = size, - .ptr = free_node->ptr, - .next = free_node, - .previous = free_node->previous, - .in_use = IN_USE, - .heap_idx = -1 - }; - - free_node->size-=size; // need to now heapify this node - free_node->ptr=free_node->ptr+size; - - free_node->previous->next=used_node; - free_node->previous=used_node; + fprintf(stderr,"REAL REAL REEALLOC!\n"); + // check for equality + if ((n->size+n->next->size)==size) { + //remove the node and swallow it + n->size+=n->next->size; + heap_remove_idx(n->heap, n->next->heap_idx); + UNLINK(n->next); + real_free((size_t)n->next); + verify_memory(head,1); + } else { + // move free space from next node to this one + n->next->size-=(size-n->size); + n->size+=(size-n->size); + //fix the free nodes place in the heap + heapify_down(n->heap,n->next->heap_idx); + verify_memory(head,1); + } + //change the size of the mmap + + int ret = ftruncate(n->fd, size); //resize the file + if (ret!=0) { + fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); + return NULL; + } + void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); + if (ret_ptr==MAP_FAILED) { + fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac, + strerror(errno)); + return NULL; + } } - pthread_mutex_unlock(&lock); + pthread_mutex_unlock(&lock); - //allocated memory is big enough + //allocated memory is big enough //existing chunk is not big enough void *p = NULL; From a6308ea20c1364b635886710fd417d25fa11f7fb Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 17:04:35 -0800 Subject: [PATCH 09/32] real realloc might be working --- bigmaac.c | 134 ++++++++++++++++++++++++++++--------------------- test_bigmaac.c | 36 +++++++++---- 2 files changed, 101 insertions(+), 69 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 5725fe9..6a882c0 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -13,7 +13,7 @@ #include #include -#define BIGMAAC_SIGNAL +#define BIGMAAC_SIGNAL1 #ifdef BIGMAAC_SIGNAL #include #endif @@ -35,7 +35,7 @@ tmp->next->previous=tmp->previous; \ tmp->previous->next=tmp->next; \ } -#define BIGMAAC_EXPECTED(size) ((unsigned int)(size*1.25)) +#define BIGMAAC_EXPECTED(size) ((unsigned int)(size*1.0)) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, NOT_LOADED=0, @@ -145,8 +145,8 @@ void log_bm(const char *data, ...){ } static inline void verify_memory(node * head, int global) { - //print_heap(head->heap); - //print_ll(head); + print_heap(head->heap); + print_ll(head); size_t heap_free=0; for (int i =0; iheap->used; i++) { assert(head->heap->node_array[i]->ptr!=NULL); @@ -178,7 +178,7 @@ static inline void verify_memory(node * head, int global) { static void print_ll(node * head) { while (head!=NULL) { - fprintf(stderr,"%p n=%p, u=%d, p=%p, size=%ld, ptr=%p\n",head,head->next,head->in_use,head->previous,head->size,head->ptr); + fprintf(stderr,"%p n=%p, u=%d, p=%p, size=%ld, ptr=%p, heap=%p, heap_idx=%d\n",head,head->next,head->in_use,head->previous,head->size,head->ptr,head->heap,head->heap_idx); head=head->next; } } @@ -354,6 +354,7 @@ static node * heap_pop_split(node * const head, const size_t size) { .previous = free_node->previous, .in_use = IN_USE, .heap_idx = -1, + .heap = heap, .fd = -1 }; @@ -413,7 +414,8 @@ static node * ll_new(void * const ptr, const size_t size) { if (head->heap==NULL) { fprintf(stderr,"BigMalloc heap failed\n"); return NULL; - } + } + head[1].heap=head->heap; head->heap->node_array=(node**)real_malloc(sizeof(node*)*1); if (head->heap->node_array==NULL) { fprintf(stderr,"BigMalloc heap failed 2\n"); @@ -713,6 +715,7 @@ void *reallocarray(void * ptr, size_t size,size_t count) { void *realloc(void * ptr, size_t size) { + fprintf(stderr,"RALLOC\n"); if(load_state==NOT_LOADED && real_malloc==NULL) { bigmaac_init(); } @@ -735,62 +738,77 @@ void *realloc(void * ptr, size_t size) pthread_mutex_unlock(&lock); return NULL; } - + if (n->size>=size) { - pthread_mutex_unlock(&lock); - return ptr; - } + fprintf(stderr,"ALREADY BIG ENGOUHG\n"); + pthread_mutex_unlock(&lock); + return ptr; + } #ifdef BIGMAAC_SIGNAL - kill(getpid(), SIGUSR1); + kill(getpid(), SIGUSR1); #endif - fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); - fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac); - - node * head = ptrnext!=NULL && n->in_use==FREE && (n->size+n->next->size)>=size) { - fprintf(stderr,"REAL REAL REEALLOC!\n"); - // check for equality - if ((n->size+n->next->size)==size) { - //remove the node and swallow it - n->size+=n->next->size; - heap_remove_idx(n->heap, n->next->heap_idx); - UNLINK(n->next); - real_free((size_t)n->next); - verify_memory(head,1); - } else { - // move free space from next node to this one - n->next->size-=(size-n->size); - n->size+=(size-n->size); - //fix the free nodes place in the heap - heapify_down(n->heap,n->next->heap_idx); - verify_memory(head,1); - } - //change the size of the mmap - - int ret = ftruncate(n->fd, size); //resize the file - if (ret!=0) { - fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); - return NULL; - } - void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); - if (ret_ptr==MAP_FAILED) { - fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac, - strerror(errno)); - return NULL; - } - - } - pthread_mutex_unlock(&lock); - - //allocated memory is big enough + fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); + fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac); + + node * head = ptrnext!=NULL,n->next->in_use==FREE ,(n->size+n->next->size)>=size); + if (n->next!=NULL && n->next->in_use==FREE && (n->size+n->next->size)>=size) { + fprintf(stderr,"REAL REAL REEALLOC!\n"); + // check for equality + if ((n->size+n->next->size)==size) { + fprintf(stderr,"REAL REAL REEALLOC1!\n"); + //remove the node and swallow it + n->size+=n->next->size; + heap_remove_idx(n->heap, n->next->heap_idx); + UNLINK(n->next); + real_free((size_t)n->next); + verify_memory(head,1); + } else { + // move free space from next node to this one + verify_memory(head,1); + used_bigmaacs+=(size-n->size); + n->next->size-=(size-n->size); + n->size+=(size-n->size); + //fix the free nodes place in the heap + heapify_down(n->heap,n->next->heap_idx); + verify_memory(head,1); + } + //change the size of the mmapp + + fprintf(stderr,"REAL REAL REEALLOC!\n"); + int ret = ftruncate(n->fd, size); //resize the file + if (ret!=0) { + fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); + return NULL; + } + fprintf(stderr,"REAL REAL REEALLOC3!\n"); + void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); + if (ret_ptr==MAP_FAILED) { + fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac, + strerror(errno)); + return NULL; + } + fprintf(stderr,"REAL REAL REEALLOC!\n"); + pthread_mutex_unlock(&lock); + return ptr; + + } + pthread_mutex_unlock(&lock); + + //allocated memory is big enough //existing chunk is not big enough void *p = NULL; diff --git a/test_bigmaac.c b/test_bigmaac.c index d4dfa63..97d6dda 100644 --- a/test_bigmaac.c +++ b/test_bigmaac.c @@ -2,7 +2,7 @@ #include #include "bigmaac.h" -#define N 20 +#define N 15 int seed=0xbeef; @@ -16,30 +16,42 @@ int checksum(int * p, size_t n ) { } int main() { + int n_ints=100+DEFAULT_MIN_BIGMAAC_SIZE/sizeof(int); + + /*void * v1=(void*)malloc(sizeof(int)*n_ints); + fprintf(stderr,"WTF\n"); + realloc(v1, 2*sizeof(int)*n_ints); + return 0; + //void * v1=(void*)malloc(sizeof(int)*n_ints);*/ + + int * chunks[N]; int checksums[N]; int sizes[N]; - int n_ints=DEFAULT_MIN_BIGMAAC_SIZE/(16*sizeof(int)); n_ints++; //make sure its big enough to trigger //lets test some mallocs fprintf(stderr,"Malloc\n"); for (int i=0; i 0 ? i : -i))); + //more=more > 0 ? more : -more; + fprintf(stderr,"MORE IS %d\n",more); sizes[i]=more+n_ints; + fprintf(stderr,"ALLOC %d\n",sizes[i]); chunks[i]=(int*)malloc(sizeof(int)*sizes[i]); if (chunks[i]==NULL) { fprintf(stderr,"Failed to malloc chunk %d\n",i); exit(1); } for (int j=0; ji/2) { @@ -67,7 +79,7 @@ int main() { checksums[i]=checksum(chunks[i],sizes[i]); fprintf(stdout,"%d %d\n",i,checksums[i]); } - } + }*/ fprintf(stderr,"Realloc\n"); for (int i=0; ii/2) { - free(chunks[i]); - chunks[i]=NULL; + //free(chunks[i]); + //chunks[i]=NULL; } else { int old_size=sizes[i]; - sizes[i]+=(seed%97); - if (seed%2==0) { + sizes[i]+=10000; //#(seed%9700001); + /*if (seed%2==0) { seed+=((seed%9)*(seed%7)+1)%(seed-1-seed%2); sizes[i]-=(seed%101); - } + }*/ + fprintf(stderr,"OLD SIZE %d new size %d\n",old_size,sizes[i]); chunks[i]=(int*)realloc(chunks[i],sizeof(int)*sizes[i]); + chunks[i]=(int*)realloc(chunks[i],sizeof(int)*sizes[i]+100000); if (sizes[i]>old_size) { for (int j=old_size; j Date: Wed, 1 Feb 2023 20:11:25 -0800 Subject: [PATCH 10/32] more debug --- bigmaac.c | 73 ++++++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 6a882c0..fc24063 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -38,11 +38,11 @@ #define BIGMAAC_EXPECTED(size) ((unsigned int)(size*1.0)) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, - NOT_LOADED=0, - LOADING_MEM_FUNCS=1, - LOADING_LIBRARY=2, - LOADED=3 - }; + NOT_LOADED=0, + LOADING_MEM_FUNCS=1, + LOADING_LIBRARY=2, + LOADED=3 +}; typedef struct heap { size_t used; @@ -110,7 +110,7 @@ static size_t fry_size_multiple=DEFAULT_FRY_SIZE_MULTIPLE; static size_t used_fries=0; static size_t used_bigmaacs=0; -static size_t page_size = 0; +static size_t page_size = 0; static enum load_status load_state=NOT_LOADED; @@ -455,7 +455,7 @@ static void bigmaac_init(void) log_bm("OPEN LIB\n"); - page_size = sysconf(_SC_PAGE_SIZE); + page_size = sysconf(_SC_PAGE_SIZE); //load enviornment variables const char * env_template=getenv("BIGMAAC_TEMPLATE"); @@ -552,22 +552,22 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); return -1; } - void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); + void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); if (ret_ptr==MAP_FAILED) { fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac, - strerror(errno)); + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac, + strerror(errno)); return -1; } active_mmaps++; /*ret = close(fd);//mmap keeps the fd open now - if (ret==-1) { - fprintf(stderr,"BigMaac: close fd failed! %s\n", strerror(errno)); - return -1; - }*/ + if (ret==-1) { + fprintf(stderr,"BigMaac: close fd failed! %s\n", strerror(errno)); + return -1; + }*/ return fd; } @@ -575,7 +575,7 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { static void * create_chunk(size_t size) { node * const head = size>min_size_bigmaac ? _head_bigmaacs : _head_fries; //TODO lock per head? pthread_mutex_lock(&lock); //keep lock here so that verify is consistent - //page align the size requested + //page align the size requested if (head==_head_bigmaacs) { size=SIZE_TO_MULTIPLE(size,page_size); used_bigmaacs+=size; @@ -596,7 +596,7 @@ static void * create_chunk(size_t size) { if (fd<0) { return NULL; } - heap_chunk->fd=fd; + heap_chunk->fd=fd; } return heap_chunk->ptr; @@ -620,14 +620,14 @@ static int remove_chunk_with_ptr(void * const ptr, void * const new_ptr, const s node * head = ptrptr, n->size, PROT_NONE, MAP_ANONYMOUS | MAP_FIXED | MAP_PRIVATE, -1, 0); + const void * remap = mmap(n->ptr, n->size, PROT_NONE, MAP_ANONYMOUS | MAP_FIXED | MAP_PRIVATE, -1, 0); if (remap==MAP_FAILED) { fprintf(stderr,"BigMaac: wrong with munmap()! %s\n", strerror(errno)); pthread_mutex_unlock(&lock); return 0; } - close(n->fd); - n->fd=-1; + close(n->fd); + n->fd=-1; active_mmaps--; used_bigmaacs-=n->size; } else { @@ -660,13 +660,13 @@ void *malloc(size_t size) if (size>min_size_fry) { #ifdef BIGMAAC_SIGNAL - kill(getpid(), SIGUSR1); + kill(getpid(), SIGUSR1); #endif - fprintf(stderr,"BigMaac: Malloc %lu\n",size); + fprintf(stderr,"BigMaac: Malloc %lu\n",size); fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac); + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac); void * p=create_chunk(BIGMAAC_EXPECTED(size)); if (p==NULL) { OOM(); return NULL; @@ -694,7 +694,7 @@ void *calloc(size_t count, size_t size) //library is loaded and count/size are reasonable if (size>min_size_fry) { #ifdef BIGMAAC_SIGNAL - kill(getpid(), SIGUSR1); + kill(getpid(), SIGUSR1); #endif void * p=create_chunk(size); if (p==NULL) { @@ -705,7 +705,7 @@ void *calloc(size_t count, size_t size) } return p; } - + return real_calloc(count,size); } @@ -715,7 +715,6 @@ void *reallocarray(void * ptr, size_t size,size_t count) { void *realloc(void * ptr, size_t size) { - fprintf(stderr,"RALLOC\n"); if(load_state==NOT_LOADED && real_malloc==NULL) { bigmaac_init(); } @@ -763,10 +762,8 @@ void *realloc(void * ptr, size_t size) //check if adjacent node is free, knock knock | requires lock? fprintf(stderr,"%p FREE%d %d %d %d\n",n,FREE,n->next!=NULL,n->next->in_use==FREE ,(n->size+n->next->size)>=size); if (n->next!=NULL && n->next->in_use==FREE && (n->size+n->next->size)>=size) { - fprintf(stderr,"REAL REAL REEALLOC!\n"); // check for equality if ((n->size+n->next->size)==size) { - fprintf(stderr,"REAL REAL REEALLOC1!\n"); //remove the node and swallow it n->size+=n->next->size; heap_remove_idx(n->heap, n->next->heap_idx); @@ -785,14 +782,12 @@ void *realloc(void * ptr, size_t size) } //change the size of the mmapp - fprintf(stderr,"REAL REAL REEALLOC!\n"); int ret = ftruncate(n->fd, size); //resize the file if (ret!=0) { fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); return NULL; } - fprintf(stderr,"REAL REAL REEALLOC3!\n"); - void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); + void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); if (ret_ptr==MAP_FAILED) { fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", active_mmaps, @@ -803,8 +798,8 @@ void *realloc(void * ptr, size_t size) } fprintf(stderr,"REAL REAL REEALLOC!\n"); pthread_mutex_unlock(&lock); - return ptr; - + return ptr; + } pthread_mutex_unlock(&lock); @@ -893,12 +888,12 @@ int main() { sizes[i]=0; } -omp_set_num_threads(T); + omp_set_num_threads(T); #pragma omp parallel { - int t = omp_get_thread_num(); + int t = omp_get_thread_num(); fprintf(stderr,"T%d\n",t); - srand(123+t); + srand(123+t); for (int i=1; i Date: Wed, 1 Feb 2023 20:12:05 -0800 Subject: [PATCH 11/32] more debug --- bigmaac.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index fc24063..b0590a3 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -13,7 +13,7 @@ #include #include -#define BIGMAAC_SIGNAL1 +#define BIGMAAC_SIGNAL #ifdef BIGMAAC_SIGNAL #include #endif @@ -662,7 +662,6 @@ void *malloc(size_t size) #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif - fprintf(stderr,"BigMaac: Malloc %lu\n",size); fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", active_mmaps, 1.0-((float)used_fries)/size_fries, From 88cff9e5ed4fa53c79286b54695566b6779eb845 Mon Sep 17 00:00:00 2001 From: Misko Date: Wed, 1 Feb 2023 20:16:06 -0800 Subject: [PATCH 12/32] more debug --- Makefile | 8 +++++++- bigmaac.c | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5e5e83a..0b9c51b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: bigmaac.so bigmaac_debug.so preload test_bigmaac test bigmaac_main bigmaac_main_debug +all: bigmaac.so bigmaac_debug.so bigmaac_signal.so bigmaac_debug_signal.so preload test_bigmaac test bigmaac_main bigmaac_main_debug bigmaac_main: bigmaac.c bigmaac.h gcc -DMAIN bigmaac.c -o bigmaac_main -Wall -g -ldl -fopenmp @@ -12,6 +12,12 @@ bigmaac.so: bigmaac.c bigmaac.h bigmaac_debug.so: bigmaac.c bigmaac.h gcc -shared -DDEBUG -fPIC bigmaac.c -o bigmaac_debug.so -ldl -Wall -g +bigmaac_signal.so: bigmaac.c bigmaac.h + gcc -shared -fPIC bigmaac.c -o bigmaac_signal.so -ldl -Wall -O3 -DBIGMAAC_SIGNAL + +bigmaac_debug_signal.so: bigmaac.c bigmaac.h + gcc -shared -DDEBUG -fPIC bigmaac.c -o bigmaac_debug_signal.so -ldl -Wall -g -DBIGMAAC_SIGNAL + preload: preload.c gcc -Wall preload.c -o preload diff --git a/bigmaac.c b/bigmaac.c index b0590a3..aebba89 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -13,7 +13,6 @@ #include #include -#define BIGMAAC_SIGNAL #ifdef BIGMAAC_SIGNAL #include #endif From 43b4d36ca0aaa6c9b4a6edba9d162270588e0aab Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:13:10 -0800 Subject: [PATCH 13/32] start adding requested size --- bigmaac.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index aebba89..c64741d 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -13,6 +13,13 @@ #include #include +/* + +Allocate X% more space in VM to use for re-alloc +Each node needs actual size and used size + +*/ + #ifdef BIGMAAC_SIGNAL #include #endif @@ -34,7 +41,7 @@ tmp->next->previous=tmp->previous; \ tmp->previous->next=tmp->next; \ } -#define BIGMAAC_EXPECTED(size) ((unsigned int)(size*1.0)) +#define BIGMAAC_BUFFERED_SIZE(size) ((unsigned int)(size*2.0)) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, NOT_LOADED=0, @@ -55,7 +62,8 @@ typedef struct node { enum memory_use in_use; int heap_idx; char * ptr; - size_t size; + size_t size; //The actual size of mmap + size_t requested_size; //The user requested this much heap * heap; int fd; } node; @@ -305,7 +313,8 @@ static int heap_free_node(node * const head, node * const n) { return 0; } -static node * heap_pop_split(node * const head, const size_t size) { +static node * heap_pop_split(node * const head, const size_t requested_size) { + size_t size = BIGMAAC_BUFFERED_SIZE(requested_size); // the actual size we are going to alloc verify_memory(head,0); if (head->heap->used==0) { return NULL; @@ -336,6 +345,7 @@ static node * heap_pop_split(node * const head, const size_t size) { if (free_node->size==size) { //free node is exactly good size wise! heap_remove_idx(heap, free_node->heap_idx); free_node->in_use=IN_USE; + free_node->requested_size=requested_size; verify_memory(head,1); return free_node; } @@ -348,6 +358,7 @@ static node * heap_pop_split(node * const head, const size_t size) { //heapify from this node down *used_node = (node){ .size = size, + .requested_size = requested_size, .ptr = free_node->ptr, .next = free_node, .previous = free_node->previous, @@ -591,7 +602,7 @@ static void * create_chunk(size_t size) { } if (head==_head_bigmaacs) { - int fd = mmap_tmpfile(heap_chunk->ptr,size); + int fd = mmap_tmpfile(heap_chunk->ptr,heap_chunk->requested_size); if (fd<0) { return NULL; } @@ -661,11 +672,11 @@ void *malloc(size_t size) #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif - fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", + fprintf(stderr,"BigMaac: malloc mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", active_mmaps, 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac); - void * p=create_chunk(BIGMAAC_EXPECTED(size)); + void * p=create_chunk(size); if (p==NULL) { OOM(); return NULL; } @@ -806,7 +817,7 @@ void *realloc(void * ptr, size_t size) //existing chunk is not big enough void *p = NULL; if (size>min_size_fry) { - p=create_chunk(BIGMAAC_EXPECTED(size)); + p=create_chunk(size); if (p==NULL) { OOM(); //set errno } @@ -836,7 +847,7 @@ void *realloc(void * ptr, size_t size) return NULL; //errno already set } - void * p=create_chunk(BIGMAAC_EXPECTED(size)); + void * p=create_chunk(size); if (p!=NULL) { memcpy(p,mallocd_p,size); real_free((size_t)mallocd_p); From da5fb51fe825a5d90003fa642e78bfe855fa89cd Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:14:54 -0800 Subject: [PATCH 14/32] make sure its always bigger --- bigmaac.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index c64741d..4874dc1 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -19,6 +19,7 @@ Allocate X% more space in VM to use for re-alloc Each node needs actual size and used size */ +#define MAX(a,b) (a > b ? a : b) #ifdef BIGMAAC_SIGNAL #include @@ -41,7 +42,7 @@ Each node needs actual size and used size tmp->next->previous=tmp->previous; \ tmp->previous->next=tmp->next; \ } -#define BIGMAAC_BUFFERED_SIZE(size) ((unsigned int)(size*2.0)) +#define BIGMAAC_BUFFERED_SIZE(size) (MAX(size,(unsigned int)(size*2.0))) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, NOT_LOADED=0, From 8369b328ec610399c4f0c27848c0358a0f85f752 Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:23:43 -0800 Subject: [PATCH 15/32] make sure its always bigger --- bigmaac.c | 58 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 4874dc1..7b17a28 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -535,6 +535,25 @@ static void bigmaac_init(void) // BigMaac helper functions +static int resize_node(node * n, const size_t requested_size) { + int ret = ftruncate(n->fd, requested_size); //resize the file + if (ret!=0) { + fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); + return -1; + } + void * ret_ptr = mmap(n->ptr, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); + if (ret_ptr==MAP_FAILED) { + fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac, + strerror(errno)); + return -1; + } + return 0; + +} + static int mmap_tmpfile(void * const ptr, const size_t size) { char * const filename=(char*)real_malloc(sizeof(char)*(strlen(template)+1)); if (filename==NULL) { @@ -748,12 +767,6 @@ void *realloc(void * ptr, size_t size) return NULL; } - if (n->size>=size) { - fprintf(stderr,"ALREADY BIG ENGOUHG\n"); - pthread_mutex_unlock(&lock); - return ptr; - } - #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif @@ -763,14 +776,24 @@ void *realloc(void * ptr, size_t size) 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac); + node * head = ptrnext!=NULL,n->next->in_use==FREE ,(n->size+n->next->size)>=size); + + if (n->size>=size) { + fprintf(stderr,"Realloc handle #1\n"); + int ret = resize_node(n,size); + if (ret!=0) { + fprintf(stderr,"BigMaac: Failed to resize mmap\n"); + } + pthread_mutex_unlock(&lock); + return ptr; + } + if (n->next!=NULL && n->next->in_use==FREE && (n->size+n->next->size)>=size) { // check for equality if ((n->size+n->next->size)==size) { @@ -790,22 +813,13 @@ void *realloc(void * ptr, size_t size) heapify_down(n->heap,n->next->heap_idx); verify_memory(head,1); } - //change the size of the mmapp - int ret = ftruncate(n->fd, size); //resize the file - if (ret!=0) { - fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); - return NULL; - } - void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); - if (ret_ptr==MAP_FAILED) { - fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac, - strerror(errno)); - return NULL; + //change the size of the mmapp + int ret = resize_node(n,size); + if (ret!=0) { + fprintf(stderr,"BigMaac: Failed to resize mmap\n"); } + fprintf(stderr,"REAL REAL REEALLOC!\n"); pthread_mutex_unlock(&lock); return ptr; From a1a505c0d2c85e7e2414620243fe3202f31802b1 Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:26:21 -0800 Subject: [PATCH 16/32] make sure its always bigger --- bigmaac.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigmaac.c b/bigmaac.c index 7b17a28..a5567fe 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -795,8 +795,10 @@ void *realloc(void * ptr, size_t size) } if (n->next!=NULL && n->next->in_use==FREE && (n->size+n->next->size)>=size) { + fprintf(stderr,"Realloc handle #2\n"); // check for equality if ((n->size+n->next->size)==size) { + fprintf(stderr,"Realloc handle #2a\n"); //remove the node and swallow it n->size+=n->next->size; heap_remove_idx(n->heap, n->next->heap_idx); @@ -804,6 +806,7 @@ void *realloc(void * ptr, size_t size) real_free((size_t)n->next); verify_memory(head,1); } else { + fprintf(stderr,"Realloc handle #2b\n"); // move free space from next node to this one verify_memory(head,1); used_bigmaacs+=(size-n->size); From af862b3f0d47d5a4cff9cfe2cf50bff400030c4b Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:31:00 -0800 Subject: [PATCH 17/32] make sure its always bigger --- bigmaac.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index a5567fe..970648a 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -536,6 +536,8 @@ static void bigmaac_init(void) // BigMaac helper functions static int resize_node(node * n, const size_t requested_size) { + assert(n->size>=requested_size); + n->requested_size=requested_size; int ret = ftruncate(n->fd, requested_size); //resize the file if (ret!=0) { fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); @@ -784,17 +786,8 @@ void *realloc(void * ptr, size_t size) size=SIZE_TO_MULTIPLE(size,fry_size_multiple); } - if (n->size>=size) { - fprintf(stderr,"Realloc handle #1\n"); - int ret = resize_node(n,size); - if (ret!=0) { - fprintf(stderr,"BigMaac: Failed to resize mmap\n"); - } - pthread_mutex_unlock(&lock); - return ptr; - } - if (n->next!=NULL && n->next->in_use==FREE && (n->size+n->next->size)>=size) { + if (n->next!=NULL && n->next->in_use==FREE && (n->sizesize+n->next->size)>=size) { fprintf(stderr,"Realloc handle #2\n"); // check for equality if ((n->size+n->next->size)==size) { @@ -816,17 +809,16 @@ void *realloc(void * ptr, size_t size) heapify_down(n->heap,n->next->heap_idx); verify_memory(head,1); } + } - //change the size of the mmapp + if (n->size>=size) { + fprintf(stderr,"Realloc handle #1\n"); int ret = resize_node(n,size); if (ret!=0) { fprintf(stderr,"BigMaac: Failed to resize mmap\n"); } - - fprintf(stderr,"REAL REAL REEALLOC!\n"); pthread_mutex_unlock(&lock); return ptr; - } pthread_mutex_unlock(&lock); From 6b80a11037c78ddd86d5e83bea0d49174fb0d31d Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:31:56 -0800 Subject: [PATCH 18/32] make sure its always bigger --- bigmaac.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigmaac.c b/bigmaac.c index 970648a..b24f059 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -185,6 +185,7 @@ static inline void verify_memory(node * head, int global) { } static void print_ll(node * head) { + fprintf(stderr,"PRINT_LL()\n"); while (head!=NULL) { fprintf(stderr,"%p n=%p, u=%d, p=%p, size=%ld, ptr=%p, heap=%p, heap_idx=%d\n",head,head->next,head->in_use,head->previous,head->size,head->ptr,head->heap,head->heap_idx); head=head->next; @@ -192,6 +193,7 @@ static void print_ll(node * head) { } static void print_heap(heap* heap) { + fprintf(stderr,"PRINT_HEAP()\n"); for (int i =0; iused; i++) { fprintf(stderr,"parent %d node %d , ptr=%p size=%ld\n", (i-1)/2, i, From 3619d25604c4974adfb5c864c2c0038d680b5381 Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 09:35:19 -0800 Subject: [PATCH 19/32] make sure its always bigger --- bigmaac.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigmaac.h b/bigmaac.h index 612983d..f2219b2 100644 --- a/bigmaac.h +++ b/bigmaac.h @@ -1,7 +1,7 @@ #define DEFAULT_MIN_BIGMAAC_SIZE (1024*1024*300) //300MB #define DEFAULT_MIN_FRY_SIZE 0 // disabled #define DEFAULT_TEMPLATE "/tmp/bigmaax.XXXXXXXX" -#define DEFAULT_MAX_BIGMAAC (1024L*1024*1024*512) //512GB -#define DEFAULT_MAX_FRIES (1024L*1024*1024*512) //512GB +#define DEFAULT_MAX_BIGMAAC (1024L*1024*1024*1024) //1024GB +#define DEFAULT_MAX_FRIES (1024L*1024*1024*1024) //1024GB #define DEFAULT_FRY_SIZE_MULTIPLE 256 From 33837f4b13d958782f3bccf8ce2764a65d95ea57 Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 2 Feb 2023 10:46:35 -0800 Subject: [PATCH 20/32] add in signal sending --- bigmaac.c | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index b24f059..7a977b7 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -125,6 +125,14 @@ static enum load_status load_state=NOT_LOADED; //debug functions static inline void verify_memory(node * head,int global); static inline void log_bm(const char *data, ...); +static void print_stats() { + fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac, + strerror(errno)); + +} #ifdef DEBUG static void print_ll(node * head); static void print_heap(heap* heap); @@ -132,6 +140,7 @@ static pthread_mutex_t log_lock = PTHREAD_MUTEX_INITIALIZER; static FILE * f; static int this_pid = 0; + void log_bm(const char *data, ...){ pthread_mutex_lock(&log_lock); int pid = getpid(); @@ -318,6 +327,8 @@ static int heap_free_node(node * const head, node * const n) { static node * heap_pop_split(node * const head, const size_t requested_size) { size_t size = BIGMAAC_BUFFERED_SIZE(requested_size); // the actual size we are going to alloc + + verify_memory(head,0); if (head->heap->used==0) { return NULL; @@ -378,6 +389,16 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { free_node->previous=used_node; heapify_down(heap,free_node->heap_idx); + + //update used metrics + if (head==_head_bigmaacs) { + size=SIZE_TO_MULTIPLE(size,page_size); + used_bigmaacs+=size; + } else { + size=SIZE_TO_MULTIPLE(size,fry_size_multiple); + used_fries+=size; + } + verify_memory(head,1); return used_node; @@ -547,11 +568,7 @@ static int resize_node(node * n, const size_t requested_size) { } void * ret_ptr = mmap(n->ptr, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); if (ret_ptr==MAP_FAILED) { - fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac, - strerror(errno)); + print_stats(); return -1; } return 0; @@ -588,11 +605,7 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { } void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); if (ret_ptr==MAP_FAILED) { - fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac, - strerror(errno)); + print_stats(); return -1; } active_mmaps++; @@ -610,13 +623,6 @@ static void * create_chunk(size_t size) { node * const head = size>min_size_bigmaac ? _head_bigmaacs : _head_fries; //TODO lock per head? pthread_mutex_lock(&lock); //keep lock here so that verify is consistent //page align the size requested - if (head==_head_bigmaacs) { - size=SIZE_TO_MULTIPLE(size,page_size); - used_bigmaacs+=size; - } else { - size=SIZE_TO_MULTIPLE(size,fry_size_multiple); - used_fries+=size; - } node * heap_chunk=heap_pop_split(head, size); pthread_mutex_unlock(&lock); @@ -696,10 +702,7 @@ void *malloc(size_t size) #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif - fprintf(stderr,"BigMaac: malloc mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac); + print_stats(); void * p=create_chunk(size); if (p==NULL) { OOM(); return NULL; @@ -775,10 +778,7 @@ void *realloc(void * ptr, size_t size) kill(getpid(), SIGUSR1); #endif fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); - fprintf(stderr,"BigMaac: mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac); + print_stats(); node * head = ptr Date: Sun, 5 Feb 2023 12:33:43 -0800 Subject: [PATCH 21/32] fix heap remove bug --- bigmaac.c | 77 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index 7a977b7..afe6d11 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -15,10 +15,10 @@ /* -Allocate X% more space in VM to use for re-alloc -Each node needs actual size and used size + Allocate X% more space in VM to use for re-alloc + Each node needs actual size and used size -*/ + */ #define MAX(a,b) (a > b ? a : b) #ifdef BIGMAAC_SIGNAL @@ -126,11 +126,11 @@ static enum load_status load_state=NOT_LOADED; static inline void verify_memory(node * head,int global); static inline void log_bm(const char *data, ...); static void print_stats() { - fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", - active_mmaps, - 1.0-((float)used_fries)/size_fries, - 1.0-((float)used_bigmaacs)/size_bigmaac, - strerror(errno)); + fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + active_mmaps, + 1.0-((float)used_fries)/size_fries, + 1.0-((float)used_bigmaacs)/size_bigmaac, + strerror(errno)); } #ifdef DEBUG @@ -228,12 +228,17 @@ static void heap_remove_idx(heap * const heap, const int idx) { } //take the last one and place it here - heap->node_array[idx]->heap_idx=-1; // node is out of the heap - heap->node_array[heap->used-1]->heap_idx=idx; //node has moved up in the heap - heap->node_array[idx]=heap->node_array[heap->used-1]; - heap->used--; //the heap is now smaller + if (idx==heap->used-1) { + //this is the last node in the array, we can just drop it + heap->used--; //the heap is now smaller + } else { + heap->node_array[idx]->heap_idx=-1; // node is out of the heap + heap->node_array[heap->used-1]->heap_idx=idx; //node has moved up in the heap + heap->node_array[idx]=heap->node_array[heap->used-1]; + heap->used--; //the heap is now smaller - heapify_down(heap,idx); + heapify_down(heap,idx); + } } static void heapify_up(heap * const heap, const int idx) { @@ -359,7 +364,7 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { if (free_node->size==size) { //free node is exactly good size wise! heap_remove_idx(heap, free_node->heap_idx); free_node->in_use=IN_USE; - free_node->requested_size=requested_size; + free_node->requested_size=requested_size; verify_memory(head,1); return free_node; } @@ -372,7 +377,7 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { //heapify from this node down *used_node = (node){ .size = size, - .requested_size = requested_size, + .requested_size = requested_size, .ptr = free_node->ptr, .next = free_node, .previous = free_node->previous, @@ -559,19 +564,19 @@ static void bigmaac_init(void) // BigMaac helper functions static int resize_node(node * n, const size_t requested_size) { - assert(n->size>=requested_size); - n->requested_size=requested_size; - int ret = ftruncate(n->fd, requested_size); //resize the file - if (ret!=0) { - fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); - return -1; - } - void * ret_ptr = mmap(n->ptr, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); - if (ret_ptr==MAP_FAILED) { - print_stats(); - return -1; - } - return 0; + assert(n->size>=requested_size); + n->requested_size=requested_size; + int ret = ftruncate(n->fd, requested_size); //resize the file + if (ret!=0) { + fprintf(stderr,"BigMaac: ftruncate failed! %s\n", strerror(errno)); + return -1; + } + void * ret_ptr = mmap(n->ptr, requested_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, n->fd, 0); + if (ret_ptr==MAP_FAILED) { + print_stats(); + return -1; + } + return 0; } @@ -605,7 +610,7 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { } void * ret_ptr = mmap(ptr, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); if (ret_ptr==MAP_FAILED) { - print_stats(); + print_stats(); return -1; } active_mmaps++; @@ -702,7 +707,7 @@ void *malloc(size_t size) #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif - print_stats(); + print_stats(); void * p=create_chunk(size); if (p==NULL) { OOM(); return NULL; @@ -778,7 +783,7 @@ void *realloc(void * ptr, size_t size) kill(getpid(), SIGUSR1); #endif fprintf(stderr,"BigMaac: Realloc current %lu vs new %lu\n",n->size,size); - print_stats(); + print_stats(); node * head = ptrsize+n->next->size)==size) { - fprintf(stderr,"Realloc handle #2a\n"); + fprintf(stderr,"Realloc handle #2a\n"); //remove the node and swallow it n->size+=n->next->size; heap_remove_idx(n->heap, n->next->heap_idx); @@ -801,7 +806,7 @@ void *realloc(void * ptr, size_t size) real_free((size_t)n->next); verify_memory(head,1); } else { - fprintf(stderr,"Realloc handle #2b\n"); + fprintf(stderr,"Realloc handle #2b\n"); // move free space from next node to this one verify_memory(head,1); used_bigmaacs+=(size-n->size); @@ -815,9 +820,9 @@ void *realloc(void * ptr, size_t size) if (n->size>=size) { fprintf(stderr,"Realloc handle #1\n"); - int ret = resize_node(n,size); - if (ret!=0) { - fprintf(stderr,"BigMaac: Failed to resize mmap\n"); + int ret = resize_node(n,size); + if (ret!=0) { + fprintf(stderr,"BigMaac: Failed to resize mmap\n"); } pthread_mutex_unlock(&lock); return ptr; From 510d1991e849beb348b8ef5190fccc0a484ad96c Mon Sep 17 00:00:00 2001 From: Misko Date: Sun, 5 Feb 2023 21:20:50 -0800 Subject: [PATCH 22/32] fix heap remove bug --- bigmaac.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigmaac.c b/bigmaac.c index afe6d11..605fff9 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -230,11 +230,14 @@ static void heap_remove_idx(heap * const heap, const int idx) { //take the last one and place it here if (idx==heap->used-1) { //this is the last node in the array, we can just drop it + heap->node_array[idx]->heap_idx=-1; // node is out of the heap heap->used--; //the heap is now smaller } else { heap->node_array[idx]->heap_idx=-1; // node is out of the heap + heap->node_array[heap->used-1]->heap_idx=idx; //node has moved up in the heap heap->node_array[idx]=heap->node_array[heap->used-1]; + heap->used--; //the heap is now smaller heapify_down(heap,idx); From be060061643724b0a7d4c63da11770f773bb721d Mon Sep 17 00:00:00 2001 From: Misko Date: Sun, 5 Feb 2023 21:22:07 -0800 Subject: [PATCH 23/32] fix heap remove bug --- bigmaac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index 605fff9..e3868bb 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -126,7 +126,7 @@ static enum load_status load_state=NOT_LOADED; static inline void verify_memory(node * head,int global); static inline void log_bm(const char *data, ...); static void print_stats() { - fprintf(stderr,"BigMaac: mmap failed! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + fprintf(stderr,"BigMaac: stats! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", active_mmaps, 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac, From d9b2a4fd236bcebdf27c0bde2bd1a4300818c146 Mon Sep 17 00:00:00 2001 From: Misko Date: Sun, 5 Feb 2023 21:24:52 -0800 Subject: [PATCH 24/32] fix heap remove bug --- bigmaac.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index e3868bb..a8dccdf 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -707,6 +707,7 @@ void *malloc(size_t size) } if (size>min_size_fry) { + fprintf(stderr,"MALLOC %lu\n",size); #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif @@ -737,6 +738,7 @@ void *calloc(size_t count, size_t size) //library is loaded and count/size are reasonable if (size>min_size_fry) { + fprintf(stderr,"CALLOC\n"); #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif @@ -770,9 +772,9 @@ void *realloc(void * ptr, size_t size) if (ptr==NULL || size==0) { return malloc(size); } - //currently managed by BigMaac if (ptr>=base_fries && ptr Date: Sun, 5 Feb 2023 21:32:00 -0800 Subject: [PATCH 25/32] fix heap remove bug --- bigmaac.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index a8dccdf..bfbf456 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -169,8 +169,8 @@ static inline void verify_memory(node * head, int global) { assert(head->heap->node_array[i]->ptr!=NULL); heap_free+=head->heap->node_array[i]->size; } - size_t t=0; - size_t ll_free=0; + size_t t=0; //this is how much space is in the linked list + size_t ll_free=0; //this is how much free space is in the linked list node * prev=NULL; node *c =head; while(c!=NULL) { @@ -334,14 +334,23 @@ static int heap_free_node(node * const head, node * const n) { } static node * heap_pop_split(node * const head, const size_t requested_size) { - size_t size = BIGMAAC_BUFFERED_SIZE(requested_size); // the actual size we are going to alloc - verify_memory(head,0); if (head->heap->used==0) { return NULL; } + size_t size = BIGMAAC_BUFFERED_SIZE(requested_size); // the actual size we are going to alloc + //update used metrics + if (head==_head_bigmaacs) { + size=SIZE_TO_MULTIPLE(size,page_size); + used_bigmaacs+=size; + } else { + size=SIZE_TO_MULTIPLE(size,fry_size_multiple); + used_fries+=size; + } + fprintf(stderr,"requested size %lu , size to allocate %lu\n",requested_size,size); + heap * heap = head->heap; node ** node_array = heap->node_array; @@ -398,14 +407,6 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { heapify_down(heap,free_node->heap_idx); - //update used metrics - if (head==_head_bigmaacs) { - size=SIZE_TO_MULTIPLE(size,page_size); - used_bigmaacs+=size; - } else { - size=SIZE_TO_MULTIPLE(size,fry_size_multiple); - used_fries+=size; - } verify_memory(head,1); From 0e42aaab609411c7470d3320765e4fcfb8f9b5a0 Mon Sep 17 00:00:00 2001 From: Misko Date: Sun, 5 Feb 2023 21:36:16 -0800 Subject: [PATCH 26/32] fix heap remove bug --- bigmaac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index bfbf456..e04638f 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -42,7 +42,7 @@ tmp->next->previous=tmp->previous; \ tmp->previous->next=tmp->next; \ } -#define BIGMAAC_BUFFERED_SIZE(size) (MAX(size,(unsigned int)(size*2.0))) +#define BIGMAAC_BUFFERED_SIZE(size) (MAX(size,(unsigned long)(size*2.0))) enum memory_use { IN_USE=0, FREE=1 }; enum load_status { LIBRARY_FAIL=-1, NOT_LOADED=0, From d67db97c7b2778c7b1ff5dbaab7a94a751e3df04 Mon Sep 17 00:00:00 2001 From: Misko Date: Mon, 6 Feb 2023 14:41:38 -0800 Subject: [PATCH 27/32] catch fork --- bigmaac.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index e04638f..a8fcac6 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -91,6 +91,7 @@ static void* create_chunk(const size_t size); static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; +static pid_t (*real_fork)()=NULL; static void* (*real_malloc)(size_t)=NULL; static void* (*real_calloc)(size_t,size_t)=NULL; static void* (*real_free)(size_t)=NULL; @@ -486,12 +487,13 @@ static void bigmaac_init(void) } fprintf(stderr,"Loading Bigmaac Heap X2! PID:%d PPID:%d\n",getpid(),getppid()); load_state=LOADING_MEM_FUNCS; + real_fork = dlsym(RTLD_NEXT, "fork"); real_malloc = dlsym(RTLD_NEXT, "malloc"); real_free = dlsym(RTLD_NEXT, "free"); real_calloc = dlsym(RTLD_NEXT, "calloc"); real_realloc = dlsym(RTLD_NEXT, "realloc"); real_reallocarray = dlsym(RTLD_NEXT, "reallocarray"); - if (!real_malloc || !real_free || !real_calloc || !real_realloc || !real_reallocarray) { + if (!real_malloc || !real_free || !real_calloc || !real_realloc || !real_reallocarray || !real_fork) { fprintf(stderr, "Error in `dlsym`: %s\n", dlerror()); } load_state=LOADING_LIBRARY; @@ -901,6 +903,17 @@ void free(void* ptr) { } } +pid_t fork() { + if(load_state==NOT_LOADED && real_malloc==NULL) { + bigmaac_init(); + } + + fprintf(stderr,"FORK!\n"); + assert(1==0); + pid_t r = real_fork(); + return r; +} + #ifdef MAIN #define T 32 #define N (4096*16) From 3361ee209cc015c40fa40f9f5e953938cdf7337d Mon Sep 17 00:00:00 2001 From: Misko Date: Mon, 6 Feb 2023 15:05:18 -0800 Subject: [PATCH 28/32] catch fork --- bigmaac.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index a8fcac6..09d1916 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -777,7 +777,7 @@ void *realloc(void * ptr, size_t size) } //currently managed by BigMaac if (ptr>=base_fries && ptr> address space reserved fr mmap if (chunks_removed==0) { @@ -909,7 +910,6 @@ pid_t fork() { } fprintf(stderr,"FORK!\n"); - assert(1==0); pid_t r = real_fork(); return r; } From 96ae01f23570a2148a8a4acdee08eefd5cdbee26 Mon Sep 17 00:00:00 2001 From: Misko Date: Mon, 6 Feb 2023 15:23:09 -0800 Subject: [PATCH 29/32] catch fork --- bigmaac.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigmaac.c b/bigmaac.c index 09d1916..cb91a97 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -818,7 +818,10 @@ void *realloc(void * ptr, size_t size) // move free space from next node to this one verify_memory(head,1); used_bigmaacs+=(size-n->size); + //update the next node n->next->size-=(size-n->size); + n->next->ptr=((char*)n->next->ptr)+(size-n->size); + //update current node n->size+=(size-n->size); //fix the free nodes place in the heap heapify_down(n->heap,n->next->heap_idx); From eef2eb71a69752a021bd788de0174f19f8ce215f Mon Sep 17 00:00:00 2001 From: Misko Date: Mon, 6 Feb 2023 20:42:30 -0800 Subject: [PATCH 30/32] catch fork --- bigmaac.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index cb91a97..e704596 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -342,7 +342,7 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { } size_t size = BIGMAAC_BUFFERED_SIZE(requested_size); // the actual size we are going to alloc - //update used metrics + //update used metrics if (head==_head_bigmaacs) { size=SIZE_TO_MULTIPLE(size,page_size); used_bigmaacs+=size; @@ -350,7 +350,7 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { size=SIZE_TO_MULTIPLE(size,fry_size_multiple); used_fries+=size; } - fprintf(stderr,"requested size %lu , size to allocate %lu\n",requested_size,size); + fprintf(stderr,"Heap pop split requested size %lu , size to allocate %lu\n",requested_size,size); heap * heap = head->heap; node ** node_array = heap->node_array; @@ -485,7 +485,7 @@ static void bigmaac_init(void) fprintf(stderr,"Already init %d\n",load_state); return; } - fprintf(stderr,"Loading Bigmaac Heap X2! PID:%d PPID:%d\n",getpid(),getppid()); + fprintf(stderr,"Loading Bigmaac Heap! PID:%d PPID:%d\n",getpid(),getppid()); load_state=LOADING_MEM_FUNCS; real_fork = dlsym(RTLD_NEXT, "fork"); real_malloc = dlsym(RTLD_NEXT, "malloc"); @@ -593,7 +593,7 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { return -1; } strcpy(filename,template); - fprintf(stderr,"BIGMAAC: make file %0.2f MB\n",((double)size)/(1024.0*1024.0)); + fprintf(stderr,"Bigmaac: make tmp file %0.2f MB\n",((double)size)/(1024.0*1024.0)); const int fd=mkstemp(filename); if (fd<0) { fprintf(stderr,"Bigmaac: Failed to make temp file %s\n", strerror(errno)); @@ -621,12 +621,6 @@ static int mmap_tmpfile(void * const ptr, const size_t size) { } active_mmaps++; - /*ret = close(fd);//mmap keeps the fd open now - if (ret==-1) { - fprintf(stderr,"BigMaac: close fd failed! %s\n", strerror(errno)); - return -1; - }*/ - return fd; } @@ -710,7 +704,7 @@ void *malloc(size_t size) } if (size>min_size_fry) { - fprintf(stderr,"MALLOC %lu\n",size); + fprintf(stderr,"Bigmaac: malloc() %luMB\n",size/(1024*1024)); #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif @@ -741,7 +735,7 @@ void *calloc(size_t count, size_t size) //library is loaded and count/size are reasonable if (size>min_size_fry) { - fprintf(stderr,"CALLOC\n"); + fprintf(stderr,"Bigmaac: calloc() %luMB\n",size/(1024*1024)); #ifdef BIGMAAC_SIGNAL kill(getpid(), SIGUSR1); #endif @@ -777,7 +771,7 @@ void *realloc(void * ptr, size_t size) } //currently managed by BigMaac if (ptr>=base_fries && ptrsize,size); + fprintf(stderr,"BigMaac: Realloc current %luMB vs new %luMB\n",n->size/(1024*1024),size/(1024*1024)); print_stats(); @@ -818,10 +812,10 @@ void *realloc(void * ptr, size_t size) // move free space from next node to this one verify_memory(head,1); used_bigmaacs+=(size-n->size); - //update the next node + //update the next node n->next->size-=(size-n->size); - n->next->ptr=((char*)n->next->ptr)+(size-n->size); - //update current node + n->next->ptr=((char*)n->next->ptr)+(size-n->size); + //update current node n->size+=(size-n->size); //fix the free nodes place in the heap heapify_down(n->heap,n->next->heap_idx); From 2fa2d458f38e0d3403ad078eaa349b8677287e32 Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 9 Feb 2023 07:59:10 -0800 Subject: [PATCH 31/32] add in flag for requested capacity --- bigmaac.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bigmaac.c b/bigmaac.c index e704596..f42e757 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -119,6 +119,7 @@ static size_t fry_size_multiple=DEFAULT_FRY_SIZE_MULTIPLE; static size_t used_fries=0; static size_t used_bigmaacs=0; +static size_t requested_bigmaacs=0; static size_t page_size = 0; static enum load_status load_state=NOT_LOADED; @@ -127,10 +128,11 @@ static enum load_status load_state=NOT_LOADED; static inline void verify_memory(node * head,int global); static inline void log_bm(const char *data, ...); static void print_stats() { - fprintf(stderr,"BigMaac: stats! mmap() [ active mmaps %d , bigmaac capacity free: %0.2f , fries capacity free: %0.2f, check /proc/sys/vm/max_map_count : %s\n", + fprintf(stderr,"BigMaac: stats! mmap() [ active mmaps %d , fires (free: %0.2f) , bigmaac (free: %0.2f, requested: %0.2f) check /proc/sys/vm/max_map_count : %s\n", active_mmaps, 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac, + 1.0-((float)requested_bigmaacs)/size_bigmaac, strerror(errno)); } @@ -346,6 +348,7 @@ static node * heap_pop_split(node * const head, const size_t requested_size) { if (head==_head_bigmaacs) { size=SIZE_TO_MULTIPLE(size,page_size); used_bigmaacs+=size; + requested_bigmaacs+=requested_size; } else { size=SIZE_TO_MULTIPLE(size,fry_size_multiple); used_fries+=size; @@ -675,6 +678,7 @@ static int remove_chunk_with_ptr(void * const ptr, void * const new_ptr, const s n->fd=-1; active_mmaps--; used_bigmaacs-=n->size; + requested_bigmaacs-=n->requested_size; } else { used_fries-=n->size; } @@ -796,9 +800,10 @@ void *realloc(void * ptr, size_t size) } + //if the current node cannot satisfy demand, check if stealing from next node can help if (n->next!=NULL && n->next->in_use==FREE && (n->sizesize+n->next->size)>=size) { fprintf(stderr,"Realloc handle #2\n"); - // check for equality + // fully consuming the next node is what we need to do if ((n->size+n->next->size)==size) { fprintf(stderr,"Realloc handle #2a\n"); //remove the node and swallow it @@ -806,12 +811,11 @@ void *realloc(void * ptr, size_t size) heap_remove_idx(n->heap, n->next->heap_idx); UNLINK(n->next); real_free((size_t)n->next); - verify_memory(head,1); } else { + //shave only a bit off the neighbor fprintf(stderr,"Realloc handle #2b\n"); // move free space from next node to this one verify_memory(head,1); - used_bigmaacs+=(size-n->size); //update the next node n->next->size-=(size-n->size); n->next->ptr=((char*)n->next->ptr)+(size-n->size); @@ -819,12 +823,16 @@ void *realloc(void * ptr, size_t size) n->size+=(size-n->size); //fix the free nodes place in the heap heapify_down(n->heap,n->next->heap_idx); - verify_memory(head,1); } + used_bigmaacs+=size-n->size; + requested_bigmaacs+=size-n->size; + verify_memory(head,1); } + //we are shinking in this case if (n->size>=size) { fprintf(stderr,"Realloc handle #1\n"); + requested_bigmaacs-=n->size-size; int ret = resize_node(n,size); if (ret!=0) { fprintf(stderr,"BigMaac: Failed to resize mmap\n"); From 0760e2efc2fefcfa7c0c616ed6a1a1c390c740dc Mon Sep 17 00:00:00 2001 From: Misko Date: Thu, 9 Feb 2023 08:06:08 -0800 Subject: [PATCH 32/32] add in flag for requested capacity --- bigmaac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigmaac.c b/bigmaac.c index f42e757..483f8ee 100644 --- a/bigmaac.c +++ b/bigmaac.c @@ -128,7 +128,7 @@ static enum load_status load_state=NOT_LOADED; static inline void verify_memory(node * head,int global); static inline void log_bm(const char *data, ...); static void print_stats() { - fprintf(stderr,"BigMaac: stats! mmap() [ active mmaps %d , fires (free: %0.2f) , bigmaac (free: %0.2f, requested: %0.2f) check /proc/sys/vm/max_map_count : %s\n", + fprintf(stderr,"BigMaac: stats! mmap() [ active mmaps %d , fires (free: %0.2f) , bigmaac (free: %0.2f, requested free: %0.2f) check /proc/sys/vm/max_map_count : %s\n", active_mmaps, 1.0-((float)used_fries)/size_fries, 1.0-((float)used_bigmaacs)/size_bigmaac,