From e77d437906f40ab14a7a3b35ac2cd4437cb22b87 Mon Sep 17 00:00:00 2001 From: Davidson Souza Date: Tue, 14 Nov 2023 20:27:17 -0300 Subject: [PATCH] feat: delete --- makefile | 2 +- src/flat_file.h | 14 +++- src/flat_file_impl.h | 156 +++++++++++++++++++++++----------------- src/forest_test.c | 16 ++--- src/map_forest_impl.h | 38 +++++++--- src/mmap_forest.c | 1 - src/test_flat_file.c | 1 + src/test_util_methods.c | 4 +- src/test_utils.h | 4 +- src/util.h | 14 ++-- 10 files changed, 151 insertions(+), 99 deletions(-) diff --git a/makefile b/makefile index 98e7e58..9fb1a52 100644 --- a/makefile +++ b/makefile @@ -1,5 +1,5 @@ CC=gcc -CFLAGS=-Wall -We -g -pedantic -std=c99 +CFLAGS=-Wall -W -g -pedantic -std=c99 all: build-release diff --git a/src/flat_file.h b/src/flat_file.h index 21b708b..a52e21b 100644 --- a/src/flat_file.h +++ b/src/flat_file.h @@ -69,12 +69,20 @@ struct utreexo_forest_page_header { /* Our internal representation of a file, this struct doesn't get persisted on * our file, it just keep pointers to the actual stuff at runtime. */ struct utreexo_forest_file { + struct utreexo_forest_file_header *header; const char *filename; - char *map; // The actual map - struct utreexo_forest_page_header *wrt_page; // Which page are we on + char *map; // The actual map int fd; - uint64_t filesize; +} __attribute__((__packed__)); + +/* Things we need to keep through different sessions, they are persisted at the + * beginning of a file + * */ +struct utreexo_forest_file_header { + uint64_t magic; + struct utreexo_forest_page_header *wrt_page; // Which page are we on uint32_t n_pages; + uint64_t filesize; utreexo_forest_free_page *fpg; // The first free page } __attribute__((__packed__)); diff --git a/src/flat_file_impl.h b/src/flat_file_impl.h index a23db4e..419ccc4 100644 --- a/src/flat_file_impl.h +++ b/src/flat_file_impl.h @@ -14,11 +14,7 @@ #include "forest_node.h" #include "util.h" -#ifndef TEST -#define MAP_ORIGIN (void *)(1 << 10) -#else -#define MAP_ORIGIN NULL -#endif +#define MAP_ORIGIN (void *)(1 << 20) #ifdef TEST #define MAP_SIZE 1024 @@ -29,81 +25,108 @@ int posix_fallocate(int fd, off_t offset, off_t len); static inline void utreexo_forest_file_close(struct utreexo_forest_file *file) { - munmap(file->map, file->filesize); + munmap(file->map - sizeof(struct utreexo_forest_file_header), + file->header->filesize); close(file->fd); free(file); } static inline void utreexo_forest_file_init(struct utreexo_forest_file **file, const char *filename) { + debug_print("Openning file %s\n", filename); - int fd = 0; - - *file = - (struct utreexo_forest_file *)malloc(sizeof(struct utreexo_forest_file)); + int fd = open(filename, O_RDWR | O_CREAT, 0644); - fd = open(filename, O_RDWR | O_CREAT, 0644); if (fd < 0) { perror("open"); exit(1); } + struct utreexo_forest_file *pfile = + (struct utreexo_forest_file *)malloc(sizeof(struct utreexo_forest_file)); + if (pfile == NULL) { + perror("malloc"); + exit(1); + } + const int fsize = lseek(fd, 0, SEEK_END); char *data = (char *)mmap(MAP_ORIGIN, MAP_SIZE, PROT_READ | PROT_WRITE | PROT_GROWSUP, - MAP_FILE | MAP_SHARED, fd, 0); + MAP_FILE | MAP_SHARED | MAP_FIXED, fd, 0); - if (data == MAP_FAILED) { + if (data == MAP_FAILED || data != MAP_ORIGIN) { perror("mmap"); exit(1); } - (*file)->wrt_page = (struct utreexo_forest_page_header *)(data + sizeof(int)); + debug_print("File mapped to %p\n", data); + const size_t header_size = sizeof(struct utreexo_forest_file_header); + + pfile->map = data + header_size; + pfile->header = (struct utreexo_forest_file_header *)data; + pfile->filename = filename; + pfile->fd = fd; - (*file)->fd = fd; - (*file)->filename = filename; - (*file)->filesize = fsize; - (*file)->n_pages = 0; - (*file)->map = data + sizeof(int); - (*file)->fpg = NULL; + const struct utreexo_forest_file_header *pheader = + (struct utreexo_forest_file_header *)data; /* This is a new file, we need to initialize at least the first page */ - if (fsize < 4 || *(int *)data != FILE_MAGIC) { - DEBUG_PRINT("No pages found, creating new file\n"); - posix_fallocate(fd, 0, 4); - utreexo_forest_page_alloc(*file); - *(int *)data = FILE_MAGIC; - return; + if (fsize < 4 || pheader->magic != FILE_MAGIC) { + debug_print("No pages found, creating new file\n"); + + posix_fallocate(fd, 0, header_size); + + pfile->header->filesize = header_size; + pfile->header->n_pages = 0; + pfile->header->fpg = NULL; + pfile->header->magic = FILE_MAGIC; + pfile->header->wrt_page = + (struct utreexo_forest_page_header *)(data + header_size); } - (*file)->n_pages = 1; - (*file)->wrt_page = (struct utreexo_forest_page_header *)(data + 4); - DEBUG_PRINT("Found %d pages\n", (*file)->n_pages); + + if (pheader->n_pages == 0) { + utreexo_forest_page_alloc(pfile); + } + + debug_print("Found %d pages writting in %p\n", pfile->header->n_pages, + pfile->header->wrt_page); + *file = pfile; } static inline int utreexo_forest_page_alloc(struct utreexo_forest_file *file) { - DEBUG_PRINT("Grabbing a new page\n"); - if (file->fpg != NULL) { - DEBUG_PRINT("Found a free page"); - utreexo_forest_free_page *nhead = file->fpg->next; - file->wrt_page = (struct utreexo_forest_page_header *)file->fpg; - file->fpg = nhead; + debug_print("Grabbing a new page\n"); + // We have a free page + if (file->header->fpg != NULL) { + debug_print("Found a free page"); + utreexo_forest_free_page *nhead = file->header->fpg->next; + file->header->wrt_page = + (struct utreexo_forest_page_header *)file->header->fpg; + file->header->fpg = nhead; + file->header->n_pages++; return EXIT_SUCCESS; } - DEBUG_PRINT("Creating a new page\n"); - const int page_offset = file->n_pages; - file->n_pages++; - file->filesize += PAGE_SIZE; - posix_fallocate(file->fd, file->filesize, (PAGE_SIZE)*file->n_pages); + + // We need to create a new one + debug_print("Creating a new page\n"); + + const int page_offset = file->header->n_pages; + file->header->n_pages++; + file->header->filesize += PAGE_SIZE; + + posix_fallocate(file->fd, file->header->filesize, + (PAGE_SIZE)*file->header->n_pages); char *pg = (((char *)file->map) + PAGE_SIZE * page_offset); - file->wrt_page = (struct utreexo_forest_page_header *)pg; - - utreexo_forest_mkpg(file, file->wrt_page); - DEBUG_PRINT("Allocated page %d\n", page_offset); - DEBUG_ASSERT(file->wrt_page->n_nodes == 0) - DEBUG_ASSERT(file->wrt_page->pg_magic == MAGIC) - DEBUG_ASSERT(file->n_pages == page_offset + 1) - return EXIT_SUCCESS; + file->header->wrt_page = (struct utreexo_forest_page_header *)pg; + + utreexo_forest_mkpg(file, file->header->wrt_page); + + debug_print("Allocated page %d\n", page_offset); + debug_assert(file->header->wrt_page->n_nodes == 0); + debug_assert(file->header->wrt_page->pg_magic == MAGIC); + debug_assert(file->header->n_pages == page_offset + 1); + + return EXIT_SUCCESS; } static inline void utreexo_forest_mkpg(struct utreexo_forest_file *file, @@ -111,27 +134,27 @@ static inline void utreexo_forest_mkpg(struct utreexo_forest_file *file, pg->pg_magic = MAGIC; pg->n_nodes = 0; - DEBUG_PRINT("Initialized page %d\n", file->n_pages - 1); - DEBUG_ASSERT(pg->n_nodes == 0) - DEBUG_ASSERT(pg->pg_magic == MAGIC) + debug_print("Initialized page %d\n", file->header->n_pages - 1); + debug_assert(pg->n_nodes == 0) debug_assert(pg->pg_magic == MAGIC) } static inline utreexo_forest_node * utreexo_forest_file_node_alloc(struct utreexo_forest_file *file) { - uint64_t page_nodes = file->wrt_page->n_nodes; + uint64_t page_nodes = file->header->wrt_page->n_nodes; if (page_nodes == NODES_PER_PAGE) { - DEBUG_PRINT("Page is full, allocating new page\n"); + debug_print("Page is full, allocating new page\n"); if (utreexo_forest_page_alloc(file)) { fprintf(stderr, "Failed to allocate page\n"); exit(1); } page_nodes = 0; // we've just created a new page } - DEBUG_PRINT("Writing node %d to page %d offset=%d\n", page_nodes, - file->n_pages - 1, page_nodes / sizeof(utreexo_forest_node)); + debug_print("Writing node %d to page %d offset=%d\n", page_nodes, + file->header->n_pages - 1, + page_nodes / sizeof(utreexo_forest_node)); utreexo_forest_node *ptr = - (utreexo_forest_node *)((char *)file->wrt_page + 16) + page_nodes; - ++(file->wrt_page->n_nodes); + (utreexo_forest_node *)((char *)file->header->wrt_page + 16) + page_nodes; + ++(file->header->wrt_page->n_nodes); return ptr; } @@ -143,23 +166,28 @@ utreexo_forest_file_node_del(struct utreexo_forest_file *file, struct utreexo_forest_page_header *pg = (struct utreexo_forest_page_header *)PAGE(file->map, npage); - DEBUG_PRINT("Deleting node from page %d remaining: %u\n", npage, pg->n_nodes); - DEBUG_ASSERT(pg->n_nodes != 0); - DEBUG_ASSERT(file->n_pages >= npage); + debug_print("Deleting node from page %d remaining: %u\n", npage, pg->n_nodes); + debug_assert(pg->n_nodes != 0); + debug_assert(file->header->n_pages > npage); if (--pg->n_nodes == 0) { - DEBUG_PRINT("Deallocating page %d\n", npage); - utreexo_forest_free_page *pg = file->fpg; + debug_print("Deallocating page %d\n", npage); + --file->header->n_pages; + utreexo_forest_free_page *pg = file->header->fpg; + utreexo_forest_free_page *npg = + (utreexo_forest_free_page *)PAGE(file->map, npage); + npg->next = NULL; + // This is the first free page if (pg == NULL) { - file->fpg = (utreexo_forest_free_page *)PAGE(file->map, npage); + file->header->fpg = npg; return; } // Walk the list until find the last element while (pg->next != NULL) pg = (utreexo_forest_free_page *)pg->next; - pg->next = (utreexo_forest_free_page *)PAGE(file->map, npage); + pg->next = npg; } } #endif diff --git a/src/forest_test.c b/src/forest_test.c index bbe7b6c..8f3fb92 100644 --- a/src/forest_test.c +++ b/src/forest_test.c @@ -232,25 +232,25 @@ void test_from_test_cases(void) { void test_grab_node() { struct utreexo_forest_file *file = NULL; utreexo_forest_file_init(&file, "test_grab_node.bin"); - + const unsigned char *expected = (unsigned char[]){ + 0xe7, 0x7b, 0x9a, 0x9a, 0xe9, 0xe3, 0x0b, 0x0d, 0xbd, 0xb6, 0xf5, + 0x10, 0xa2, 0x64, 0xef, 0x9d, 0xe7, 0x81, 0x50, 0x1d, 0x7b, 0x6b, + 0x92, 0xae, 0x89, 0xeb, 0x05, 0x9c, 0x5a, 0xb7, 0x43, 0xdb}; struct utreexo_forest p = { .data = file, .roots = {0}, .nLeaf = 0, }; - for (size_t i = 0; i < 15; ++i) { + for (size_t i = 0; i < 8; ++i) { utreexo_node_hash leaf = {.hash = {0}}; hash_from_u8(leaf.hash, i); utreexo_forest_add(&p, leaf); } utreexo_forest_node *node = NULL, *sibling = NULL, *parent = NULL; - grab_node(&p, &node, &sibling, &parent, 4); - for (size_t i = 0; i < 32; ++i) - printf("%02x", node->hash.hash[i]); - - printf("\n"); + grab_node(&p, &node, &sibling, &parent, 5); + ASSERT_ARRAY_EQ(node->hash.hash, expected, 32); } int main() { @@ -259,6 +259,6 @@ int main() { test_add_two(); test_add_many(); test_from_test_cases(); - // test_grab_node(); + test_grab_node(); return 0; } diff --git a/src/map_forest_impl.h b/src/map_forest_impl.h index 5e8131b..d7332dd 100644 --- a/src/map_forest_impl.h +++ b/src/map_forest_impl.h @@ -18,6 +18,7 @@ #include "util.h" static const char UTREEXO_ZERO_HASH[32] = {0}; + static inline void utreexo_forest_add(struct utreexo_forest *p, utreexo_node_hash leaf) { utreexo_forest_node *pnode = utreexo_forest_file_node_alloc(p->data); @@ -30,7 +31,7 @@ static inline void utreexo_forest_add(struct utreexo_forest *p, while ((nLeaves >> height & 1) == 1) { utreexo_forest_node *root = p->roots[height]; - DEBUG_ASSERT(root != NULL); + debug_assert(root != NULL); p->roots[height] = NULL; if (memcmp(root->hash.hash, UTREEXO_ZERO_HASH, 32) == 0) { break; @@ -48,7 +49,7 @@ static inline void utreexo_forest_add(struct utreexo_forest *p, pnode = proot; height++; } - DEBUG_ASSERT(p->roots[height] == NULL); + debug_assert(p->roots[height] == NULL); p->roots[height] = pnode; ++p->nLeaf; } @@ -59,24 +60,32 @@ static inline void grab_node(struct utreexo_forest *f, utreexo_forest_node **parent, uint64_t pos) { node_offset offset = detect_offset(pos, f->nLeaf); - DEBUG_ASSERT(offset.tree < 64); + debug_assert(offset.tree < 64); utreexo_forest_node *pnode = f->roots[offset.tree]; utreexo_forest_node *psibling = NULL; utreexo_forest_node *pparent = NULL; + if (offset.depth == 0) { + *node = pnode; + *sibling = psibling; + *parent = pparent; - for (size_t h = 0; h < offset.depth; ++h) { - if (pnode->right_child == NULL) - break; + return; + } + for (size_t h = offset.depth; h > 0; --h) { + uint32_t mask = 1 << (h - 1); pparent = pnode; - if ((offset.bits >> h & 1) == 1) { + if (pnode->right_child == NULL) + break; + + if (mask & pos) { pnode = pparent->right_child; psibling = pparent->left_child; } else { psibling = pparent->right_child; - pnode = pparent->right_child; + pnode = pparent->left_child; } } @@ -85,14 +94,21 @@ static inline void grab_node(struct utreexo_forest *f, *parent = pparent; } -static inline void utreexo_forest_remove(struct utreexo_forest *p, - uint64_t leaf_number) {} - static inline void utreexo_forest_free(struct utreexo_forest *forest) { utreexo_forest_file_close(forest->data); free(forest); } +static inline void recompute_parent_hash(struct utreexo_forest *f, + utreexo_forest_node *origin) { + utreexo_forest_node *pnode = origin->parent; + while (pnode != NULL) { + parent_hash(pnode->hash.hash, pnode->left_child->hash.hash, + pnode->right_child->hash.hash); + pnode = pnode->parent; + } +} + static inline void delete_single(struct utreexo_forest *f, uint64_t pos) { utreexo_forest_node *pnode, *psibling, *pparent; diff --git a/src/mmap_forest.c b/src/mmap_forest.c index a13649e..f5b3ad1 100644 --- a/src/mmap_forest.c +++ b/src/mmap_forest.c @@ -34,6 +34,5 @@ void utreexo_forest_init(struct utreexo_forest **p, const char *filename) { forest->data = file; forest->nLeaf = 0; - DEBUG_ASSERT(*(uint32_t *)(*file).map == MAGIC); *p = forest; } diff --git a/src/test_flat_file.c b/src/test_flat_file.c index f0f844a..c880a16 100644 --- a/src/test_flat_file.c +++ b/src/test_flat_file.c @@ -78,6 +78,7 @@ utreexo_forest_node *test_create_nodes(struct utreexo_forest_file *file) { .left_child = left_child_pos, .right_child = right_child_pos, }; + TEST_END; return parent_pos; diff --git a/src/test_util_methods.c b/src/test_util_methods.c index afedfca..0a9e7ad 100644 --- a/src/test_util_methods.c +++ b/src/test_util_methods.c @@ -20,8 +20,8 @@ void test_detect_row() { void test_detect_offset() { node_offset offset = detect_offset(0, 3); - ASSERT_EQ(offset.depth, 2); - ASSERT_EQ(offset.tree, 0); + ASSERT_EQ(offset.depth, 1); + ASSERT_EQ(offset.tree, 1); ASSERT_EQ(offset.bits, 1); } diff --git a/src/test_utils.h b/src/test_utils.h index b1859b0..489161a 100644 --- a/src/test_utils.h +++ b/src/test_utils.h @@ -2,8 +2,8 @@ do { \ if (a != b) { \ printf("Assertion failed: %s != %s\n", #a, #b); \ - printf(" %s = %lu\n", #a, (unsigned long)a); \ - printf(" %s = %lu\n", #b, (unsigned long)b); \ + printf(" %s = %08x\n", #a, (unsigned int)a); \ + printf(" %s = %08x\n", #b, (unsigned int)b); \ exit(1); \ } \ } while (0) diff --git a/src/util.h b/src/util.h index a7203b1..64c96ee 100644 --- a/src/util.h +++ b/src/util.h @@ -3,18 +3,18 @@ #include #ifdef DEBUG -#define DEBUG_PRINT(...) \ +#define debug_print(...) \ do { \ fprintf(stderr, __VA_ARGS__); \ } while (0) -#define DEBUG_ASSERT(x) \ +#define debug_assert(x) \ if (!(x)) { \ fprintf(stderr, "Assertion failed: %s\n", #x); \ exit(1); \ } #else -#define DEBUG_PRINT(...) -#define DEBUG_ASSERT(x) +#define debug_print(...) +#define debug_assert(x) #endif #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) @@ -67,9 +67,9 @@ static inline node_offset detect_offset(uint64_t pos, uint64_t num_leaves) { tr -= 1; } return (node_offset){ - .bits = !marker, - .tree = bigger_trees, - .depth = (uint8_t)tr - nr, + .bits = pos, + .tree = (uint8_t)bigger_trees, + .depth = (uint8_t)(tr - nr), }; }