From 0c8d74051fec48ca0c7389f4a34d95b5f4d5ed06 Mon Sep 17 00:00:00 2001 From: kakwa Date: Mon, 11 Mar 2024 23:40:55 +0100 Subject: [PATCH 01/24] properly unpack the header and handle its endianess --- FORMAT.md | 4 ++-- inc/wows-depack.h | 1 + lib/debug.c | 2 +- lib/deflate.c | 2 +- lib/index-parser.c | 50 +++++++++++++++++++++++++++++++++------ lib/memory-helpers.c | 1 + lib/wows-depack-private.h | 10 ++++---- tests/tests.c | 10 +++++--- 8 files changed, 62 insertions(+), 18 deletions(-) diff --git a/FORMAT.md b/FORMAT.md index 4ae4a06..d151ee1 100644 --- a/FORMAT.md +++ b/FORMAT.md @@ -94,7 +94,7 @@ The index file is composed of 5 sections: +====+====+====+====++====+====+====+====++====+====+====+====++====+====+====+====+ | MA | MA | MA | MA || 00 | 00 | 00 | 02 || ID | ID | ID | ID || 40 | 00 | 00 | 00 | +====+====+====+====++====+====+====+====++====+====+====+====++====+====+====+====+ -|<----- magic ----->||<--- unknown_1 --->||<------- id ------>||<--- unknown_2 --->| +|<----- magic ----->||<--- endianess --->||<------- id ------>||<--- unknown_2 --->| | 32 bits || 32 bits || 32 bits || 32 bits | +====+====+====+====++====+====+====+====++====+====+====+====++====+====+====+====+ @@ -121,7 +121,7 @@ The index file is composed of 5 sections: | Field | size | Description | |----------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------| | `magic` | 32 bits | Magic bytes, always "ISFP" | -| `unknown_1` | 32 bits | Unknown, always 0x2000000 | +| `endianess` | 32 bits | Endianess marker, always 0x2000000 if LE | | `id` | 32 bits | Unsure, unique per index file, not referenced anywhere else | | `unknown_2` | 32 bits | Unknown, always 0x40, maybe some offset | | `file_dir_count` | 32 bits | Number of files + directories (Nfd), also number of entries in the metadata section and the file names section | diff --git a/inc/wows-depack.h b/inc/wows-depack.h index ce45933..0cd2b6a 100644 --- a/inc/wows-depack.h +++ b/inc/wows-depack.h @@ -61,6 +61,7 @@ typedef struct { void **indexes; // Array of structures representing each index file (WOWS_INDEX // **indexes;) uint32_t index_count; // Size of the array + bool is_le; // Flag for endianess (true if LE, false if BE) char *err_msg; // Last error message } WOWS_CONTEXT; diff --git a/lib/debug.c b/lib/debug.c index e17f05c..bb41982 100644 --- a/lib/debug.c +++ b/lib/debug.c @@ -8,7 +8,7 @@ int print_header(WOWS_INDEX_HEADER *header) { printf("* magic: %.4s\n", (char *)&header->magic); - printf("* unknown_1: 0x%x\n", header->unknown_1); + printf("* endianess: 0x%x\n", header->endianess); printf("* id: 0x%x\n", header->id); printf("* unknown_2: 0x%x\n", header->unknown_2); printf("* file_dir_count: %u\n", header->file_dir_count); diff --git a/lib/deflate.c b/lib/deflate.c index acb55d9..0dd52c5 100644 --- a/lib/deflate.c +++ b/lib/deflate.c @@ -254,7 +254,7 @@ int wows_write_pkg(WOWS_CONTEXT *context, char *in_path, char *name_pkg, FILE *p // Setup header memcpy(index->header->magic, "ISFP", 4); - index->header->unknown_1 = 0x2000000; + index->header->endianess = 0x2000000; index->header->id = (uint32_t)rand(); index->header->unknown_2 = 0x40; index->header->file_dir_count = writer->file_plus_dir_count; diff --git a/lib/index-parser.c b/lib/index-parser.c index 60fe56c..12411fd 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -1,4 +1,5 @@ #define _POSIX_C_SOURCE 200809L +#define _DEFAULT_SOURCE #define _XOPEN_SOURCE 500 // TODO clean-up this mess #include @@ -17,6 +18,7 @@ #include #include #include +#include #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) #include @@ -102,27 +104,61 @@ WOWS_CONTEXT *wows_init_context(uint8_t debug_level) { return context; } +uint32_t datatoh32(char *data, size_t offset, WOWS_CONTEXT *context) { + uint32_t *ret = (uint32_t *)(data + offset); + if (context->is_le) { + return le32toh(*ret); + } else { + return be32toh(*ret); + } +} + +uint64_t datatoh64(char *data, size_t offset, WOWS_CONTEXT *context) { + uint64_t *ret = (uint64_t *)(data + offset); + if (context->is_le) { + return le64toh(*ret); + } else { + return be64toh(*ret); + } +} + // Map the different section of the index file content to an index struct -int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in) { +int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context) { WOWS_INDEX *index = calloc(sizeof(WOWS_INDEX), 1); index->start_address = contents; index->end_address = contents + length; index->length = length; - // Get the header section - WOWS_INDEX_HEADER *header = (WOWS_INDEX_HEADER *)contents; - index->header = header; // Check header section boundaries - returnOutIndex((char *)header, contents + sizeof(WOWS_INDEX_HEADER), index); + returnOutIndex((char *)contents, contents + SIZE_WOWS_INDEX_HEADER, index); + + // Allocate an header section + WOWS_INDEX_HEADER *header = (WOWS_INDEX_HEADER *)calloc(sizeof(WOWS_INDEX_HEADER), 1); + index->header = header; + strncpy(header->magic, contents, 4); // Check that the magic do match if (strncmp(header->magic, "ISFP", 4) != 0) { return WOWS_ERROR_BAD_MAGIC; } + // Recover the endianess marker and set endianess in context + memcpy(&(header->endianess), contents + 4, sizeof(uint32_t)); + context->is_le = (header->endianess == 0x2000000); + + // Extract the header data + header->id = datatoh32(contents, 8, context); + header->unknown_2 = datatoh32(contents, 12, context); + header->file_dir_count = datatoh32(contents, 16, context); + header->file_count = datatoh32(contents, 20, context); + header->unknown_3 = datatoh32(contents, 24, context); + header->header_size = datatoh64(contents, 32, context); + header->offset_idx_data_section = datatoh64(contents, 40, context); + header->offset_idx_footer_section = datatoh64(contents, 48, context); + // Get the start of the metadata array WOWS_INDEX_METADATA_ENTRY *metadatas; - metadatas = (WOWS_INDEX_METADATA_ENTRY *)(contents + sizeof(WOWS_INDEX_HEADER)); + metadatas = (WOWS_INDEX_METADATA_ENTRY *)(contents + SIZE_WOWS_INDEX_HEADER); index->metadata = metadatas; // Check metadatas section boundaries returnOutIndex((char *)metadatas, (char *)&metadatas[header->file_dir_count], index); @@ -243,7 +279,7 @@ int wows_parse_index_buffer(char *contents, size_t length, const char *index_fil int i; WOWS_INDEX *index; - int err = map_index_file(contents, length, &index); + int err = map_index_file(contents, length, &index, context); if (err != 0) { return err; } diff --git a/lib/memory-helpers.c b/lib/memory-helpers.c index 989b602..ce5cf0a 100644 --- a/lib/memory-helpers.c +++ b/lib/memory-helpers.c @@ -32,6 +32,7 @@ int wows_free_context_internal(WOWS_CONTEXT *context, int flag) { close(index->fd); } free(index->index_file_path); + free(index->header); free(index); } diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index dddb6da..b04dd74 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -3,8 +3,6 @@ #include #include "hashmap.h" -#pragma pack(1) - /* Limits */ // Maximum number of directories in a path (protection against infinite loops) @@ -13,6 +11,8 @@ // Put some limits on the file name size #define WOWS_PATH_MAX 4096 +#pragma pack(1) + /* ---------- */ /* Errors */ @@ -31,7 +31,7 @@ // INDEX file header typedef struct { char magic[4]; - uint32_t unknown_1; + uint32_t endianess; uint32_t id; uint32_t unknown_2; uint32_t file_dir_count; @@ -42,6 +42,8 @@ typedef struct { uint64_t offset_idx_footer_section; } WOWS_INDEX_HEADER; +#define SIZE_WOWS_INDEX_HEADER 56 + // INDEX file metadata entry typedef struct { uint64_t file_name_size; @@ -149,7 +151,7 @@ int get_metadata_filename(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, c int get_metadata_filename_unsafe(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, char **out); int get_footer_filename(WOWS_INDEX_FOOTER *footer, WOWS_INDEX *index, char **out); int wows_parse_index(char *index_file_path, WOWS_CONTEXT *context); -int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in); +int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context); char *wows_render_str(char *fmt, ...); void wows_set_error_details(WOWS_CONTEXT *context, char *fmt, ...); int print_header(WOWS_INDEX_HEADER *header); diff --git a/tests/tests.c b/tests/tests.c index 543db04..d4a455b 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -203,7 +203,8 @@ static void test_map_index_file() { memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(WOWS_INDEX_FOOTER)); // Test map_index_file function - int result = map_index_file(contents, TEST_DATA_SIZE, &index); + WOWS_CONTEXT *context = calloc(sizeof(WOWS_CONTEXT), 1); + int result = map_index_file(contents, TEST_DATA_SIZE, &index, context); CU_ASSERT_EQUAL(result, 0); CU_ASSERT_PTR_NOT_NULL(index); @@ -248,6 +249,7 @@ static void test_map_index_file() { CU_ASSERT_EQUAL(index->footer->id, 42); free(index); + free(context); } void test_wows_parse_index_buffer() { @@ -343,7 +345,7 @@ void test_wows_parse_index_buffer() { void test_wows_dump_index_to_file(void) { WOWS_INDEX_HEADER header = {.magic = "WoWS", - .unknown_1 = 0x10000, + .endianess = 0x20000, .id = 0x12345678, .unknown_2 = 0x20000, .file_dir_count = 2, @@ -752,7 +754,8 @@ void test_get_pkg_filepath() { memcpy(contents + MAGIC_SECTION_OFFSET + 1024 + sizeof(WOWS_INDEX_FOOTER), "foot", 5); WOWS_INDEX *index; - map_index_file(contents, 2048, &index); + WOWS_CONTEXT *context = wows_init_context(0); + map_index_file(contents, 2048, &index, context); index->index_file_path = "/path/to/wows/bin/6831266/idx/basecontent.idx"; char *out; @@ -764,6 +767,7 @@ void test_get_pkg_filepath() { CU_ASSERT_STRING_EQUAL("/path/to/wows/res_packages/foot", out); free(out); + wows_free_context(context); free(contents); free(index); } From d955721a3260364a2c7ac4d4e595c9ef9c25b4ae Mon Sep 17 00:00:00 2001 From: kakwa Date: Wed, 13 Mar 2024 01:54:36 +0100 Subject: [PATCH 02/24] complete properly putting index in memory --- lib/debug.c | 39 ++++--------------- lib/deflate.c | 3 +- lib/index-parser.c | 81 +++++++++++++++++++++++++-------------- lib/inode.c | 16 ++------ lib/memory-helpers.c | 7 ++++ lib/wows-depack-private.h | 8 ++++ 6 files changed, 80 insertions(+), 74 deletions(-) diff --git a/lib/debug.c b/lib/debug.c index bb41982..06fefd0 100644 --- a/lib/debug.c +++ b/lib/debug.c @@ -63,13 +63,8 @@ int print_debug_raw(WOWS_INDEX *index) { WOWS_INDEX_METADATA_ENTRY *entry = &metadatas[i]; printf("Metadata entry [%d]:\n", i); print_metadata_entry(entry); - char *filename; - int ret = get_metadata_filename(entry, index, &filename); - if (ret == 0) { - printf("* filename: %.*s\n", (int)entry->file_name_size, filename); - } else { - printf("* pkg filename: Error %d\n", ret); - } + char *filename = entry->_file_name; + printf("* filename: %.*s\n", (int)entry->file_name_size, filename); printf("\n"); } @@ -85,13 +80,8 @@ int print_debug_raw(WOWS_INDEX *index) { printf("Index Footer Content:\n"); print_footer(footer); - char *pkg_filename; - int ret = get_footer_filename(footer, index, &pkg_filename); - if (ret == 0) { - printf("* pkg filename: %.*s\n", (int)footer->pkg_file_name_size, pkg_filename); - } else { - printf("* pkg filename: Error %d\n", ret); - } + char *pkg_filename = footer->_file_name; + printf("* pkg filename: %.*s\n", (int)footer->pkg_file_name_size, pkg_filename); printf("\n"); return 0; } @@ -113,14 +103,8 @@ int print_debug_files(WOWS_INDEX *index, struct hashmap *map) { printf("File entry [%d]:\n", i); print_data_file_entry(fentry); print_metadata_entry(mentry); - char *filename; - int ret = get_metadata_filename(mentry, index, &filename); - - if (ret == 0) { - printf("* filename: %.*s\n", (int)mentry->file_name_size, filename); - } else { - printf("* pkg filename: Error %d\n", ret); - } + char *filename = mentry->_file_name; + printf("* filename: %.*s\n", (int)mentry->file_name_size, filename); WOWS_INDEX_METADATA_ENTRY *m_parent_entry_search = &(WOWS_INDEX_METADATA_ENTRY){.id = mentry->parent_id}; WOWS_INDEX_METADATA_ENTRY **mparent_entry = (WOWS_INDEX_METADATA_ENTRY **)hashmap_get(map, &m_parent_entry_search); @@ -128,15 +112,8 @@ int print_debug_files(WOWS_INDEX *index, struct hashmap *map) { while (mparent_entry != NULL && level < WOWS_DIR_MAX_LEVEL) { printf("parent [%d]:\n", level); print_metadata_entry(*mparent_entry); - char *filename; - int ret = get_metadata_filename(*mparent_entry, index, &filename); - - if (ret == 0) { - printf("* filename: %.*s\n", (int)mentry->file_name_size, filename); - } else { - printf("* pkg filename: Error %d\n", ret); - } - + char *filename = (*mparent_entry)->_file_name; + printf("* filename: %.*s\n", (int)mentry->file_name_size, filename); mentry_search = &(WOWS_INDEX_METADATA_ENTRY){.id = (*mparent_entry)->parent_id}; mparent_entry = NULL; mparent_entry = (WOWS_INDEX_METADATA_ENTRY **)hashmap_get(map, &mentry_search); diff --git a/lib/deflate.c b/lib/deflate.c index 0dd52c5..7e13166 100644 --- a/lib/deflate.c +++ b/lib/deflate.c @@ -356,8 +356,7 @@ int wows_dump_index_to_file(WOWS_INDEX *index, FILE *f) { // Write the file name strings for (size_t i = 0; i < index->header->file_dir_count; i++) { WOWS_INDEX_METADATA_ENTRY *metadata_entry = &index->metadata[i]; - char *file_name; - get_metadata_filename_unsafe(metadata_entry, index, &file_name); + char *file_name = metadata_entry->_file_name; fwrite(file_name, metadata_entry->file_name_size, 1, f); general_offset += metadata_entry->file_name_size; } diff --git a/lib/index-parser.c b/lib/index-parser.c index 12411fd..df8214e 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -33,9 +33,7 @@ #include "hashmap.h" int get_metadata_filename_unsafe(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, char **out) { - char *filename = (char *)entry; - filename += entry->offset_idx_file_name; - *out = filename; + *out = entry->_file_name; return 0; } @@ -124,6 +122,7 @@ uint64_t datatoh64(char *data, size_t offset, WOWS_CONTEXT *context) { // Map the different section of the index file content to an index struct int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context) { + int i; WOWS_INDEX *index = calloc(sizeof(WOWS_INDEX), 1); index->start_address = contents; index->end_address = contents + length; @@ -157,25 +156,63 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO header->offset_idx_footer_section = datatoh64(contents, 48, context); // Get the start of the metadata array - WOWS_INDEX_METADATA_ENTRY *metadatas; - metadatas = (WOWS_INDEX_METADATA_ENTRY *)(contents + SIZE_WOWS_INDEX_HEADER); - index->metadata = metadatas; - // Check metadatas section boundaries - returnOutIndex((char *)metadatas, (char *)&metadatas[header->file_dir_count], index); + char *metadata_section = (contents + SIZE_WOWS_INDEX_HEADER); + // Check the metadata section + returnOutIndex(metadata_section, metadata_section + header->file_dir_count * SIZE_WOWS_INDEX_METADATA_ENTRY, index); + + WOWS_INDEX_METADATA_ENTRY *metadata = calloc(sizeof(WOWS_INDEX_METADATA_ENTRY), header->file_dir_count); + index->metadata = metadata; + for (i = 0; i < header->file_dir_count; i++) { + size_t offset = i * SIZE_WOWS_INDEX_METADATA_ENTRY; + metadata[i].file_name_size = datatoh64(metadata_section + offset, 0, context); + metadata[i].offset_idx_file_name = datatoh64(metadata_section + offset, 8, context); + metadata[i].id = datatoh64(metadata_section + offset, 16, context); + metadata[i].parent_id = datatoh64(metadata_section + offset, 24, context); + + char *file_name = calloc(sizeof(char), metadata[i].file_name_size); + char *file_name_src = metadata_section + offset + metadata[i].offset_idx_file_name; + returnOutIndex(file_name_src, file_name_src + metadata[i].offset_idx_file_name, index); + strncpy(file_name, file_name_src, metadata[i].file_name_size); + metadata[i]._file_name = file_name; + } // Get the start pkg data pointer section + char *data_file_entry_section = (contents + header->offset_idx_data_section + MAGIC_SECTION_OFFSET); + + //// Check data_file_entries section boundaries + returnOutIndex(data_file_entry_section, + data_file_entry_section + header->file_count * SIZE_WOWS_INDEX_DATA_FILE_ENTRY, index); WOWS_INDEX_DATA_FILE_ENTRY *data_file_entry = - (WOWS_INDEX_DATA_FILE_ENTRY *)(contents + header->offset_idx_data_section + MAGIC_SECTION_OFFSET); + (WOWS_INDEX_DATA_FILE_ENTRY *)calloc(sizeof(WOWS_INDEX_DATA_FILE_ENTRY), header->file_count); index->data_file_entry = data_file_entry; - // Check data_file_entries section boundaries - returnOutIndex((char *)data_file_entry, (char *)&data_file_entry[header->file_count], index); + + for (i = 0; i < header->file_count; i++) { + size_t offset = i * SIZE_WOWS_INDEX_DATA_FILE_ENTRY; + data_file_entry[i].metadata_id = datatoh64(data_file_entry_section + offset, 0, context); + data_file_entry[i].footer_id = datatoh64(data_file_entry_section + offset, 8, context); + data_file_entry[i].offset_pkg_data = datatoh64(data_file_entry_section + offset, 16, context); + data_file_entry[i].type_1 = datatoh32(data_file_entry_section + offset, 24, context); + data_file_entry[i].type_2 = datatoh32(data_file_entry_section + offset, 28, context); + data_file_entry[i].size_pkg_data = datatoh64(data_file_entry_section + offset, 32, context); + data_file_entry[i].id_pkg_data = datatoh32(data_file_entry_section + offset, 40, context); + } // Get the footer section - WOWS_INDEX_FOOTER *footer = - (WOWS_INDEX_FOOTER *)(contents + header->offset_idx_footer_section + MAGIC_SECTION_OFFSET); - index->footer = footer; + char *footer_src = (contents + header->offset_idx_footer_section + MAGIC_SECTION_OFFSET); // Check footer section boundaries - returnOutIndex((char *)footer, (char *)footer + sizeof(WOWS_INDEX_FOOTER), index); + returnOutIndex((char *)footer_src, (char *)footer_src + SIZE_WOWS_INDEX_FOOTER, index); + + WOWS_INDEX_FOOTER *footer = calloc(sizeof(WOWS_INDEX_FOOTER), 1); + index->footer = footer; + + footer->pkg_file_name_size = datatoh64(footer_src, 0, context); + footer->unknown_7 = datatoh64(footer_src, 8, context); + footer->id = datatoh64(footer_src, 16, context); + ; + footer->_file_name = calloc(sizeof(char), footer->pkg_file_name_size); + char *file_name_src = footer_src + SIZE_WOWS_INDEX_FOOTER; + returnOutIndex(file_name_src, file_name_src + footer->pkg_file_name_size, index); + strncpy(footer->_file_name, file_name_src, footer->pkg_file_name_size); *index_in = index; return 0; } @@ -298,17 +335,10 @@ int wows_parse_index_buffer(char *contents, size_t length, const char *index_fil print_debug_files(index, context->metadata_map); } - char *filler; - // Index all the metadata entries into an hmap for (i = 0; i < index->header->file_dir_count; i++) { WOWS_INDEX_METADATA_ENTRY *entry = &index->metadata[i]; // Check that we can correctly recover the file names - int ret = get_metadata_filename(entry, index, &filler); - if (ret != 0) { - wows_set_error_details(context, "problematic metadata entry id '0x%lx'", entry->id); - return ret; - } hashmap_set(context->metadata_map, &entry); } // Do the same for all the file entries @@ -318,14 +348,9 @@ int wows_parse_index_buffer(char *contents, size_t length, const char *index_fil } // Check that we can correctly recover the file names - int ret = get_footer_filename(index->footer, index, &filler); - if (ret != 0) { - wows_set_error_details(context, "problematic footer section"); - return ret; - } uint32_t current_index_context = add_index_context(context, index); err = build_inode_tree(index, current_index_context, context); - return ret; + return err; } diff --git a/lib/inode.c b/lib/inode.c index 2e26544..0ff98db 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -33,8 +33,7 @@ int build_inode_tree(WOWS_INDEX *index, int current_index_context, WOWS_CONTEXT // We go in reverse to start from the root to the immediate parent directory of the file for (int j = (depth - 1); j > -1; j--) { // We try to get the directory from the current dir (parent_inode) - char *name; - get_metadata_filename(parent_entries[j], index, &name); + char *name = parent_entries[j]->_file_name; WOWS_DIR_INODE *existing_inode = (WOWS_DIR_INODE *)get_child(parent_inode, name); // If the directory inode doesn't exist, we need to create it @@ -100,17 +99,13 @@ WOWS_DIR_INODE *init_root_inode() { // create a dir inode WOWS_DIR_INODE *init_dir_inode(uint64_t metadata_id, uint32_t current_index_context, WOWS_DIR_INODE *parent_inode, WOWS_CONTEXT *context) { - char *name; WOWS_INDEX_METADATA_ENTRY *mentry_search = &(WOWS_INDEX_METADATA_ENTRY){.id = metadata_id}; void *res = hashmap_get(context->metadata_map, &mentry_search); if (res == NULL) { return NULL; } WOWS_INDEX_METADATA_ENTRY *entry = *(WOWS_INDEX_METADATA_ENTRY **)res; - int ret = get_metadata_filename(entry, context->indexes[current_index_context], &name); - if (ret != 0) { - return NULL; - } + char *name = entry->_file_name; WOWS_DIR_INODE *dir_inode = calloc(sizeof(WOWS_DIR_INODE), 1); dir_inode->type = WOWS_INODE_TYPE_DIR; @@ -139,12 +134,7 @@ WOWS_FILE_INODE *init_file_inode(uint64_t metadata_id, uint32_t current_index_co return NULL; } WOWS_INDEX_METADATA_ENTRY *entry = *(WOWS_INDEX_METADATA_ENTRY **)res; - char *name; - int ret = get_metadata_filename(entry, context->indexes[current_index_context], &name); - if (ret != 0) { - return NULL; - } - + char *name = entry->_file_name; WOWS_FILE_INODE *file_inode = calloc(sizeof(WOWS_FILE_INODE), 1); file_inode->type = WOWS_INODE_TYPE_FILE; file_inode->id = metadata_id; diff --git a/lib/memory-helpers.c b/lib/memory-helpers.c index ce5cf0a..3029718 100644 --- a/lib/memory-helpers.c +++ b/lib/memory-helpers.c @@ -33,6 +33,13 @@ int wows_free_context_internal(WOWS_CONTEXT *context, int flag) { } free(index->index_file_path); free(index->header); + for (int j = 0; j < index->header->file_dir_count; j++) { + free(index->metadata[j]._file_name); + } + free(index->metadata); + free(index->data_file_entry); + free(index->footer->_file_name); + free(index->footer); free(index); } diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index b04dd74..9128fe6 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -50,8 +50,11 @@ typedef struct { uint64_t offset_idx_file_name; uint64_t id; uint64_t parent_id; + char *_file_name; } WOWS_INDEX_METADATA_ENTRY; +#define SIZE_WOWS_INDEX_METADATA_ENTRY 32 + // INDEX file pkg data pointer entry typedef struct { uint64_t metadata_id; @@ -64,13 +67,18 @@ typedef struct { uint32_t padding; } WOWS_INDEX_DATA_FILE_ENTRY; +#define SIZE_WOWS_INDEX_DATA_FILE_ENTRY 48 + // INDEX file footer typedef struct { uint64_t pkg_file_name_size; uint64_t unknown_7; uint64_t id; + char *_file_name; } WOWS_INDEX_FOOTER; +#define SIZE_WOWS_INDEX_FOOTER 24 + // PKG file ID + padding typedef struct { uint32_t padding_1; From 2fa69de0a6f6b09eedf01b94040c92390953817e Mon Sep 17 00:00:00 2001 From: kakwa Date: Wed, 13 Mar 2024 02:00:38 +0100 Subject: [PATCH 03/24] making look variable local --- lib/index-parser.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index df8214e..66b533a 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -122,7 +122,6 @@ uint64_t datatoh64(char *data, size_t offset, WOWS_CONTEXT *context) { // Map the different section of the index file content to an index struct int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context) { - int i; WOWS_INDEX *index = calloc(sizeof(WOWS_INDEX), 1); index->start_address = contents; index->end_address = contents + length; @@ -162,7 +161,7 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO WOWS_INDEX_METADATA_ENTRY *metadata = calloc(sizeof(WOWS_INDEX_METADATA_ENTRY), header->file_dir_count); index->metadata = metadata; - for (i = 0; i < header->file_dir_count; i++) { + for (int i = 0; i < header->file_dir_count; i++) { size_t offset = i * SIZE_WOWS_INDEX_METADATA_ENTRY; metadata[i].file_name_size = datatoh64(metadata_section + offset, 0, context); metadata[i].offset_idx_file_name = datatoh64(metadata_section + offset, 8, context); @@ -186,7 +185,7 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO (WOWS_INDEX_DATA_FILE_ENTRY *)calloc(sizeof(WOWS_INDEX_DATA_FILE_ENTRY), header->file_count); index->data_file_entry = data_file_entry; - for (i = 0; i < header->file_count; i++) { + for (int i = 0; i < header->file_count; i++) { size_t offset = i * SIZE_WOWS_INDEX_DATA_FILE_ENTRY; data_file_entry[i].metadata_id = datatoh64(data_file_entry_section + offset, 0, context); data_file_entry[i].footer_id = datatoh64(data_file_entry_section + offset, 8, context); From d2aa83544adc9328e19132b50878fd3cf5dec547 Mon Sep 17 00:00:00 2001 From: kakwa Date: Wed, 13 Mar 2024 02:05:09 +0100 Subject: [PATCH 04/24] fix recovery of the pkg path --- lib/index-parser.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index 66b533a..d56bf4a 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -55,8 +55,7 @@ int get_metadata_filename(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, c } int get_pkg_filepath(WOWS_INDEX *index, char **out) { - char *pkg_file_name; - get_footer_filename(index->footer, index, &pkg_file_name); + char *pkg_file_name = index->footer->_file_name; const int num_parents = 4; char *current_path = strdup(index->index_file_path); From 6f3a108b6b3468ab997742af7f8748067a6a46fe Mon Sep 17 00:00:00 2001 From: kakwa Date: Thu, 14 Mar 2024 21:58:51 +0100 Subject: [PATCH 05/24] fix boundary check --- lib/index-parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index d56bf4a..5b40903 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -169,7 +169,7 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO char *file_name = calloc(sizeof(char), metadata[i].file_name_size); char *file_name_src = metadata_section + offset + metadata[i].offset_idx_file_name; - returnOutIndex(file_name_src, file_name_src + metadata[i].offset_idx_file_name, index); + returnOutIndex(file_name_src, file_name_src + metadata[i].file_name_size, index); strncpy(file_name, file_name_src, metadata[i].file_name_size); metadata[i]._file_name = file_name; } @@ -177,7 +177,7 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO // Get the start pkg data pointer section char *data_file_entry_section = (contents + header->offset_idx_data_section + MAGIC_SECTION_OFFSET); - //// Check data_file_entries section boundaries + // Check data_file_entries section boundaries returnOutIndex(data_file_entry_section, data_file_entry_section + header->file_count * SIZE_WOWS_INDEX_DATA_FILE_ENTRY, index); WOWS_INDEX_DATA_FILE_ENTRY *data_file_entry = From c426b53a5ccf1d56af08ab3b0486e43c610cf28b Mon Sep 17 00:00:00 2001 From: kakwa Date: Thu, 14 Mar 2024 22:10:13 +0100 Subject: [PATCH 06/24] fix project URL in help --- wows-depack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wows-depack.c b/wows-depack.c index 6af07dd..de6ef9a 100644 --- a/wows-depack.c +++ b/wows-depack.c @@ -28,7 +28,7 @@ const char *argp_program_version = BFD_VERSION; -const char *argp_program_bug_address = "https://github.com/kakwa/wows-depack/issues"; +const char *argp_program_bug_address = "https://github.com/wows-tools/wows-depack/issues"; static char doc[] = "\nWorld of Warships resource extractor tool"; From c26edeb30f2f3e3f7f29de23cf00aab332e6932b Mon Sep 17 00:00:00 2001 From: kakwa Date: Thu, 14 Mar 2024 22:18:30 +0100 Subject: [PATCH 07/24] clean-up of cmake --- CMakeLists.txt | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3614f8e..a5bdb99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,28 +32,23 @@ set(CMAKE_CXX_FLAGS # Options option(DEBUG "compile with debug symbol" OFF) option(STATIC "compile statically" OFF) -option(USE_CLANG "build application with clang" OFF) -option(USE_GCC "build application with gcc" OFF) -option(FORCELE "force little endian architecture" OFF) option(COVERAGE "Enable code coverage" OFF) option(BUILD_DOC "Build documentation" OFF) option(BUILD_TESTS "Build tests" OFF) -if(USE_CLANG) - set(CMAKE_CXX_COMPILER "clang++") - set(CMAKE_CC_COMPILER "clang") -endif(USE_CLANG) - -if(USE_GCC) - set(CMAKE_CXX_COMPILER "g++") - set(CMAKE_CC_COMPILER "gcc") -endif(USE_GCC) - if(DEBUG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -g") set(CMAKE_BUILD_TYPE Debug) endif(DEBUG) +# Build external dependancies if we are on OSX +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Mac OS X specific code + set(EXTERNAL_ICONV "iconv") + set(EXTERNAL_ARGP "argp") + add_definitions(-DDARWIN) +endif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + if(STATIC) set(SHARED "") set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") @@ -67,14 +62,6 @@ if(UNIX) link_libraries(m) endif(UNIX) -# Build external dependancies if we are on OSX -if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - # Mac OS X specific code - set(EXTERNAL_ICONV "iconv") - set(EXTERNAL_ARGP "argp") - add_definitions(-DDARWIN) -endif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - # linking directories link_directories(${CMAKE_BINARY_DIR}/ /usr/local/lib /usr/lib/) From 3ddc667f83b9f38b22343f04db62ef2fcddeb1dd Mon Sep 17 00:00:00 2001 From: kakwa Date: Thu, 14 Mar 2024 22:49:19 +0100 Subject: [PATCH 08/24] fix static configuration --- CMakeLists.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5bdb99..0a6a177 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,15 @@ set(wows-depack_VERSION_PATCH 0) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") +if(STATIC) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(ZLIB_USE_STATIC_LIBS ON) + set(BUILD_SHARED_LIBRARIES OFF) + set(CMAKE_EXE_LINKER_FLAGS "-static") +else(STATIC) + set(SHARED "SHARED") +endif(STATIC) + find_package(ZLIB REQUIRED) find_package(PCRE REQUIRED) find_library(CUNIT_LIBRARY cunit) @@ -49,15 +58,6 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") add_definitions(-DDARWIN) endif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -if(STATIC) - set(SHARED "") - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - set(BUILD_SHARED_LIBRARIES OFF) - set(CMAKE_EXE_LINKER_FLAGS "-static") -else(STATIC) - set(SHARED "SHARED") -endif(STATIC) - if(UNIX) link_libraries(m) endif(UNIX) From db3eaca2379d5b5ec23aef49fe16e9c654b76b93 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 00:35:19 +0100 Subject: [PATCH 09/24] small fix when file names are empty --- lib/deflate.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/deflate.c b/lib/deflate.c index 7e13166..09aa88f 100644 --- a/lib/deflate.c +++ b/lib/deflate.c @@ -357,7 +357,8 @@ int wows_dump_index_to_file(WOWS_INDEX *index, FILE *f) { for (size_t i = 0; i < index->header->file_dir_count; i++) { WOWS_INDEX_METADATA_ENTRY *metadata_entry = &index->metadata[i]; char *file_name = metadata_entry->_file_name; - fwrite(file_name, metadata_entry->file_name_size, 1, f); + if (file_name) + fwrite(file_name, metadata_entry->file_name_size, 1, f); general_offset += metadata_entry->file_name_size; } From 3ef9dced7d47387813a914d97bab06928bec524b Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 00:35:46 +0100 Subject: [PATCH 10/24] make the unit tests fail early instead of segfaulting --- tests/tests.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/tests.c b/tests/tests.c index d4a455b..e111cf9 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -206,7 +206,7 @@ static void test_map_index_file() { WOWS_CONTEXT *context = calloc(sizeof(WOWS_CONTEXT), 1); int result = map_index_file(contents, TEST_DATA_SIZE, &index, context); - CU_ASSERT_EQUAL(result, 0); + CU_ASSERT_EQUAL_FATAL(result, 0); CU_ASSERT_PTR_NOT_NULL(index); CU_ASSERT_PTR_EQUAL(index->start_address, contents); CU_ASSERT_PTR_EQUAL(index->end_address, contents + TEST_DATA_SIZE); @@ -522,7 +522,7 @@ void test_compress() { fclose(nfd_idx); // TODO add more asserts - CU_ASSERT_EQUAL(result, 0); + CU_ASSERT_EQUAL_FATAL(result, 0); CU_ASSERT_NOT_EQUAL(buf_idx_size, 0); CU_ASSERT_NOT_EQUAL(buf_pkg_size, 0); free(buf_idx); @@ -681,7 +681,7 @@ void test_wows_parse_index_dir(void) { // Parse the index file int ret = wows_parse_index_dir("./tests/data/", context); // Assert that the return value is 0 (success) - CU_ASSERT_EQUAL(ret, 0); + CU_ASSERT_EQUAL_FATAL(ret, 0); char *err_msg = wows_error_string(ret, context); printf("Error: %s\n", err_msg); @@ -755,13 +755,14 @@ void test_get_pkg_filepath() { WOWS_INDEX *index; WOWS_CONTEXT *context = wows_init_context(0); - map_index_file(contents, 2048, &index, context); + int result = map_index_file(contents, 2048, &index, context); + CU_ASSERT_EQUAL_FATAL(0, result); index->index_file_path = "/path/to/wows/bin/6831266/idx/basecontent.idx"; char *out; - int result = get_pkg_filepath(index, &out); + result = get_pkg_filepath(index, &out); - CU_ASSERT_EQUAL(0, result); + CU_ASSERT_EQUAL_FATAL(0, result); CU_ASSERT_PTR_NOT_NULL(out); // Ensure the output path is correct CU_ASSERT_STRING_EQUAL("/path/to/wows/res_packages/foot", out); @@ -779,7 +780,7 @@ void test_wows_search_file(void) { // Parse the index file int ret = wows_parse_index_dir("./tests/data/", context); // Assert that the return value is 0 (success) - CU_ASSERT_EQUAL(ret, 0); + CU_ASSERT_EQUAL_FATAL(ret, 0); char *err_msg = wows_error_string(ret, context); printf("Error: %s\n", err_msg); @@ -796,10 +797,10 @@ void test_wows_search_file(void) { int result = wows_search(context, pattern, mode, &result_count, &results); // Check result code - CU_ASSERT_EQUAL(result, 0); + CU_ASSERT_EQUAL_FATAL(result, 0); // Check number of matching files - CU_ASSERT_EQUAL(result_count, expected_result_count); + CU_ASSERT_EQUAL_FATAL(result_count, expected_result_count); // Check file names CU_ASSERT_STRING_EQUAL(results[0], "d1234/c1234/b1234"); From e97bf9a59ef7748512bd902d57eb40eeaafa9840 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 00:45:27 +0100 Subject: [PATCH 11/24] add endianess constent --- tests/data/fake.idx | Bin 333 -> 333 bytes tests/data/fake2.idx | Bin 333 -> 333 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/data/fake.idx b/tests/data/fake.idx index c2abc4005cbc0e62060b1affa76ca9f85904b38c..426326b66ae60782d0973c9243df4f4af2edc581 100644 GIT binary patch delta 17 YcmX@hbe4(3GuSPFfq{W(BZn&^04h%d+5i9m delta 17 XcmX@hbe4(3GuSPF0SGp7xH19&Dn$d> diff --git a/tests/data/fake2.idx b/tests/data/fake2.idx index 8cf2f4c31ed167b864c57d243a1754765228d068..b706ee0e7eecae2d9f7419df8e8df15d082daadf 100644 GIT binary patch delta 17 YcmX@hbe4(3GuSPFfq{W(BZn&^04h%d+5i9m delta 17 XcmX@hbe4(3GuSPF0SGp7xH19&Dn$d> From d0d67eaf9572675bd6e95af3f43311b3c6678434 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 01:02:45 +0100 Subject: [PATCH 12/24] create structs specifically for the unit tests --- tests/tests.c | 191 +++++++++++++++++++++++++++++++------------------- 1 file changed, 120 insertions(+), 71 deletions(-) diff --git a/tests/tests.c b/tests/tests.c index e111cf9..4772f5c 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -11,6 +11,54 @@ #define TEST_DATA_SIZE 2048 +#pragma pack(push, 1) +typedef struct { + char magic[4]; + uint32_t endianess; + uint32_t id; + uint32_t unknown_2; + uint32_t file_dir_count; + uint32_t file_count; + uint64_t unknown_3; + uint64_t header_size; + uint64_t offset_idx_data_section; + uint64_t offset_idx_footer_section; +} TWOWS_INDEX_HEADER; + +typedef struct { + uint64_t file_name_size; + uint64_t offset_idx_file_name; + uint64_t id; + uint64_t parent_id; +} TWOWS_INDEX_METADATA_ENTRY; + +typedef struct { + uint64_t metadata_id; + uint64_t footer_id; + uint64_t offset_pkg_data; + uint32_t type_1; + uint32_t type_2; + uint32_t size_pkg_data; + uint64_t id_pkg_data; + uint32_t padding; +} TWOWS_INDEX_DATA_FILE_ENTRY; + +// INDEX file footer +typedef struct { + uint64_t pkg_file_name_size; + uint64_t unknown_7; + uint64_t id; +} TWOWS_INDEX_FOOTER; + + +// PKG file ID + padding +typedef struct { + uint32_t padding_1; + uint64_t id; + uint32_t padding_2; +} TWOWS_PKG_ID_ENTRY; +#pragma pack(pop) + void test_print_header() { WOWS_INDEX_HEADER header = {0}; int ret = print_header(&header); @@ -170,37 +218,38 @@ static void test_map_index_file() { WOWS_INDEX *index; // Prepare test data - WOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, + TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, + .endianess = 0x2000000, .file_dir_count = 1, .file_count = 1, - .header_size = sizeof(WOWS_INDEX_HEADER), + .header_size = sizeof(TWOWS_INDEX_HEADER), .offset_idx_data_section = 512, .offset_idx_footer_section = 1024}; - WOWS_INDEX_METADATA_ENTRY metadata = {.file_name_size = 6, .offset_idx_file_name = 16, .id = 1, .parent_id = 0}; - WOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 16, .id = 2, .parent_id = 0}; - WOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, + TWOWS_INDEX_METADATA_ENTRY metadata = {.file_name_size = 6, .offset_idx_file_name = 16, .id = 1, .parent_id = 0}; + TWOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 16, .id = 2, .parent_id = 0}; + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; - WOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; - WOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; - memcpy(contents, &header, sizeof(WOWS_INDEX_HEADER)); - memcpy(contents + sizeof(WOWS_INDEX_HEADER), &metadata, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY), &metadata2, - sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(WOWS_INDEX_DATA_FILE_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(WOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, - sizeof(WOWS_INDEX_DATA_FILE_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(WOWS_INDEX_FOOTER)); + TWOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; + memcpy(contents, &header, sizeof(TWOWS_INDEX_HEADER)); + memcpy(contents + sizeof(TWOWS_INDEX_HEADER), &metadata, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY), &metadata2, + sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(TWOWS_INDEX_FOOTER)); // Test map_index_file function WOWS_CONTEXT *context = calloc(sizeof(WOWS_CONTEXT), 1); @@ -259,32 +308,32 @@ void test_wows_parse_index_buffer() { char *contents = calloc(sizeof(char) * TEST_DATA_SIZE, 1); // Prepare test data - WOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, + TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, .file_dir_count = 5, .file_count = 2, - .header_size = sizeof(WOWS_INDEX_HEADER), + .header_size = sizeof(TWOWS_INDEX_HEADER), .offset_idx_data_section = 512, .offset_idx_footer_section = 1024}; - memcpy(contents, &header, sizeof(WOWS_INDEX_HEADER)); - - WOWS_INDEX_METADATA_ENTRY metadata1 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 1, .parent_id = 3}; - char *offset_metadata1 = contents + sizeof(WOWS_INDEX_HEADER); - WOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 2, .parent_id = 3}; - char *offset_metadata2 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY); - WOWS_INDEX_METADATA_ENTRY metadata3 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 3, .parent_id = 4}; - char *offset_metadata3 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY) * 2; - WOWS_INDEX_METADATA_ENTRY metadata4 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 4, .parent_id = 5}; - char *offset_metadata4 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY) * 3; - WOWS_INDEX_METADATA_ENTRY metadata5 = { + memcpy(contents, &header, sizeof(TWOWS_INDEX_HEADER)); + + TWOWS_INDEX_METADATA_ENTRY metadata1 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 1, .parent_id = 3}; + char *offset_metadata1 = contents + sizeof(TWOWS_INDEX_HEADER); + TWOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 2, .parent_id = 3}; + char *offset_metadata2 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY); + TWOWS_INDEX_METADATA_ENTRY metadata3 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 3, .parent_id = 4}; + char *offset_metadata3 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY) * 2; + TWOWS_INDEX_METADATA_ENTRY metadata4 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 4, .parent_id = 5}; + char *offset_metadata4 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY) * 3; + TWOWS_INDEX_METADATA_ENTRY metadata5 = { .file_name_size = 6, .offset_idx_file_name = 256, .id = 5, .parent_id = 1111}; - char *offset_metadata5 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY) * 3; + char *offset_metadata5 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY) * 3; - memcpy(offset_metadata1, &metadata1, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata2, &metadata2, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata3, &metadata3, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata4, &metadata4, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata5, &metadata5, sizeof(WOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata1, &metadata1, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata2, &metadata2, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata3, &metadata3, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata4, &metadata4, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata5, &metadata5, sizeof(TWOWS_INDEX_METADATA_ENTRY)); char *name1 = "a1234"; char *name2 = "b1234"; @@ -297,26 +346,26 @@ void test_wows_parse_index_buffer() { memcpy(offset_metadata4 + 256, name4, strlen(name4)); memcpy(offset_metadata5 + 256, name5, strlen(name5)); - WOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; - WOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; - WOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; - memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(WOWS_INDEX_DATA_FILE_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(WOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, - sizeof(WOWS_INDEX_DATA_FILE_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(WOWS_INDEX_FOOTER)); - memcpy(contents + MAGIC_SECTION_OFFSET + 1024 + sizeof(WOWS_INDEX_FOOTER), "foot", 5); + TWOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; + memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(TWOWS_INDEX_FOOTER)); + memcpy(contents + MAGIC_SECTION_OFFSET + 1024 + sizeof(TWOWS_INDEX_FOOTER), "foot", 5); size_t length = TEST_DATA_SIZE; char *index_file_path = "index file path"; @@ -325,7 +374,7 @@ void test_wows_parse_index_buffer() { int result = wows_parse_index_buffer(contents, length, index_file_path, 0, context); // Check the result and assert on success - CU_ASSERT_EQUAL(result, 0); + CU_ASSERT_EQUAL_FATAL(result, 0); char *err_msg = wows_error_string(result, context); printf("Error: %s\n", err_msg); @@ -345,7 +394,7 @@ void test_wows_parse_index_buffer() { void test_wows_dump_index_to_file(void) { WOWS_INDEX_HEADER header = {.magic = "WoWS", - .endianess = 0x20000, + .endianess = 0x2000000, .id = 0x12345678, .unknown_2 = 0x20000, .file_dir_count = 2, @@ -694,32 +743,32 @@ void test_wows_parse_index_dir(void) { void test_get_pkg_filepath() { // Prepare test data char *contents = calloc(sizeof(char) * TEST_DATA_SIZE, 1); - WOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, + TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, .file_dir_count = 5, .file_count = 2, - .header_size = sizeof(WOWS_INDEX_HEADER), + .header_size = sizeof(TWOWS_INDEX_HEADER), .offset_idx_data_section = 512, .offset_idx_footer_section = 1024}; - memcpy(contents, &header, sizeof(WOWS_INDEX_HEADER)); - - WOWS_INDEX_METADATA_ENTRY metadata1 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 1, .parent_id = 3}; - char *offset_metadata1 = contents + sizeof(WOWS_INDEX_HEADER); - WOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 2, .parent_id = 3}; - char *offset_metadata2 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY); - WOWS_INDEX_METADATA_ENTRY metadata3 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 3, .parent_id = 4}; - char *offset_metadata3 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY) * 2; - WOWS_INDEX_METADATA_ENTRY metadata4 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 4, .parent_id = 5}; - char *offset_metadata4 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY) * 3; - WOWS_INDEX_METADATA_ENTRY metadata5 = { + memcpy(contents, &header, sizeof(TWOWS_INDEX_HEADER)); + + TWOWS_INDEX_METADATA_ENTRY metadata1 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 1, .parent_id = 3}; + char *offset_metadata1 = contents + sizeof(TWOWS_INDEX_HEADER); + TWOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 2, .parent_id = 3}; + char *offset_metadata2 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY); + TWOWS_INDEX_METADATA_ENTRY metadata3 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 3, .parent_id = 4}; + char *offset_metadata3 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY) * 2; + TWOWS_INDEX_METADATA_ENTRY metadata4 = {.file_name_size = 6, .offset_idx_file_name = 256, .id = 4, .parent_id = 5}; + char *offset_metadata4 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY) * 3; + TWOWS_INDEX_METADATA_ENTRY metadata5 = { .file_name_size = 6, .offset_idx_file_name = 256, .id = 5, .parent_id = 1111}; - char *offset_metadata5 = contents + sizeof(WOWS_INDEX_HEADER) + sizeof(WOWS_INDEX_METADATA_ENTRY) * 3; + char *offset_metadata5 = contents + sizeof(TWOWS_INDEX_HEADER) + sizeof(TWOWS_INDEX_METADATA_ENTRY) * 3; - memcpy(offset_metadata1, &metadata1, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata2, &metadata2, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata3, &metadata3, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata4, &metadata4, sizeof(WOWS_INDEX_METADATA_ENTRY)); - memcpy(offset_metadata5, &metadata5, sizeof(WOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata1, &metadata1, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata2, &metadata2, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata3, &metadata3, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata4, &metadata4, sizeof(TWOWS_INDEX_METADATA_ENTRY)); + memcpy(offset_metadata5, &metadata5, sizeof(TWOWS_INDEX_METADATA_ENTRY)); char *name1 = "a1234"; char *name2 = "b1234"; @@ -732,26 +781,26 @@ void test_get_pkg_filepath() { memcpy(offset_metadata4 + 256, name4, strlen(name4)); memcpy(offset_metadata5 + 256, name5, strlen(name5)); - WOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; - WOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; - WOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; - memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(WOWS_INDEX_DATA_FILE_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(WOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, - sizeof(WOWS_INDEX_DATA_FILE_ENTRY)); - memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(WOWS_INDEX_FOOTER)); - memcpy(contents + MAGIC_SECTION_OFFSET + 1024 + sizeof(WOWS_INDEX_FOOTER), "foot", 5); + TWOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; + memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); + memcpy(contents + MAGIC_SECTION_OFFSET + 1024, &footer, sizeof(TWOWS_INDEX_FOOTER)); + memcpy(contents + MAGIC_SECTION_OFFSET + 1024 + sizeof(TWOWS_INDEX_FOOTER), "foot", 5); WOWS_INDEX *index; WOWS_CONTEXT *context = wows_init_context(0); From 20464d29819b56a97fc2e189ea2bf406a8d7c226 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 01:28:51 +0100 Subject: [PATCH 13/24] fix one test (now the endianess marker is important) --- tests/tests.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests.c b/tests/tests.c index 4772f5c..f171538 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -744,6 +744,7 @@ void test_get_pkg_filepath() { // Prepare test data char *contents = calloc(sizeof(char) * TEST_DATA_SIZE, 1); TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, + .endianess = 0x2000000, .file_dir_count = 5, .file_count = 2, .header_size = sizeof(TWOWS_INDEX_HEADER), From 9662acab53f3e8a897cc7f121e5595444de75f20 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 01:52:34 +0100 Subject: [PATCH 14/24] fix data alignment --- lib/index-parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index 5b40903..e7e7592 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -191,8 +191,8 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO data_file_entry[i].offset_pkg_data = datatoh64(data_file_entry_section + offset, 16, context); data_file_entry[i].type_1 = datatoh32(data_file_entry_section + offset, 24, context); data_file_entry[i].type_2 = datatoh32(data_file_entry_section + offset, 28, context); - data_file_entry[i].size_pkg_data = datatoh64(data_file_entry_section + offset, 32, context); - data_file_entry[i].id_pkg_data = datatoh32(data_file_entry_section + offset, 40, context); + data_file_entry[i].size_pkg_data = datatoh32(data_file_entry_section + offset, 32, context); + data_file_entry[i].id_pkg_data = datatoh64(data_file_entry_section + offset, 36, context); } // Get the footer section From 6aef6556dcf5afc919811b60f2b32da38ad0019b Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 01:53:24 +0100 Subject: [PATCH 15/24] fix the unit tests * add endianess const + test dataset correctness (number of files) + remove checks that are outdated --- tests/tests.c | 88 ++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/tests/tests.c b/tests/tests.c index f171538..bbbb7cb 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -50,7 +50,6 @@ typedef struct { uint64_t id; } TWOWS_INDEX_FOOTER; - // PKG file ID + padding typedef struct { uint32_t padding_1; @@ -219,28 +218,28 @@ static void test_map_index_file() { // Prepare test data TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, - .endianess = 0x2000000, - .file_dir_count = 1, - .file_count = 1, - .header_size = sizeof(TWOWS_INDEX_HEADER), - .offset_idx_data_section = 512, - .offset_idx_footer_section = 1024}; + .endianess = 0x2000000, + .file_dir_count = 2, + .file_count = 2, + .header_size = sizeof(TWOWS_INDEX_HEADER), + .offset_idx_data_section = 512, + .offset_idx_footer_section = 1024}; TWOWS_INDEX_METADATA_ENTRY metadata = {.file_name_size = 6, .offset_idx_file_name = 16, .id = 1, .parent_id = 0}; TWOWS_INDEX_METADATA_ENTRY metadata2 = {.file_name_size = 6, .offset_idx_file_name = 16, .id = 2, .parent_id = 0}; TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, - .footer_id = 42, - .offset_pkg_data = 1024, - .type_1 = 1, - .type_2 = 2, - .size_pkg_data = 100, - .id_pkg_data = 1}; - TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, + .footer_id = 42, + .offset_pkg_data = 1024, + .type_1 = 1, + .type_2 = 2, + .size_pkg_data = 100, + .id_pkg_data = 1}; TWOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; memcpy(contents, &header, sizeof(TWOWS_INDEX_HEADER)); memcpy(contents + sizeof(TWOWS_INDEX_HEADER), &metadata, sizeof(TWOWS_INDEX_METADATA_ENTRY)); @@ -256,17 +255,16 @@ static void test_map_index_file() { int result = map_index_file(contents, TEST_DATA_SIZE, &index, context); CU_ASSERT_EQUAL_FATAL(result, 0); + // print_debug_raw(index); CU_ASSERT_PTR_NOT_NULL(index); CU_ASSERT_PTR_EQUAL(index->start_address, contents); CU_ASSERT_PTR_EQUAL(index->end_address, contents + TEST_DATA_SIZE); CU_ASSERT_EQUAL(index->length, TEST_DATA_SIZE); - CU_ASSERT_PTR_EQUAL(index->header, (WOWS_INDEX_HEADER *)contents); - CU_ASSERT_EQUAL(index->header->file_dir_count, 1); - CU_ASSERT_EQUAL(index->header->file_count, 1); + CU_ASSERT_EQUAL(index->header->file_dir_count, 2); + CU_ASSERT_EQUAL(index->header->file_count, 2); CU_ASSERT_EQUAL(index->header->header_size, sizeof(WOWS_INDEX_HEADER)); CU_ASSERT_EQUAL(index->header->offset_idx_data_section, 512); CU_ASSERT_EQUAL(index->header->offset_idx_footer_section, 1024); - CU_ASSERT_PTR_EQUAL(index->metadata, (WOWS_INDEX_METADATA_ENTRY *)(contents + sizeof(WOWS_INDEX_HEADER))); CU_ASSERT_EQUAL(index->metadata[0].file_name_size, 6); CU_ASSERT_EQUAL(index->metadata[0].offset_idx_file_name, 16); CU_ASSERT_EQUAL(index->metadata[0].id, 1); @@ -276,9 +274,6 @@ static void test_map_index_file() { CU_ASSERT_EQUAL(index->metadata[1].id, 2); CU_ASSERT_EQUAL(index->metadata[1].parent_id, 0); - CU_ASSERT_PTR_EQUAL( - index->data_file_entry, - (WOWS_INDEX_DATA_FILE_ENTRY *)(contents + index->header->offset_idx_data_section + MAGIC_SECTION_OFFSET)); CU_ASSERT_EQUAL(index->data_file_entry[0].metadata_id, 1); CU_ASSERT_EQUAL(index->data_file_entry[0].footer_id, 42); CU_ASSERT_EQUAL(index->data_file_entry[0].offset_pkg_data, 1024); @@ -309,11 +304,12 @@ void test_wows_parse_index_buffer() { // Prepare test data TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, - .file_dir_count = 5, - .file_count = 2, - .header_size = sizeof(TWOWS_INDEX_HEADER), - .offset_idx_data_section = 512, - .offset_idx_footer_section = 1024}; + .endianess = 0x2000000, + .file_dir_count = 5, + .file_count = 2, + .header_size = sizeof(TWOWS_INDEX_HEADER), + .offset_idx_data_section = 512, + .offset_idx_footer_section = 1024}; memcpy(contents, &header, sizeof(TWOWS_INDEX_HEADER)); @@ -347,19 +343,19 @@ void test_wows_parse_index_buffer() { memcpy(offset_metadata5 + 256, name5, strlen(name5)); TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, - .footer_id = 42, - .offset_pkg_data = 1024, - .type_1 = 1, - .type_2 = 2, - .size_pkg_data = 100, - .id_pkg_data = 1}; - TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, + .footer_id = 42, + .offset_pkg_data = 1024, + .type_1 = 1, + .type_2 = 2, + .size_pkg_data = 100, + .id_pkg_data = 1}; TWOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, @@ -744,12 +740,12 @@ void test_get_pkg_filepath() { // Prepare test data char *contents = calloc(sizeof(char) * TEST_DATA_SIZE, 1); TWOWS_INDEX_HEADER header = {.magic = {'I', 'S', 'F', 'P'}, - .endianess = 0x2000000, - .file_dir_count = 5, - .file_count = 2, - .header_size = sizeof(TWOWS_INDEX_HEADER), - .offset_idx_data_section = 512, - .offset_idx_footer_section = 1024}; + .endianess = 0x2000000, + .file_dir_count = 5, + .file_count = 2, + .header_size = sizeof(TWOWS_INDEX_HEADER), + .offset_idx_data_section = 512, + .offset_idx_footer_section = 1024}; memcpy(contents, &header, sizeof(TWOWS_INDEX_HEADER)); @@ -783,19 +779,19 @@ void test_get_pkg_filepath() { memcpy(offset_metadata5 + 256, name5, strlen(name5)); TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry = {.metadata_id = 1, - .footer_id = 42, - .offset_pkg_data = 1024, - .type_1 = 1, - .type_2 = 2, - .size_pkg_data = 100, - .id_pkg_data = 1}; - TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, .footer_id = 42, .offset_pkg_data = 1024, .type_1 = 1, .type_2 = 2, .size_pkg_data = 100, .id_pkg_data = 1}; + TWOWS_INDEX_DATA_FILE_ENTRY data_file_entry2 = {.metadata_id = 2, + .footer_id = 42, + .offset_pkg_data = 1024, + .type_1 = 1, + .type_2 = 2, + .size_pkg_data = 100, + .id_pkg_data = 1}; TWOWS_INDEX_FOOTER footer = {.pkg_file_name_size = 5, .id = 42}; memcpy(contents + MAGIC_SECTION_OFFSET + 512, &data_file_entry, sizeof(TWOWS_INDEX_DATA_FILE_ENTRY)); memcpy(contents + MAGIC_SECTION_OFFSET + 512 + sizeof(TWOWS_INDEX_DATA_FILE_ENTRY), &data_file_entry2, From 4ba804026d3f86885313d8d236ff68db8803c603 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 02:00:44 +0100 Subject: [PATCH 16/24] remove dead code --- lib/index-parser.c | 39 --------------------------------------- lib/wows-depack-private.h | 3 --- 2 files changed, 42 deletions(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index e7e7592..a149c62 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -32,28 +32,6 @@ #include "wows-depack-private.h" #include "hashmap.h" -int get_metadata_filename_unsafe(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, char **out) { - *out = entry->_file_name; - return 0; -} - -int get_metadata_filename(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, char **out) { - char *filename = (char *)entry; - filename += entry->offset_idx_file_name; - // Check that the file name is not too long - if (entry->file_name_size > WOWS_PATH_MAX) { - return WOWS_ERROR_PATH_TOO_LONG; - } - // Check that the string is actually within the index boundaries - returnOutIndex(filename, filename + entry->file_name_size, index); - // Check that it is null terminated - if (filename[entry->file_name_size - 1] != '\0') { - return WOWS_ERROR_NON_ZERO_TERMINATED_STRING; - } - *out = filename; - return 0; -} - int get_pkg_filepath(WOWS_INDEX *index, char **out) { char *pkg_file_name = index->footer->_file_name; const int num_parents = 4; @@ -73,23 +51,6 @@ int get_pkg_filepath(WOWS_INDEX *index, char **out) { return 0; } -int get_footer_filename(WOWS_INDEX_FOOTER *footer, WOWS_INDEX *index, char **out) { - char *pkg_filename = (char *)footer; - pkg_filename += sizeof(WOWS_INDEX_FOOTER); - // Check that the file name is not too long - if (footer->pkg_file_name_size > WOWS_PATH_MAX) { - return WOWS_ERROR_PATH_TOO_LONG; - } - // Check that the string is actually within the index boundaries - returnOutIndex(pkg_filename, pkg_filename + footer->pkg_file_name_size, index); - // Check that it is null terminated - if (pkg_filename[footer->pkg_file_name_size - 1] != '\0') { - return WOWS_ERROR_NON_ZERO_TERMINATED_STRING; - } - *out = pkg_filename; - return 0; -} - // Context init function WOWS_CONTEXT *wows_init_context(uint8_t debug_level) { WOWS_CONTEXT *context = calloc(sizeof(WOWS_CONTEXT), 1); diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index 9128fe6..81ad745 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -155,9 +155,6 @@ typedef struct { /* ---------- */ bool checkOutOfIndex(char *start, char *end, WOWS_INDEX *index); -int get_metadata_filename(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, char **out); -int get_metadata_filename_unsafe(WOWS_INDEX_METADATA_ENTRY *entry, WOWS_INDEX *index, char **out); -int get_footer_filename(WOWS_INDEX_FOOTER *footer, WOWS_INDEX *index, char **out); int wows_parse_index(char *index_file_path, WOWS_CONTEXT *context); int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context); char *wows_render_str(char *fmt, ...); From 1d6ec06299ecb65b37c54cdbb0909d2eeb803ceb Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 02:30:36 +0100 Subject: [PATCH 17/24] fix for the writer and remove pragma 1 --- lib/deflate.c | 11 +++++++---- lib/wows-depack-private.h | 4 +--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/deflate.c b/lib/deflate.c index 09aa88f..4f49bf4 100644 --- a/lib/deflate.c +++ b/lib/deflate.c @@ -202,7 +202,7 @@ int write_data_blob(char *file_path, FILE *pkg_fp, uint64_t *offset, uint32_t *s */ int write_metadata_entry(WOWS_INDEX_METADATA_ENTRY **metadata, uint64_t *metadata_section_size, uint64_t metadata_id, uint64_t file_name_size, uint64_t offset_idx_file_name, uint64_t parent_id, - uint64_t *file_plus_dir_count) { + uint64_t *file_plus_dir_count, char *file_name) { // If the metadata section array is full, allocate more space for it. if ((*metadata_section_size - *file_plus_dir_count) == 0) { *metadata = realloc(*metadata, sizeof(WOWS_INDEX_METADATA_ENTRY) * (*file_plus_dir_count + CHUNK_FILE)); @@ -212,6 +212,7 @@ int write_metadata_entry(WOWS_INDEX_METADATA_ENTRY **metadata, uint64_t *metadat (*metadata)[*file_plus_dir_count].file_name_size = file_name_size; (*metadata)[*file_plus_dir_count].offset_idx_file_name = offset_idx_file_name; (*metadata)[*file_plus_dir_count].parent_id = parent_id; + (*metadata)[*file_plus_dir_count]._file_name = file_name; (*file_plus_dir_count)++; return 0; } @@ -306,7 +307,8 @@ int recursive_writer(wows_writer *writer, char *path, uint64_t parent_id) { &(writer->filename_section_size)); write_metadata_entry(&(writer->index->metadata), &(writer->metadata_section_size), metadata_id, (strlen(entry->d_name) + 1), offset_idx_file_name, parent_id, - &(writer->file_plus_dir_count)); + &(writer->file_plus_dir_count), + writer->filename_section + writer->filename_section_offset); recursive_writer(writer, full_path, metadata_id); } else if (S_ISREG(info.st_mode)) { uint64_t offset_idx_file_name = writer->filename_section_offset; @@ -320,7 +322,8 @@ int recursive_writer(wows_writer *writer, char *path, uint64_t parent_id) { &(writer->filename_section_size)); write_metadata_entry(&(writer->index->metadata), &(writer->metadata_section_size), metadata_id, (strlen(entry->d_name) + 1), offset_idx_file_name, parent_id, - &(writer->file_plus_dir_count)); + &(writer->file_plus_dir_count), + writer->filename_section + writer->filename_section_offset); write_file_pkg_entry(&(writer->index->data_file_entry), &(writer->file_section_size), metadata_id, writer->footer_id, start_offset, size, pkg_id, &(writer->file_count)); } @@ -357,7 +360,7 @@ int wows_dump_index_to_file(WOWS_INDEX *index, FILE *f) { for (size_t i = 0; i < index->header->file_dir_count; i++) { WOWS_INDEX_METADATA_ENTRY *metadata_entry = &index->metadata[i]; char *file_name = metadata_entry->_file_name; - if (file_name) + if (file_name != NULL && metadata_entry->file_name_size != 0) fwrite(file_name, metadata_entry->file_name_size, 1, f); general_offset += metadata_entry->file_name_size; } diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index 81ad745..6cce8b3 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -11,8 +11,6 @@ // Put some limits on the file name size #define WOWS_PATH_MAX 4096 -#pragma pack(1) - /* ---------- */ /* Errors */ @@ -207,7 +205,7 @@ int write_data_blob(char *file_path, FILE *pkg_fp, uint64_t *offset, uint32_t *s int write_metadata_entry(WOWS_INDEX_METADATA_ENTRY **metadata, uint64_t *metadata_section_size, uint64_t metadata_id, uint64_t file_name_size, uint64_t offset_idx_file_name, uint64_t parent_id, - uint64_t *file_plus_dir_count); + uint64_t *file_plus_dir_count, char *file_name); int recursive_writer(wows_writer *writer, char *path, uint64_t parent_id); int copy_data(FILE *in, FILE *out, long offset, size_t size); int internal_wows_extract_dir(WOWS_CONTEXT *context, char *dir_path, char *out_dir_path, FILE *magic_fp); From 18c7f0c6c3fe659110f8a1627fd5c176961743cd Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 02:54:58 +0100 Subject: [PATCH 18/24] adding an hard limit on the number of files+dir --- inc/wows-depack.h | 1 + lib/error-helpers.c | 3 +++ lib/index-parser.c | 9 +++++++++ lib/wows-depack-private.h | 1 + 4 files changed, 14 insertions(+) diff --git a/inc/wows-depack.h b/inc/wows-depack.h index 0cd2b6a..04f89b8 100644 --- a/inc/wows-depack.h +++ b/inc/wows-depack.h @@ -24,6 +24,7 @@ #define WOWS_ERROR_NOT_A_DIR 13 /**< path is not a directory */ #define WOWS_ERROR_NOT_FOUND 14 /**< file or directory not found */ #define WOWS_ERROR_FILE_WRITE 15 /**< file write error */ +#define WOWS_ERROR_MAX_FILE 16 /**< maximum number of file/dir */ /* ---------- */ diff --git a/lib/error-helpers.c b/lib/error-helpers.c index b8b894e..08dfb27 100644 --- a/lib/error-helpers.c +++ b/lib/error-helpers.c @@ -58,6 +58,9 @@ char *wows_error_string(int error_code, WOWS_CONTEXT *context) { case WOWS_ERROR_FILE_WRITE: error_string = "file write error"; break; + case WOWS_ERROR_MAX_FILE: + error_string = "maximum number of file/dir in index reached"; + break; case 0: error_string = "no recent error"; break; diff --git a/lib/index-parser.c b/lib/index-parser.c index a149c62..0709b31 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -114,6 +114,14 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO header->offset_idx_data_section = datatoh64(contents, 40, context); header->offset_idx_footer_section = datatoh64(contents, 48, context); + if (header->file_dir_count > WOWS_FILE_DIR_MAX) { + return WOWS_ERROR_MAX_FILE; + } + + if (header->file_count > WOWS_FILE_DIR_MAX) { + return WOWS_ERROR_MAX_FILE; + } + // Get the start of the metadata array char *metadata_section = (contents + SIZE_WOWS_INDEX_HEADER); // Check the metadata section @@ -141,6 +149,7 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO // Check data_file_entries section boundaries returnOutIndex(data_file_entry_section, data_file_entry_section + header->file_count * SIZE_WOWS_INDEX_DATA_FILE_ENTRY, index); + WOWS_INDEX_DATA_FILE_ENTRY *data_file_entry = (WOWS_INDEX_DATA_FILE_ENTRY *)calloc(sizeof(WOWS_INDEX_DATA_FILE_ENTRY), header->file_count); index->data_file_entry = data_file_entry; diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index 6cce8b3..51bc07e 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -11,6 +11,7 @@ // Put some limits on the file name size #define WOWS_PATH_MAX 4096 +#define WOWS_FILE_DIR_MAX 32 * 1024 * 1024 /* ---------- */ /* Errors */ From 876ce2cc82676bafb47b410926ba5fc508954677 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 07:54:51 +0100 Subject: [PATCH 19/24] adding a few corrupted indexes --- ...c:000074,time:10154,execs:39852,op:havoc,rep:4 | Bin 0 -> 348 bytes ...c:000089,time:12154,execs:47757,op:havoc,rep:1 | Bin 0 -> 345 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/corrupted/id:000000,sig:11,src:000074,time:10154,execs:39852,op:havoc,rep:4 create mode 100644 tests/corrupted/id:000001,sig:11,src:000089,time:12154,execs:47757,op:havoc,rep:1 diff --git a/tests/corrupted/id:000000,sig:11,src:000074,time:10154,execs:39852,op:havoc,rep:4 b/tests/corrupted/id:000000,sig:11,src:000074,time:10154,execs:39852,op:havoc,rep:4 new file mode 100644 index 0000000000000000000000000000000000000000..3dc44b8c60894105023e8a94b010494c9c3175f9 GIT binary patch literal 348 zcmeYab_-x&U|<4b1_vN!0b(!*NP~a|gkm@Vr4<+<3^pjg0ZOw%X?7?LlTSnAqs#lC z>EnR%Iia)>l!l3igB%D1sfI?zCJbpH@_#y*odF_$!Q7z*Q3ml41H;FFKvV#ehPe~W ihnfxYA}9<}fEeoCSO2jApghbzAQKE=_NL|MmjD2RG%2b8 literal 0 HcmV?d00001 diff --git a/tests/corrupted/id:000001,sig:11,src:000089,time:12154,execs:47757,op:havoc,rep:1 b/tests/corrupted/id:000001,sig:11,src:000089,time:12154,execs:47757,op:havoc,rep:1 new file mode 100644 index 0000000000000000000000000000000000000000..3f161340e93d43286f5d9055f60ddaac78e541a8 GIT binary patch literal 345 zcmeYab_-x&U|<4b1_vN!0b($Rfk6StWY7Q+KyUy|0tFc%3^pjg0ZOw%X?76(9}B35 zDoBIUFay9EfMPJd4^$o|&H>^B2~McE5tIgt1KHsq7XyKVp^>o(LmG(upAI75Wq`4OR;zfOZIh*lGFsB> Date: Fri, 15 Mar 2024 08:25:24 +0100 Subject: [PATCH 20/24] adding protection on the size of file/dir names --- lib/index-parser.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index 0709b31..10e2029 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -135,6 +135,9 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO metadata[i].offset_idx_file_name = datatoh64(metadata_section + offset, 8, context); metadata[i].id = datatoh64(metadata_section + offset, 16, context); metadata[i].parent_id = datatoh64(metadata_section + offset, 24, context); + if (metadata[i].file_name_size > WOWS_PATH_MAX) { + return WOWS_ERROR_PATH_TOO_LONG; + } char *file_name = calloc(sizeof(char), metadata[i].file_name_size); char *file_name_src = metadata_section + offset + metadata[i].offset_idx_file_name; @@ -176,7 +179,10 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO footer->pkg_file_name_size = datatoh64(footer_src, 0, context); footer->unknown_7 = datatoh64(footer_src, 8, context); footer->id = datatoh64(footer_src, 16, context); - ; + if (footer->pkg_file_name_size > WOWS_PATH_MAX) { + return WOWS_ERROR_PATH_TOO_LONG; + } + footer->_file_name = calloc(sizeof(char), footer->pkg_file_name_size); char *file_name_src = footer_src + SIZE_WOWS_INDEX_FOOTER; returnOutIndex(file_name_src, file_name_src + footer->pkg_file_name_size, index); From 97721c25f4b0863bd0e9146602168bce17c4c6b0 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 08:25:49 +0100 Subject: [PATCH 21/24] clean-up error messages --- lib/error-helpers.c | 18 +++++++++--------- tests/tests.c | 26 +++++++++++++++----------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/lib/error-helpers.c b/lib/error-helpers.c index 08dfb27..0b52aae 100644 --- a/lib/error-helpers.c +++ b/lib/error-helpers.c @@ -14,31 +14,31 @@ char *wows_error_string(int error_code, WOWS_CONTEXT *context) { switch (error_code) { case WOWS_ERROR_CORRUPTED_FILE: - error_string = "The index is corrupted"; + error_string = "index is corrupted"; break; case WOWS_ERROR_BAD_MAGIC: - error_string = "The index has an invalid magic number"; + error_string = "index has an invalid magic number"; break; case WOWS_ERROR_MISSING_METADATA_ENTRY: - error_string = "The file is missing a required metadata entry"; + error_string = "file is missing a required metadata entry"; break; case WOWS_ERROR_MAX_LEVEL_REACHED: - error_string = "The maximum level has been reached"; + error_string = "maximum depth level has been reached"; break; case WOWS_ERROR_NON_ZERO_TERMINATED_STRING: - error_string = "A string in the index is not null-terminated"; + error_string = "string in index file not null-terminated"; break; case WOWS_ERROR_PATH_TOO_LONG: - error_string = "The file path is too long"; + error_string = "file path too long"; break; case WOWS_ERROR_UNKNOWN: - error_string = "An unknown error occurred"; + error_string = "unknown error occurred"; break; case WOWS_ERROR_ID_COLLISION_FILE_DIR: - error_string = "There is an ID collision between a file and a directory"; + error_string = "ID collision between a file and a directory"; break; case WOWS_ERROR_FILE_OPEN_FAILURE: - error_string = "The index could not be opened"; + error_string = "index file could not be opened"; break; case WOWS_ERROR_DECOMPOSE_PATH: error_string = "failed to decompose path into [dir1, dir2, etc] + file"; diff --git a/tests/tests.c b/tests/tests.c index bbbb7cb..96b3d3c 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -86,39 +86,39 @@ void test_wows_error_string() { WOWS_CONTEXT *context = wows_init_context(0); char *error_string = wows_error_string(WOWS_ERROR_CORRUPTED_FILE, context); - CU_ASSERT_STRING_EQUAL(error_string, "The index is corrupted"); + CU_ASSERT_STRING_EQUAL(error_string, "index is corrupted"); free(error_string); error_string = wows_error_string(WOWS_ERROR_BAD_MAGIC, context); - CU_ASSERT_STRING_EQUAL(error_string, "The index has an invalid magic number"); + CU_ASSERT_STRING_EQUAL(error_string, "index has an invalid magic number"); free(error_string); error_string = wows_error_string(WOWS_ERROR_MISSING_METADATA_ENTRY, context); - CU_ASSERT_STRING_EQUAL(error_string, "The file is missing a required metadata entry"); + CU_ASSERT_STRING_EQUAL(error_string, "file is missing a required metadata entry"); free(error_string); error_string = wows_error_string(WOWS_ERROR_MAX_LEVEL_REACHED, context); - CU_ASSERT_STRING_EQUAL(error_string, "The maximum level has been reached"); + CU_ASSERT_STRING_EQUAL(error_string, "maximum depth level has been reached"); free(error_string); error_string = wows_error_string(WOWS_ERROR_NON_ZERO_TERMINATED_STRING, context); - CU_ASSERT_STRING_EQUAL(error_string, "A string in the index is not null-terminated"); + CU_ASSERT_STRING_EQUAL(error_string, "string in index file not null-terminated"); free(error_string); error_string = wows_error_string(WOWS_ERROR_PATH_TOO_LONG, context); - CU_ASSERT_STRING_EQUAL(error_string, "The file path is too long"); + CU_ASSERT_STRING_EQUAL(error_string, "file path too long"); free(error_string); error_string = wows_error_string(WOWS_ERROR_UNKNOWN, context); - CU_ASSERT_STRING_EQUAL(error_string, "An unknown error occurred"); + CU_ASSERT_STRING_EQUAL(error_string, "unknown error occurred"); free(error_string); error_string = wows_error_string(WOWS_ERROR_ID_COLLISION_FILE_DIR, context); - CU_ASSERT_STRING_EQUAL(error_string, "There is an ID collision between a file and a directory"); + CU_ASSERT_STRING_EQUAL(error_string, "ID collision between a file and a directory"); free(error_string); error_string = wows_error_string(WOWS_ERROR_FILE_OPEN_FAILURE, context); - CU_ASSERT_STRING_EQUAL(error_string, "The index could not be opened"); + CU_ASSERT_STRING_EQUAL(error_string, "index file could not be opened"); free(error_string); error_string = wows_error_string(WOWS_ERROR_DECOMPOSE_PATH, context); @@ -145,6 +145,10 @@ void test_wows_error_string() { CU_ASSERT_STRING_EQUAL(error_string, "file write error"); free(error_string); + error_string = wows_error_string(WOWS_ERROR_MAX_FILE, context); + CU_ASSERT_STRING_EQUAL(error_string, "maximum number of file/dir in index reached"); + free(error_string); + error_string = wows_error_string(9999999, context); CU_ASSERT_STRING_EQUAL(error_string, "unrecognized error code"); free(error_string); @@ -152,13 +156,13 @@ void test_wows_error_string() { // Test with a context error message. wows_set_error_details(context, "Test error message %s", "variable"); error_string = wows_error_string(WOWS_ERROR_CORRUPTED_FILE, context); - CU_ASSERT_STRING_EQUAL(error_string, "The index is corrupted: Test error message variable"); + CU_ASSERT_STRING_EQUAL(error_string, "index is corrupted: Test error message variable"); free(error_string); // Test with a context error message. wows_set_error_details(context, "Test error message 2"); error_string = wows_error_string(WOWS_ERROR_CORRUPTED_FILE, context); - CU_ASSERT_STRING_EQUAL(error_string, "The index is corrupted: Test error message 2"); + CU_ASSERT_STRING_EQUAL(error_string, "index is corrupted: Test error message 2"); free(error_string); wows_free_context(context); From fab8de720699199eab09281bef774ff17efc22ea Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 20:43:54 +0100 Subject: [PATCH 22/24] fix a few memory issues --- lib/index-parser.c | 3 ++- lib/memory-helpers.c | 36 ++++++++++++++++++++++-------------- lib/wows-depack-private.h | 1 + tests/tests.c | 6 +++--- wows-depack.c | 1 + 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/lib/index-parser.c b/lib/index-parser.c index 10e2029..d4fb6b8 100644 --- a/lib/index-parser.c +++ b/lib/index-parser.c @@ -83,6 +83,7 @@ uint64_t datatoh64(char *data, size_t offset, WOWS_CONTEXT *context) { // Map the different section of the index file content to an index struct int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context) { WOWS_INDEX *index = calloc(sizeof(WOWS_INDEX), 1); + *index_in = index; index->start_address = contents; index->end_address = contents + length; index->length = length; @@ -187,7 +188,6 @@ int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CO char *file_name_src = footer_src + SIZE_WOWS_INDEX_FOOTER; returnOutIndex(file_name_src, file_name_src + footer->pkg_file_name_size, index); strncpy(footer->_file_name, file_name_src, footer->pkg_file_name_size); - *index_in = index; return 0; } @@ -292,6 +292,7 @@ int wows_parse_index_buffer(char *contents, size_t length, const char *index_fil WOWS_INDEX *index; int err = map_index_file(contents, length, &index, context); if (err != 0) { + wows_free_index(index, 0); return err; } diff --git a/lib/memory-helpers.c b/lib/memory-helpers.c index 3029718..93fa606 100644 --- a/lib/memory-helpers.c +++ b/lib/memory-helpers.c @@ -27,20 +27,7 @@ bool checkOutOfIndex(char *start, char *end, WOWS_INDEX *index) { int wows_free_context_internal(WOWS_CONTEXT *context, int flag) { for (int i = 0; i < context->index_count; i++) { WOWS_INDEX *index = context->indexes[i]; - if (flag == FREE_WITH_CLOSE) { - munmap(index->start_address, index->length); - close(index->fd); - } - free(index->index_file_path); - free(index->header); - for (int j = 0; j < index->header->file_dir_count; j++) { - free(index->metadata[j]._file_name); - } - free(index->metadata); - free(index->data_file_entry); - free(index->footer->_file_name); - free(index->footer); - free(index); + wows_free_index(index, flag); } free_inode_tree(context->root); @@ -54,6 +41,24 @@ int wows_free_context_internal(WOWS_CONTEXT *context, int flag) { return 0; } +int wows_free_index(WOWS_INDEX *index, int flag) { + if (flag == FREE_WITH_CLOSE) { + munmap(index->start_address, index->length); + close(index->fd); + } + free(index->index_file_path); + for (int j = 0; j < index->header->file_dir_count; j++) { + free(index->metadata[j]._file_name); + } + free(index->metadata); + free(index->data_file_entry); + free(index->footer->_file_name); + free(index->footer); + free(index->header); + free(index); + return 0; +} + // Free the whole context, including un-mmaping the index files; int wows_free_context(WOWS_CONTEXT *context) { return wows_free_context_internal(context, FREE_WITH_CLOSE); @@ -72,6 +77,9 @@ bool iter_inode_free(const void *item, void *udata) { } int free_inode_tree(WOWS_BASE_INODE *inode) { + if (inode == NULL) { + return 0; + } switch (inode->type) { case WOWS_INODE_TYPE_FILE: free(inode); diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index 51bc07e..58232a6 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -154,6 +154,7 @@ typedef struct { /* ---------- */ bool checkOutOfIndex(char *start, char *end, WOWS_INDEX *index); +int wows_free_index(WOWS_INDEX *index, int flag); int wows_parse_index(char *index_file_path, WOWS_CONTEXT *context); int map_index_file(char *contents, size_t length, WOWS_INDEX **index_in, WOWS_CONTEXT *context); char *wows_render_str(char *fmt, ...); diff --git a/tests/tests.c b/tests/tests.c index 96b3d3c..23a98f4 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -296,8 +296,8 @@ static void test_map_index_file() { CU_ASSERT_EQUAL(index->footer->pkg_file_name_size, 5); CU_ASSERT_EQUAL(index->footer->id, 42); - free(index); - free(context); + wows_free_context(context); + wows_free_index(index, 0); } void test_wows_parse_index_buffer() { @@ -820,7 +820,7 @@ void test_get_pkg_filepath() { free(out); wows_free_context(context); free(contents); - free(index); + wows_free_index(index, 0); } void test_wows_search_file(void) { diff --git a/wows-depack.c b/wows-depack.c index de6ef9a..25af58f 100644 --- a/wows-depack.c +++ b/wows-depack.c @@ -116,6 +116,7 @@ int main(int argc, char **argv) { free(args); return ret; } + free(err_msg); } if (args->input == NULL && args->input_dir == NULL) { From b40a8da20443922d56824c8e730af4dfc70bd1e3 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 20:48:14 +0100 Subject: [PATCH 23/24] fix unit tests --- tests/tests.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tests.c b/tests/tests.c index 23a98f4..a2534aa 100644 --- a/tests/tests.c +++ b/tests/tests.c @@ -807,7 +807,9 @@ void test_get_pkg_filepath() { WOWS_CONTEXT *context = wows_init_context(0); int result = map_index_file(contents, 2048, &index, context); CU_ASSERT_EQUAL_FATAL(0, result); - index->index_file_path = "/path/to/wows/bin/6831266/idx/basecontent.idx"; + char *file_name = "/path/to/wows/bin/6831266/idx/basecontent.idx"; + index->index_file_path = calloc(sizeof(char), strlen(file_name) + 1); + memcpy(index->index_file_path, file_name, strlen(file_name)); char *out; result = get_pkg_filepath(index, &out); From 931af02b3e7d20e88cd3a6311ead2b640fc50363 Mon Sep 17 00:00:00 2001 From: kakwa Date: Fri, 15 Mar 2024 22:21:20 +0100 Subject: [PATCH 24/24] rework, simplification and correction of wows_writer handling of file names --- lib/deflate.c | 94 +++++++++++++++------------------------ lib/wows-depack-private.h | 8 ++-- 2 files changed, 39 insertions(+), 63 deletions(-) diff --git a/lib/deflate.c b/lib/deflate.c index 4f49bf4..e338a1d 100644 --- a/lib/deflate.c +++ b/lib/deflate.c @@ -47,22 +47,20 @@ uint64_t random_uint64() { /** Appends a file name to the input buffer at the given offset. - - @param input_buffer A pointer to a pointer to the input buffer. - @param offset A pointer to the offset in the input buffer where the file name should be written. - @param name The file name to write to the input buffer. - @param current_size A pointer to the current size of the input buffer. - - @return Returns 0 on success, or a non-zero value on failure. */ -int write_file_name(char **input_buffer, size_t *offset, char *name, size_t *current_size) { +int write_file_name(wows_writer *writer, char *name) { + char *input_buffer = writer->filename_section; + size_t offset = writer->filename_section_offset; + size_t current_size = writer->filename_section_size; size_t len = strlen(name) + 1; - if ((*current_size - *offset) < len) { - *input_buffer = realloc(*input_buffer, *current_size + CHUNK_NAME_SECTION); - *current_size += CHUNK_NAME_SECTION; - } - memcpy(*input_buffer + *offset, name, len); - *offset += len; + + char *new_buffer = calloc(sizeof(char), current_size + len); + memcpy(new_buffer, input_buffer, current_size); + free(writer->filename_section); + writer->filename_section = new_buffer; + writer->filename_section_size = current_size + len; + memcpy(writer->filename_section + offset, name, len); + writer->filename_section_offset += len; return 0; } @@ -183,37 +181,28 @@ int write_data_blob(char *file_path, FILE *pkg_fp, uint64_t *offset, uint32_t *s } /** - Writes an entry for metadata in a World of Warships (WoWS) package metadata section. +*/ - @param metadata A pointer to a pointer to an array of WOWS_INDEX_METADATA_ENTRY structs representing the metadata - section. - @param metadata_section_size A pointer to a 64-bit unsigned integer representing the size of the metadata section - array. - @param metadata_id A 64-bit unsigned integer representing the ID for the metadata entry. - @param file_name_size A 64-bit unsigned integer representing the size of the file name for the metadata entry. - @param offset_idx_file_name A 64-bit unsigned integer representing the offset of the file name in the file name - section. - @param parent_id A 64-bit unsigned integer representing the ID of the parent directory for the metadata entry. - @param file_plus_dir_count A pointer to a 64-bit unsigned integer representing the number of files and directories - in the metadata section array. +int write_metadata_entry(wows_writer *writer, uint64_t metadata_id, uint64_t parent_id, uint64_t offset_idx_file_name, + char *file_name) { + uint64_t metadata_section_size = writer->metadata_section_size; + uint64_t file_plus_dir_count = writer->file_plus_dir_count; - @return Returns 0 on success, or a non-zero value on failure. -*/ -int write_metadata_entry(WOWS_INDEX_METADATA_ENTRY **metadata, uint64_t *metadata_section_size, uint64_t metadata_id, - uint64_t file_name_size, uint64_t offset_idx_file_name, uint64_t parent_id, - uint64_t *file_plus_dir_count, char *file_name) { // If the metadata section array is full, allocate more space for it. - if ((*metadata_section_size - *file_plus_dir_count) == 0) { - *metadata = realloc(*metadata, sizeof(WOWS_INDEX_METADATA_ENTRY) * (*file_plus_dir_count + CHUNK_FILE)); - (*metadata_section_size) += CHUNK_FILE; + if ((metadata_section_size - file_plus_dir_count) == 0) { + writer->index->metadata = + realloc(writer->index->metadata, sizeof(WOWS_INDEX_METADATA_ENTRY) * (file_plus_dir_count + CHUNK_FILE)); + writer->metadata_section_size += CHUNK_FILE; } - (*metadata)[*file_plus_dir_count].id = metadata_id; - (*metadata)[*file_plus_dir_count].file_name_size = file_name_size; - (*metadata)[*file_plus_dir_count].offset_idx_file_name = offset_idx_file_name; - (*metadata)[*file_plus_dir_count].parent_id = parent_id; - (*metadata)[*file_plus_dir_count]._file_name = file_name; - (*file_plus_dir_count)++; + (writer->index->metadata)[file_plus_dir_count].id = metadata_id; + (writer->index->metadata)[file_plus_dir_count].file_name_size = strlen(file_name); + (writer->index->metadata)[file_plus_dir_count].offset_idx_file_name = offset_idx_file_name; + (writer->index->metadata)[file_plus_dir_count].parent_id = parent_id; + char *file_name_cpy = calloc(sizeof(char), strlen(file_name) + 1); + strcpy(file_name_cpy, file_name); + (writer->index->metadata)[file_plus_dir_count]._file_name = file_name_cpy; + writer->file_plus_dir_count++; return 0; } @@ -233,6 +222,8 @@ int wows_write_pkg(WOWS_CONTEXT *context, char *in_path, char *name_pkg, FILE *p index->footer->pkg_file_name_size = strlen(name_pkg) + 1; index->footer->id = writer->footer_id; index->footer->unknown_7 = random_uint64(); // FIXME dubious, but we don't know what this field does. + index->footer->_file_name = calloc(sizeof(char), strlen(name_pkg) + 1); + memcpy(index->footer->_file_name, name_pkg, strlen(name_pkg) + 1); memcpy((char *)(index->footer) + sizeof(WOWS_INDEX_FOOTER), name_pkg, strlen(name_pkg) + 1); uint64_t parent_id = random_uint64(); @@ -265,11 +256,7 @@ int wows_write_pkg(WOWS_CONTEXT *context, char *in_path, char *name_pkg, FILE *p index->header->offset_idx_data_section = 0; // Recomputed by wows_dump_index_to_file index->header->offset_idx_footer_section = 0; // Same wows_dump_index_to_file(index, idx_fp); - free(index->header); - free(index->metadata); - free(index->data_file_entry); - free(index->footer); - free(index); + wows_free_index(index, 0); free(writer); return 0; } @@ -302,13 +289,8 @@ int recursive_writer(wows_writer *writer, char *path, uint64_t parent_id) { uint64_t offset_idx_file_name = writer->filename_section_offset; // TODO use the context to check if this entry already exists (if so we must reuse the same ID) uint64_t metadata_id = random_uint64(); - - write_file_name(&(writer->filename_section), &(writer->filename_section_offset), entry->d_name, - &(writer->filename_section_size)); - write_metadata_entry(&(writer->index->metadata), &(writer->metadata_section_size), metadata_id, - (strlen(entry->d_name) + 1), offset_idx_file_name, parent_id, - &(writer->file_plus_dir_count), - writer->filename_section + writer->filename_section_offset); + write_file_name(writer, entry->d_name); + write_metadata_entry(writer, metadata_id, parent_id, offset_idx_file_name, entry->d_name); recursive_writer(writer, full_path, metadata_id); } else if (S_ISREG(info.st_mode)) { uint64_t offset_idx_file_name = writer->filename_section_offset; @@ -318,12 +300,8 @@ int recursive_writer(wows_writer *writer, char *path, uint64_t parent_id) { // Shift by 16 bits to mimic wows IDs uint64_t pkg_id = random_uint64() >> 16; write_data_blob(full_path, writer->pkg_fp, &(writer->pkg_cur_offset), &size, pkg_id); - write_file_name(&(writer->filename_section), &(writer->filename_section_offset), entry->d_name, - &(writer->filename_section_size)); - write_metadata_entry(&(writer->index->metadata), &(writer->metadata_section_size), metadata_id, - (strlen(entry->d_name) + 1), offset_idx_file_name, parent_id, - &(writer->file_plus_dir_count), - writer->filename_section + writer->filename_section_offset); + write_file_name(writer, entry->d_name); + write_metadata_entry(writer, metadata_id, parent_id, offset_idx_file_name, entry->d_name); write_file_pkg_entry(&(writer->index->data_file_entry), &(writer->file_section_size), metadata_id, writer->footer_id, start_offset, size, pkg_id, &(writer->file_count)); } diff --git a/lib/wows-depack-private.h b/lib/wows-depack-private.h index 58232a6..d2fb06f 100644 --- a/lib/wows-depack-private.h +++ b/lib/wows-depack-private.h @@ -198,16 +198,14 @@ pcre *compile_regex(const char *pattern); bool match_regex(pcre *re, const char *subject); int free_regex(pcre *re); -int write_file_name(char **input_buffer, size_t *offset, char *name, size_t *current_size); - +int write_file_name(wows_writer *writer, char *name); int write_file_pkg_entry(WOWS_INDEX_DATA_FILE_ENTRY **file_section, uint64_t *file_section_size, uint64_t metadata_id, uint64_t footer_id, uint64_t offset, uint32_t size, uint64_t pkg_id, uint64_t *file_count); int write_data_blob(char *file_path, FILE *pkg_fp, uint64_t *offset, uint32_t *size_written, uint64_t pkg_id); -int write_metadata_entry(WOWS_INDEX_METADATA_ENTRY **metadata, uint64_t *metadata_section_size, uint64_t metadata_id, - uint64_t file_name_size, uint64_t offset_idx_file_name, uint64_t parent_id, - uint64_t *file_plus_dir_count, char *file_name); +int write_metadata_entry(wows_writer *writer, uint64_t metadata_id, uint64_t parent_id, uint64_t offset_idx_file_name, + char *file_name); int recursive_writer(wows_writer *writer, char *path, uint64_t parent_id); int copy_data(FILE *in, FILE *out, long offset, size_t size); int internal_wows_extract_dir(WOWS_CONTEXT *context, char *dir_path, char *out_dir_path, FILE *magic_fp);