From 1e83c0c6d2d5fcc1257358059b8176a966de52a6 Mon Sep 17 00:00:00 2001 From: rfuzzo Date: Tue, 19 Dec 2023 20:20:25 +0100 Subject: [PATCH 01/14] int --- src/archive.rs | 474 +++++--------------------------- src/archive_file.rs | 557 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 12 +- tests/functional_tests.rs | 59 +--- 4 files changed, 638 insertions(+), 464 deletions(-) create mode 100644 src/archive_file.rs diff --git a/src/archive.rs b/src/archive.rs index ef23a21..e7cd793 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -2,35 +2,33 @@ /// ARCHIVE ///////////////////////////////////////////////////////////////////////////////////////// use std::cmp::Ordering; -use std::collections::{HashMap, HashSet}; -use std::fs::{create_dir_all, File}; -use std::io::{self, BufWriter, Read, Result, Seek, SeekFrom, Write}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, Read, Result, Write}; use std::mem; use std::path::{Path, PathBuf}; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use strum::IntoEnumIterator; -use walkdir::WalkDir; +use crate::fnv1a64_hash_string; +use crate::io::*; +use crate::kraken::*; -use crate::cr2w::read_cr2w_header; -use crate::io::{read_null_terminated_string, write_null_terminated_string, FromReader}; -use crate::kraken::{ - self, compress, decompress, get_compressed_buffer_size_needed, CompressionLevel, -}; -use crate::{fnv1a64_hash_string, sha1_hash_file, ERedExtension}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; #[derive(Debug, Clone, Default)] -pub struct Archive { - pub header: Header, - pub index: Index, +pub(crate) struct Archive { + pub(crate) header: Header, + pub(crate) index: Index, // custom - pub file_names: HashMap, + pub(crate) file_names: HashMap, } impl Archive { // Function to read a Header from a file - pub fn from_file(file_path: &PathBuf) -> Result { + pub fn from_file

(file_path: &P) -> Result + where + P: AsRef, + { let mut file = File::open(file_path)?; let mut buffer = Vec::with_capacity(mem::size_of::

()); @@ -51,7 +49,7 @@ impl Archive { let mut file_names: HashMap = HashMap::default(); if let Ok(custom_data_length) = cursor.read_u32::() { if custom_data_length > 0 { - cursor.set_position(HEADER_EXTENDED_SIZE); + cursor.set_position(Header::HEADER_EXTENDED_SIZE); if let Ok(footer) = LxrsFooter::from_reader(&mut cursor) { // add files to hashmap for f in footer.files { @@ -74,7 +72,7 @@ impl Archive { } // get filehashes - pub fn get_file_hashes(&self) -> Vec { + pub(crate) fn get_file_hashes(&self) -> Vec { self.index .file_entries .iter() @@ -83,12 +81,8 @@ impl Archive { } } -//static HEADER_MAGIC: u32 = 1380009042; -//static HEADER_SIZE: i32 = 40; -static HEADER_EXTENDED_SIZE: u64 = 0xAC; - #[derive(Debug, Clone, Copy)] -pub struct Header { +pub(crate) struct Header { pub magic: u32, pub version: u32, pub index_position: u64, @@ -98,6 +92,12 @@ pub struct Header { pub filesize: u64, } +impl Header { + //static HEADER_MAGIC: u32 = 1380009042; + //static HEADER_SIZE: i32 = 40; + pub const HEADER_EXTENDED_SIZE: u64 = 0xAC; +} + impl Default for Header { fn default() -> Self { Self { @@ -346,403 +346,59 @@ impl FromReader for LxrsFooter { } } -fn pad_until_page(writer: &mut W) -> io::Result<()> { - let pos = writer.stream_position()?; - let modulo = pos / 4096; - let diff = ((modulo + 1) * 4096) - pos; - let padding = vec![0xD9; diff as usize]; - writer.write_all(padding.as_slice())?; - - Ok(()) -} - -/// Decompresses and writes a kraken-compressed segment from an archive to a stream -/// -/// # Errors -/// -/// This function will return an error if . -fn decompress_segment( - archive_reader: &mut R, - segment: &FileSegment, - file_writer: &mut W, -) -> Result<()> { - archive_reader.seek(SeekFrom::Start(segment.offset))?; - - let magic = archive_reader.read_u32::()?; - if magic == kraken::MAGIC { - // read metadata - let mut size = segment.size; - let size_in_header = archive_reader.read_u32::()?; - if size_in_header != size { - size = size_in_header; - } - let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; - archive_reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); - - // write - file_writer.write_all(&output_buffer)?; - } else { - // incorrect data, fall back to direct copy - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - }; - - Ok(()) -} - ///////////////////////////////////////////////////////////////////////////////////////// -/// Lib +/// TESTS ///////////////////////////////////////////////////////////////////////////////////////// -/// Extracts all files from an archive and writes them to a folder -/// -/// # Panics -/// -/// Panics if file path operations fail -/// -/// # Errors -/// -/// This function will return an error if any parsing fails -pub fn extract_archive( - in_file: &PathBuf, - out_dir: &Path, - hash_map: &HashMap, -) -> io::Result<()> { - // parse archive headers - let archive = Archive::from_file(in_file)?; - - let archive_file = File::open(in_file)?; - let mut archive_reader = io::BufReader::new(archive_file); - - for (hash, file_entry) in archive.index.file_entries.iter() { - // get filename - let mut name_or_hash: String = format!("{}.bin", hash); - if let Some(name) = hash_map.get(hash) { - name_or_hash = name.to_owned(); - } - if let Some(name) = archive.file_names.get(hash) { - name_or_hash = name.to_owned(); - } +#[cfg(test)] +mod integration_tests { + use std::{ + fs, + io::{self, Read}, + path::PathBuf, + }; - // name or hash is a relative path - let outfile = out_dir.join(name_or_hash); - create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; - - // extract to stream - let mut fs = File::create(outfile)?; - let mut file_writer = BufWriter::new(&mut fs); - // decompress main file - let start_index = file_entry.segments_start; - let next_index = file_entry.segments_end; - if let Some(segment) = archive.index.file_segments.get(start_index as usize) { - // read and decompress from main archive stream - - // kraken decompress - if segment.size == segment.z_size { - // just copy over - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } else { - decompress_segment(&mut archive_reader, segment, &mut file_writer)?; - } - } + use super::LxrsFooter; + use super::{Archive, FromReader}; - // extract additional buffers - for i in start_index + 1..next_index { - if let Some(segment) = archive.index.file_segments.get(i as usize) { - // do not decompress with oodle - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } - } - } + #[test] + fn read_srxl() { + let file_path = PathBuf::from("tests").join("srxl.bin"); + let mut file = fs::File::open(file_path).expect("Could not open file"); + let mut buffer: Vec = vec![]; + file.read_to_end(&mut buffer).expect("Could not read file"); - Ok(()) -} + let mut cursor = io::Cursor::new(&buffer); -/// Packs redengine 4 resource file in a folder to an archive -/// -/// # Panics -/// -/// Panics if any path conversions fail -/// -/// # Errors -/// -/// This function will return an error if any parsing or IO fails -pub fn write_archive( - in_folder: &Path, - out_folder: &Path, - modname: Option<&str>, - hash_map: HashMap, -) -> io::Result<()> { - if !in_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); + let _srxl = LxrsFooter::from_reader(&mut cursor).unwrap(); } - if !out_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); + #[test] + fn read_archive() { + let archive_path = PathBuf::from("tests").join("test1.archive"); + let result = Archive::from_file(&archive_path); + assert!(result.is_ok()); } - let archive_name = if let Some(name) = modname { - format!("{}.archive", name) - } else { - let folder_name = in_folder.file_name().expect("Could not get in_dir name"); - format!("{}.archive", folder_name.to_string_lossy()) - }; - - // collect files - let mut included_extensions = ERedExtension::iter() - .map(|variant| variant.to_string()) - .collect::>(); - included_extensions.push(String::from("bin")); - - // get only resource files - let allfiles = WalkDir::new(in_folder) - .into_iter() - .filter_map(|e| e.ok()) - .map(|f| f.into_path()) - .filter(|p| { - if let Some(ext) = p.extension() { - if let Some(ext) = ext.to_str() { - return included_extensions.contains(&ext.to_owned()); - } - } - false - }) - .collect::>(); - - // sort by hash - let mut hashed_paths = allfiles - .iter() - .filter_map(|f| { - if let Ok(relative_path) = f.strip_prefix(in_folder) { - if let Some(path_str) = relative_path.to_str() { - let hash = fnv1a64_hash_string(&path_str.to_string()); - return Some((f.clone(), hash)); - } - } - None - }) - .collect::>(); - hashed_paths.sort_by_key(|k| k.1); - - let outfile = out_folder.join(archive_name); - let mut fs = File::create(outfile)?; - let mut archive_writer = BufWriter::new(&mut fs); - - // write temp header - let mut archive = Archive::default(); - let header = Header::default(); - header.serialize(&mut archive_writer)?; - archive_writer.write_all(&[0u8; 132])?; // some weird padding - - // write custom header - assert_eq!(HEADER_EXTENDED_SIZE, archive_writer.stream_position()?); - let custom_paths = hashed_paths - .iter() - .filter(|(_p, k)| hash_map.contains_key(k)) - .filter_map(|(f, _h)| { - if let Ok(path) = f.strip_prefix(in_folder) { - return Some(path.to_string_lossy().to_string()); - } - None - }) - .collect::>(); - - let mut custom_data_length = 0; - if !custom_paths.is_empty() { - let wfooter = LxrsFooter { - files: custom_paths, - }; - wfooter.serialize(&mut archive_writer)?; - custom_data_length = archive_writer.stream_position()? - HEADER_EXTENDED_SIZE; + #[test] + fn read_archive2() { + let archive_path = PathBuf::from("tests").join("nci.archive"); + let result = Archive::from_file(&archive_path); + assert!(result.is_ok()); } - // write files - let mut imports_hash_set: HashSet = HashSet::new(); - for (path, hash) in hashed_paths { - // read file - let mut file = File::open(&path)?; - let mut file_buffer = Vec::new(); - file.read_to_end(&mut file_buffer)?; - - let firstimportidx = imports_hash_set.len(); - let mut lastimportidx = imports_hash_set.len(); - let firstoffsetidx = archive.index.file_segments.len(); - let mut lastoffsetidx = 0; - let mut flags = 0; - - let mut file_cursor = io::Cursor::new(&file_buffer); - if let Ok(info) = read_cr2w_header(&mut file_cursor) { - // get main file - file_cursor.seek(SeekFrom::Start(0))?; - let size = info.header.objects_end; - let mut resource_buffer = vec![0; size as usize]; - file_cursor.read_exact(&mut resource_buffer[..])?; - // get archive offset before writing - let archive_offset = archive_writer.stream_position()?; - - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &resource_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - - // write compressed main file archive - // KARK header - archive_writer.write_u32::(kraken::MAGIC)?; //magic - archive_writer.write_u32::(size)?; //uncompressed buffer length - archive_writer.write_all(&compressed_buffer)?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: archive_offset, - size, - z_size: zsize as u32, - }); - - // write buffers (bytes after the main file) - for buffer_info in info.buffers_table.iter() { - let mut buffer = vec![0; buffer_info.disk_size as usize]; - file_cursor.read_exact(&mut buffer[..])?; - - let bsize = buffer_info.mem_size; - let bzsize = buffer_info.disk_size; - let boffset = archive_writer.stream_position()?; - archive_writer.write_all(buffer.as_slice())?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: boffset, - size: bsize, - z_size: bzsize, - }); - } - - //register imports - for import in info.imports.iter() { - // TODO fix flags - // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) - imports_hash_set.insert(import.depot_path.to_owned()); - } - - lastimportidx = imports_hash_set.len(); - lastoffsetidx = archive.index.file_segments.len(); - flags = if !info.buffers_table.is_empty() { - info.buffers_table.len() - 1 - } else { - 0 - }; - } else { - // write non-cr2w file - file_cursor.seek(SeekFrom::Start(0))?; - if let Some(os_ext) = path.extension() { - let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); - if get_aligned_file_extensions().contains(&ext) { - pad_until_page(&mut archive_writer)?; - } - - let offset = archive_writer.stream_position()?; - let size = file_buffer.len() as u32; - let mut final_zsize = file_buffer.len() as u32; - if get_uncompressed_file_extensions().contains(&ext) { - // direct copy - archive_writer.write_all(&file_buffer)?; - } else { - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &file_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - final_zsize = zsize as u32; - // write - archive_writer.write_all(&compressed_buffer)?; - } - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset, - size, - z_size: final_zsize, - }); - } - } - - // update archive metadata - let sha1_hash = sha1_hash_file(&file_buffer); - - let entry = FileEntry { - name_hash_64: hash, - timestamp: 0, // TODO proper timestamps - num_inline_buffer_segments: flags as u32, - segments_start: firstoffsetidx as u32, - segments_end: lastoffsetidx as u32, - resource_dependencies_start: firstimportidx as u32, - resource_dependencies_end: lastimportidx as u32, - sha1_hash, - }; - archive.index.file_entries.insert(hash, entry); + #[test] + fn read_custom_data() { + let archive_path = PathBuf::from("tests").join("test1.archive"); + let archive = Archive::from_file(&archive_path).expect("Could not parse archive"); + let mut file_names = archive + .file_names + .values() + .map(|f| f.to_owned()) + .collect::>(); + file_names.sort(); + + let expected: Vec = vec!["base\\cycleweapons\\localization\\en-us.json".to_owned()]; + assert_eq!(expected, file_names); } - - // write footers - // padding - pad_until_page(&mut archive_writer)?; - - // write tables - let tableoffset = archive_writer.stream_position()?; - archive.index.serialize(&mut archive_writer)?; - let tablesize = archive_writer.stream_position()? - tableoffset; - - // padding - pad_until_page(&mut archive_writer)?; - let filesize = archive_writer.stream_position()?; - - // write the header again - archive.header.index_position = tableoffset; - archive.header.index_size = tablesize as u32; - archive.header.filesize = filesize; - archive_writer.seek(SeekFrom::Start(0))?; - archive.header.serialize(&mut archive_writer)?; - archive_writer.write_u32::(custom_data_length as u32)?; - - Ok(()) -} - -/// . -fn get_aligned_file_extensions() -> Vec { - let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; - files.into_iter().map(|f| f.to_owned()).collect::>() -} - -/// . -fn get_uncompressed_file_extensions() -> Vec { - let files = vec![ - ".bk2", - ".bnk", - ".opusinfo", - ".wem", - ".bin", - ".dat", - ".opuspak", - ]; - files.into_iter().map(|f| f.to_owned()).collect::>() } diff --git a/src/archive_file.rs b/src/archive_file.rs new file mode 100644 index 0000000..3d46e0d --- /dev/null +++ b/src/archive_file.rs @@ -0,0 +1,557 @@ +///////////////////////////////////////////////////////////////////////////////////////// +// ARCHIVE_FILE +// https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.zipfile?view=net-8.0#methods +// ZipFile -> namespace +// Provides static methods for creating, extracting, and opening zip archives. +// +// ZipArchive -> Archive +// Represents a package of compressed files in the zip archive format. +// +// ZipArchiveEntry -> ArchiveEntry +// Represents a compressed file within a zip archive. +///////////////////////////////////////////////////////////////////////////////////////// + +use std::{ + collections::{HashMap, HashSet}, + fs::{create_dir_all, File}, + io::{self, BufWriter, Read, Result, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use strum::IntoEnumIterator; +use walkdir::WalkDir; + +use crate::{ + archive::*, + cr2w::*, + fnv1a64_hash_string, get_red4_hashes, + kraken::{self, *}, + sha1_hash_file, ERedExtension, +}; + +// public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); + +/// Creates a zip archive in the specified stream that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory( + source_directory_name: &P, + destination: W, + hash_map: Option>, +) -> io::Result<()> +where + P: AsRef, + W: Write + Seek, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + write_archive(source_directory_name, destination, map) +} + +// public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); + +/// Creates a zip archive that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory_path

( + source_directory_name: &P, + destination: &P, + hash_map: Option>, +) -> io::Result<()> +where + P: AsRef, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + let fs: File = File::create(destination)?; + write_archive(source_directory_name, fs, map) +} + +// public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); + +/// Extracts all the files from the zip archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory( + source: R, + destination_directory_name: P, + overwrite_files: bool, +) -> io::Result<()> +where + P: AsRef, + R: Read, +{ +} + +// public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); + +/// Extracts all of the files in the specified archive to a directory on the file system. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory_path

( + source_archive_file_name: P, + destination_directory_name: P, + overwrite_files: bool, +) -> io::Result<()> +where + P: AsRef, +{ + extract_archive( + source_archive_file_name, + destination_directory_name, + overwrite_files, + ) +} + +pub enum ArchiveMode { + Create, + Read, + Update, +} + +// public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); + +/// Opens a zip archive at the specified path and in the specified mode. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open

(archive_file_name: P, mode: ArchiveMode) -> io::Result +where + P: AsRef, +{ + todo!() +} + +// public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); + +/// Opens a zip archive for reading at the specified path. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open_read

(archive_file_name: P) -> io::Result +where + P: AsRef, +{ + todo!() +} + +///////////////////////////////////////////////////////////////////////////////////////// +/// Lib +///////////////////////////////////////////////////////////////////////////////////////// + +/// Extracts all files from an archive and writes them to a folder +/// +/// # Panics +/// +/// Panics if file path operations fail +/// +/// # Errors +/// +/// This function will return an error if any parsing fails +fn extract_archive

(in_file: &P, out_dir: &P, hash_map: &HashMap) -> io::Result<()> +where + P: AsRef, +{ + // parse archive headers + let archive = Archive::from_file(in_file)?; + + let archive_file = File::open(in_file)?; + let mut archive_reader = io::BufReader::new(archive_file); + + for (hash, file_entry) in archive.index.file_entries.iter() { + // get filename + let mut name_or_hash: String = format!("{}.bin", hash); + if let Some(name) = hash_map.get(hash) { + name_or_hash = name.to_owned(); + } + if let Some(name) = archive.file_names.get(hash) { + name_or_hash = name.to_owned(); + } + + // name or hash is a relative path + let mut: PathBuf + let outfile = out_dir.join(name_or_hash); + create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; + + // extract to stream + let mut fs = File::create(outfile)?; + let mut file_writer = BufWriter::new(&mut fs); + // decompress main file + let start_index = file_entry.segments_start; + let next_index = file_entry.segments_end; + if let Some(segment) = archive.index.file_segments.get(start_index as usize) { + // read and decompress from main archive stream + + // kraken decompress + if segment.size == segment.z_size { + // just copy over + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } else { + decompress_segment(&mut archive_reader, segment, &mut file_writer)?; + } + } + + // extract additional buffers + for i in start_index + 1..next_index { + if let Some(segment) = archive.index.file_segments.get(i as usize) { + // do not decompress with oodle + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } + } + } + + Ok(()) +} + +/// Packs redengine 4 resource file in a folder to an archive +/// +/// # Panics +/// +/// Panics if any path conversions fail +/// +/// # Errors +/// +/// This function will return an error if any parsing or IO fails +fn write_archive( + in_folder: &P, + out_stream: W, + hash_map: HashMap, +) -> io::Result<()> +where + P: AsRef, + W: Write + Seek, +{ + /*if !in_folder.exists() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); + } + + if !out_folder.exists() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); + } + // check extension + if !out_folder.exists() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); + }*/ + + // collect files + let mut included_extensions = ERedExtension::iter() + .map(|variant| variant.to_string()) + .collect::>(); + included_extensions.push(String::from("bin")); + + // get only resource files + let allfiles = WalkDir::new(in_folder) + .into_iter() + .filter_map(|e| e.ok()) + .map(|f| f.into_path()) + .filter(|p| { + if let Some(ext) = p.extension() { + if let Some(ext) = ext.to_str() { + return included_extensions.contains(&ext.to_owned()); + } + } + false + }) + .collect::>(); + + // sort by hash + let mut hashed_paths = allfiles + .iter() + .filter_map(|f| { + if let Ok(relative_path) = f.strip_prefix(in_folder) { + if let Some(path_str) = relative_path.to_str() { + let hash = fnv1a64_hash_string(&path_str.to_string()); + return Some((f.clone(), hash)); + } + } + None + }) + .collect::>(); + hashed_paths.sort_by_key(|k| k.1); + + let mut archive_writer = BufWriter::new(out_stream); + + // write temp header + let mut archive = Archive::default(); + let header = Header::default(); + header.serialize(&mut archive_writer)?; + archive_writer.write_all(&[0u8; 132])?; // some weird padding + + // write custom header + assert_eq!( + Header::HEADER_EXTENDED_SIZE, + archive_writer.stream_position()? + ); + let custom_paths = hashed_paths + .iter() + .filter(|(_p, k)| hash_map.contains_key(k)) + .filter_map(|(f, _h)| { + if let Ok(path) = f.strip_prefix(in_folder) { + return Some(path.to_string_lossy().to_string()); + } + None + }) + .collect::>(); + + let mut custom_data_length = 0; + if !custom_paths.is_empty() { + let wfooter = LxrsFooter { + files: custom_paths, + }; + wfooter.serialize(&mut archive_writer)?; + custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; + } + + // write files + let mut imports_hash_set: HashSet = HashSet::new(); + for (path, hash) in hashed_paths { + // read file + let mut file = File::open(&path)?; + let mut file_buffer = Vec::new(); + file.read_to_end(&mut file_buffer)?; + + let firstimportidx = imports_hash_set.len(); + let mut lastimportidx = imports_hash_set.len(); + let firstoffsetidx = archive.index.file_segments.len(); + let mut lastoffsetidx = 0; + let mut flags = 0; + + let mut file_cursor = io::Cursor::new(&file_buffer); + if let Ok(info) = read_cr2w_header(&mut file_cursor) { + // get main file + file_cursor.seek(SeekFrom::Start(0))?; + let size = info.header.objects_end; + let mut resource_buffer = vec![0; size as usize]; + file_cursor.read_exact(&mut resource_buffer[..])?; + // get archive offset before writing + let archive_offset = archive_writer.stream_position()?; + + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &resource_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + + // write compressed main file archive + // KARK header + archive_writer.write_u32::(kraken::MAGIC)?; //magic + archive_writer.write_u32::(size)?; //uncompressed buffer length + archive_writer.write_all(&compressed_buffer)?; + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset: archive_offset, + size, + z_size: zsize as u32, + }); + + // write buffers (bytes after the main file) + for buffer_info in info.buffers_table.iter() { + let mut buffer = vec![0; buffer_info.disk_size as usize]; + file_cursor.read_exact(&mut buffer[..])?; + + let bsize = buffer_info.mem_size; + let bzsize = buffer_info.disk_size; + let boffset = archive_writer.stream_position()?; + archive_writer.write_all(buffer.as_slice())?; + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset: boffset, + size: bsize, + z_size: bzsize, + }); + } + + //register imports + for import in info.imports.iter() { + // TODO fix flags + // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) + imports_hash_set.insert(import.depot_path.to_owned()); + } + + lastimportidx = imports_hash_set.len(); + lastoffsetidx = archive.index.file_segments.len(); + flags = if !info.buffers_table.is_empty() { + info.buffers_table.len() - 1 + } else { + 0 + }; + } else { + // write non-cr2w file + file_cursor.seek(SeekFrom::Start(0))?; + if let Some(os_ext) = path.extension() { + let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); + if get_aligned_file_extensions().contains(&ext) { + pad_until_page(&mut archive_writer)?; + } + + let offset = archive_writer.stream_position()?; + let size = file_buffer.len() as u32; + let mut final_zsize = file_buffer.len() as u32; + if get_uncompressed_file_extensions().contains(&ext) { + // direct copy + archive_writer.write_all(&file_buffer)?; + } else { + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &file_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + final_zsize = zsize as u32; + // write + archive_writer.write_all(&compressed_buffer)?; + } + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset, + size, + z_size: final_zsize, + }); + } + } + + // update archive metadata + let sha1_hash = sha1_hash_file(&file_buffer); + + let entry = FileEntry { + name_hash_64: hash, + timestamp: 0, // TODO proper timestamps + num_inline_buffer_segments: flags as u32, + segments_start: firstoffsetidx as u32, + segments_end: lastoffsetidx as u32, + resource_dependencies_start: firstimportidx as u32, + resource_dependencies_end: lastimportidx as u32, + sha1_hash, + }; + archive.index.file_entries.insert(hash, entry); + } + + // write footers + // padding + pad_until_page(&mut archive_writer)?; + + // write tables + let tableoffset = archive_writer.stream_position()?; + archive.index.serialize(&mut archive_writer)?; + let tablesize = archive_writer.stream_position()? - tableoffset; + + // padding + pad_until_page(&mut archive_writer)?; + let filesize = archive_writer.stream_position()?; + + // write the header again + archive.header.index_position = tableoffset; + archive.header.index_size = tablesize as u32; + archive.header.filesize = filesize; + archive_writer.seek(SeekFrom::Start(0))?; + archive.header.serialize(&mut archive_writer)?; + archive_writer.write_u32::(custom_data_length as u32)?; + + Ok(()) +} + +/// Decompresses and writes a kraken-compressed segment from an archive to a stream +/// +/// # Errors +/// +/// This function will return an error if . +fn decompress_segment( + archive_reader: &mut R, + segment: &FileSegment, + file_writer: &mut W, +) -> Result<()> { + archive_reader.seek(SeekFrom::Start(segment.offset))?; + + let magic = archive_reader.read_u32::()?; + if magic == kraken::MAGIC { + // read metadata + let mut size = segment.size; + let size_in_header = archive_reader.read_u32::()?; + if size_in_header != size { + size = size_in_header; + } + let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; + archive_reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); + + // write + file_writer.write_all(&output_buffer)?; + } else { + // incorrect data, fall back to direct copy + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + }; + + Ok(()) +} + +/// . +fn get_aligned_file_extensions() -> Vec { + let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; + files.into_iter().map(|f| f.to_owned()).collect::>() +} + +/// . +fn get_uncompressed_file_extensions() -> Vec { + let files = vec![ + ".bk2", + ".bnk", + ".opusinfo", + ".wem", + ".bin", + ".dat", + ".opuspak", + ]; + files.into_iter().map(|f| f.to_owned()).collect::>() +} + +fn pad_until_page(writer: &mut W) -> io::Result<()> { + let pos = writer.stream_position()?; + let modulo = pos / 4096; + let diff = ((modulo + 1) * 4096) - pos; + let padding = vec![0xD9; diff as usize]; + writer.write_all(padding.as_slice())?; + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index acf07f2..f19959b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,11 @@ #![warn(clippy::all, rust_2018_idioms)] -pub mod archive; -pub mod cr2w; -pub mod io; -pub mod kraken; +pub(crate) mod archive; +mod cr2w; +mod io; +mod kraken; + +pub mod archive_file; use std::collections::HashMap; use std::fs::{self}; @@ -233,7 +235,7 @@ pub fn get_red4_hashes() -> HashMap { } ///////////////////////////////////////////////////////////////////////////////////////// -/// TESTS +// TESTS ///////////////////////////////////////////////////////////////////////////////////////// #[cfg(test)] diff --git a/tests/functional_tests.rs b/tests/functional_tests.rs index d08e81c..57c38c4 100644 --- a/tests/functional_tests.rs +++ b/tests/functional_tests.rs @@ -1,17 +1,14 @@ ///////////////////////////////////////////////////////////////////////////////////////// -/// TESTS +// TESTS ///////////////////////////////////////////////////////////////////////////////////////// #[cfg(test)] mod tests { use std::fs::create_dir_all; - use std::io::{self, Read}; - use std::path::Path; + //use std::path::Path; use std::time::Instant; use std::{fs, path::PathBuf}; - use red4lib::archive::*; - use red4lib::io::FromReader; use red4lib::*; #[test] @@ -24,47 +21,7 @@ mod tests { println!("Execution time csv: {:?}", duration); } - #[test] - fn read_srxl() { - let file_path = PathBuf::from("tests").join("srxl.bin"); - let mut file = fs::File::open(file_path).expect("Could not open file"); - let mut buffer: Vec = vec![]; - file.read_to_end(&mut buffer).expect("Could not read file"); - - let mut cursor = io::Cursor::new(&buffer); - - let _srxl = LxrsFooter::from_reader(&mut cursor).unwrap(); - } - - #[test] - fn read_archive() { - let archive_path = PathBuf::from("tests").join("test1.archive"); - let result = Archive::from_file(&archive_path); - assert!(result.is_ok()); - } - - #[test] - fn read_archive2() { - let archive_path = PathBuf::from("tests").join("nci.archive"); - let result = Archive::from_file(&archive_path); - assert!(result.is_ok()); - } - - #[test] - fn read_custom_data() { - let archive_path = PathBuf::from("tests").join("test1.archive"); - let archive = Archive::from_file(&archive_path).expect("Could not parse archive"); - let mut file_names = archive - .file_names - .values() - .map(|f| f.to_owned()) - .collect::>(); - file_names.sort(); - - let expected: Vec = vec!["base\\cycleweapons\\localization\\en-us.json".to_owned()]; - assert_eq!(expected, file_names); - } - + /* #[test] fn test_extract_archive() { let archive_path = PathBuf::from("tests").join("test1.archive"); @@ -137,13 +94,13 @@ mod tests { assert!(fs::remove_dir_all(&dst_path).is_ok()); } } + */ #[test] fn test_pack_archive() { // pack test data let data_path = PathBuf::from("tests").join("data"); - let dst_path = PathBuf::from("tests").join("out2"); - let hash_map = get_red4_hashes(); + let dst_path = PathBuf::from("tests").join("out2").join("data.archive"); // delete folder if exists if dst_path.exists() { @@ -151,7 +108,7 @@ mod tests { } create_dir_all(&dst_path).expect("Could not create folder"); - let result = write_archive(&data_path, &dst_path, None, hash_map); + let result = archive_file::create_from_directory_path(&data_path, &dst_path, None); assert!(result.is_ok()); // checks @@ -169,9 +126,10 @@ mod tests { } ///////////////////////////////////////////////////////////////////////////////////////// - /// HELPERS + // HELPERS ///////////////////////////////////////////////////////////////////////////////////////// + /* fn assert_binary_equality(e: &PathBuf, f: &PathBuf) { // compare bytes let mut fe = fs::File::open(e).expect("Could not open file"); @@ -213,4 +171,5 @@ mod tests { // Return an empty vector if there's an error Vec::new() } + */ } From 3f5907f669b6c3752d4ebecd76d61364fb9cc9bb Mon Sep 17 00:00:00 2001 From: rfuzzo Date: Wed, 20 Dec 2023 18:34:03 +0100 Subject: [PATCH 02/14] fix build --- src/archive.rs | 43 +++++++++++--------- src/archive_file.rs | 82 ++++++++++++++++++++++++++++----------- tests/functional_tests.rs | 8 ++-- 3 files changed, 88 insertions(+), 45 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index e7cd793..158d380 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -4,9 +4,9 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::fs::File; -use std::io::{self, Read, Result, Write}; +use std::io::{self, Read, Result, Seek, Write}; use std::mem; -use std::path::{Path, PathBuf}; +use std::path::Path; use crate::fnv1a64_hash_string; use crate::io::*; @@ -15,23 +15,22 @@ use crate::kraken::*; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; #[derive(Debug, Clone, Default)] -pub(crate) struct Archive { - pub(crate) header: Header, - pub(crate) index: Index, +pub struct Archive { + pub header: Header, + pub index: Index, // custom - pub(crate) file_names: HashMap, + pub file_names: HashMap, } impl Archive { // Function to read a Header from a file - pub fn from_file

(file_path: &P) -> Result + pub fn from_file

(file_path: P) -> Result where P: AsRef, { let mut file = File::open(file_path)?; let mut buffer = Vec::with_capacity(mem::size_of::

()); - file.read_to_end(&mut buffer)?; // Ensure that the buffer has enough bytes to represent a Header @@ -43,14 +42,22 @@ impl Archive { } let mut cursor = io::Cursor::new(&buffer); - let header = Header::from_reader(&mut cursor)?; + + Archive::from_reader(&mut cursor) + } + + pub fn from_reader(cursor: &mut R) -> Result + where + R: Read + Seek, + { + let header = Header::from_reader(cursor)?; // read custom data let mut file_names: HashMap = HashMap::default(); if let Ok(custom_data_length) = cursor.read_u32::() { if custom_data_length > 0 { - cursor.set_position(Header::HEADER_EXTENDED_SIZE); - if let Ok(footer) = LxrsFooter::from_reader(&mut cursor) { + cursor.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; + if let Ok(footer) = LxrsFooter::from_reader(cursor) { // add files to hashmap for f in footer.files { let hash = fnv1a64_hash_string(&f); @@ -61,8 +68,8 @@ impl Archive { } // move to offset Header.IndexPosition - cursor.set_position(header.index_position); - let index = Index::from_reader(&mut cursor)?; + cursor.seek(io::SeekFrom::Start(header.index_position))?; + let index = Index::from_reader(cursor)?; Ok(Archive { header, @@ -72,7 +79,7 @@ impl Archive { } // get filehashes - pub(crate) fn get_file_hashes(&self) -> Vec { + pub fn get_file_hashes(&self) -> Vec { self.index .file_entries .iter() @@ -82,7 +89,7 @@ impl Archive { } #[derive(Debug, Clone, Copy)] -pub(crate) struct Header { +pub struct Header { pub magic: u32, pub version: u32, pub index_position: u64, @@ -376,21 +383,21 @@ mod integration_tests { #[test] fn read_archive() { let archive_path = PathBuf::from("tests").join("test1.archive"); - let result = Archive::from_file(&archive_path); + let result = Archive::from_file(archive_path); assert!(result.is_ok()); } #[test] fn read_archive2() { let archive_path = PathBuf::from("tests").join("nci.archive"); - let result = Archive::from_file(&archive_path); + let result = Archive::from_file(archive_path); assert!(result.is_ok()); } #[test] fn read_custom_data() { let archive_path = PathBuf::from("tests").join("test1.archive"); - let archive = Archive::from_file(&archive_path).expect("Could not parse archive"); + let archive = Archive::from_file(archive_path).expect("Could not parse archive"); let mut file_names = archive .file_names .values() diff --git a/src/archive_file.rs b/src/archive_file.rs index 3d46e0d..2cf67d3 100644 --- a/src/archive_file.rs +++ b/src/archive_file.rs @@ -15,7 +15,7 @@ use std::{ collections::{HashMap, HashSet}, fs::{create_dir_all, File}, io::{self, BufWriter, Read, Result, Seek, SeekFrom, Write}, - path::{Path, PathBuf}, + path::Path, }; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; @@ -32,7 +32,7 @@ use crate::{ // public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); -/// Creates a zip archive in the specified stream that contains the files and directories from the specified directory. +/// Creates an archive in the specified stream that contains the files and directories from the specified directory. /// /// # Errors /// @@ -57,7 +57,7 @@ where // public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); -/// Creates a zip archive that contains the files and directories from the specified directory. +/// Creates an archive that contains the files and directories from the specified directory. /// /// # Errors /// @@ -82,20 +82,28 @@ where // public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); -/// Extracts all the files from the zip archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. +/// Extracts all the files from the archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. /// /// # Errors /// /// This function will return an error if any io fails. pub fn extract_to_directory( - source: R, - destination_directory_name: P, + source: &mut R, + destination_directory_name: &P, overwrite_files: bool, + hash_map: Option>, ) -> io::Result<()> where P: AsRef, - R: Read, + R: Read + Seek, { + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + extract_archive(source, destination_directory_name, overwrite_files, &map) } // public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); @@ -106,17 +114,28 @@ where /// /// This function will return an error if any io fails. pub fn extract_to_directory_path

( - source_archive_file_name: P, - destination_directory_name: P, + source_archive_file_name: &P, + destination_directory_name: &P, overwrite_files: bool, + hash_map: Option>, ) -> io::Result<()> where P: AsRef, { + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + let archive_file = File::open(source_archive_file_name)?; + let mut archive_reader = io::BufReader::new(archive_file); + extract_archive( - source_archive_file_name, + &mut archive_reader, destination_directory_name, overwrite_files, + &map, ) } @@ -126,9 +145,14 @@ pub enum ArchiveMode { Update, } +/* +TODO We don't support different modes for now +needs a wrapper class for archives + + // public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); -/// Opens a zip archive at the specified path and in the specified mode. +/// Opens an archive at the specified path and in the specified mode. /// /// # Errors /// @@ -140,9 +164,11 @@ where todo!() } + */ + // public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); -/// Opens a zip archive for reading at the specified path. +/// Opens an archive for reading at the specified path. /// /// # Errors /// @@ -151,7 +177,7 @@ pub fn open_read

(archive_file_name: P) -> io::Result where P: AsRef, { - todo!() + Archive::from_file(archive_file_name) } ///////////////////////////////////////////////////////////////////////////////////////// @@ -167,15 +193,18 @@ where /// # Errors /// /// This function will return an error if any parsing fails -fn extract_archive

(in_file: &P, out_dir: &P, hash_map: &HashMap) -> io::Result<()> +fn extract_archive( + archive_reader: &mut R, + out_dir: &P, + overwrite_files: bool, + hash_map: &HashMap, +) -> io::Result<()> where P: AsRef, + R: Read + Seek, { // parse archive headers - let archive = Archive::from_file(in_file)?; - - let archive_file = File::open(in_file)?; - let mut archive_reader = io::BufReader::new(archive_file); + let archive = Archive::from_reader(archive_reader)?; for (hash, file_entry) in archive.index.file_entries.iter() { // get filename @@ -188,12 +217,21 @@ where } // name or hash is a relative path - let mut: PathBuf - let outfile = out_dir.join(name_or_hash); + let outfile = out_dir.as_ref().join(name_or_hash); create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; // extract to stream - let mut fs = File::create(outfile)?; + let mut fs = if overwrite_files { + File::create(outfile)? + } else { + File::options() + .read(true) + .write(true) + .create_new(true) + .open(outfile)? + }; + + //let mut fs = File::create(outfile)?; let mut file_writer = BufWriter::new(&mut fs); // decompress main file let start_index = file_entry.segments_start; @@ -209,7 +247,7 @@ where archive_reader.read_exact(&mut buffer[..])?; file_writer.write_all(&buffer)?; } else { - decompress_segment(&mut archive_reader, segment, &mut file_writer)?; + decompress_segment(archive_reader, segment, &mut file_writer)?; } } diff --git a/tests/functional_tests.rs b/tests/functional_tests.rs index 57c38c4..260db33 100644 --- a/tests/functional_tests.rs +++ b/tests/functional_tests.rs @@ -5,10 +5,12 @@ #[cfg(test)] mod tests { use std::fs::create_dir_all; + use std::path::Path; //use std::path::Path; use std::time::Instant; use std::{fs, path::PathBuf}; + use red4lib::archive_file::extract_to_directory_path; use red4lib::*; #[test] @@ -21,7 +23,6 @@ mod tests { println!("Execution time csv: {:?}", duration); } - /* #[test] fn test_extract_archive() { let archive_path = PathBuf::from("tests").join("test1.archive"); @@ -34,7 +35,7 @@ mod tests { assert!(fs::remove_dir_all(&dst_path).is_ok()); } - let result = extract_archive(&archive_path, &dst_path, &hashes); + let result = extract_to_directory_path(&archive_path, &dst_path, true, Some(hashes)); assert!(result.is_ok()); // check @@ -94,7 +95,6 @@ mod tests { assert!(fs::remove_dir_all(&dst_path).is_ok()); } } - */ #[test] fn test_pack_archive() { @@ -129,7 +129,6 @@ mod tests { // HELPERS ///////////////////////////////////////////////////////////////////////////////////////// - /* fn assert_binary_equality(e: &PathBuf, f: &PathBuf) { // compare bytes let mut fe = fs::File::open(e).expect("Could not open file"); @@ -171,5 +170,4 @@ mod tests { // Return an empty vector if there's an error Vec::new() } - */ } From 2ca801876419e2b20622ddfce365bb06b9b529e2 Mon Sep 17 00:00:00 2001 From: rfuzzo Date: Wed, 20 Dec 2023 19:05:39 +0100 Subject: [PATCH 03/14] Update functional_tests.rs --- tests/functional_tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional_tests.rs b/tests/functional_tests.rs index 260db33..6b61f7a 100644 --- a/tests/functional_tests.rs +++ b/tests/functional_tests.rs @@ -100,7 +100,8 @@ mod tests { fn test_pack_archive() { // pack test data let data_path = PathBuf::from("tests").join("data"); - let dst_path = PathBuf::from("tests").join("out2").join("data.archive"); + let dst_path = PathBuf::from("tests").join("out2"); + let dst_file = dst_path.join("data.archive"); // delete folder if exists if dst_path.exists() { @@ -108,12 +109,11 @@ mod tests { } create_dir_all(&dst_path).expect("Could not create folder"); - let result = archive_file::create_from_directory_path(&data_path, &dst_path, None); + let result = archive_file::create_from_directory_path(&data_path, &dst_file, None); assert!(result.is_ok()); // checks - let created_path = dst_path.join("data.archive"); - assert!(created_path.exists()); + assert!(dst_file.exists()); // TODO binary equality // let existing_path = PathBuf::from("tests").join("test1.archive"); From da60f24ca40ac15dcc31131dfdcffb554f33466b Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Thu, 21 Dec 2023 07:34:43 +0100 Subject: [PATCH 04/14] refactor api --- src/archive.rs | 592 ++++++++++++++++++++++++++++++++++++- src/archive_file.rs | 595 -------------------------------------- src/lib.rs | 47 +-- tests/functional_tests.rs | 7 +- 4 files changed, 600 insertions(+), 641 deletions(-) delete mode 100644 src/archive_file.rs diff --git a/src/archive.rs b/src/archive.rs index 158d380..bb518b1 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -2,17 +2,22 @@ /// ARCHIVE ///////////////////////////////////////////////////////////////////////////////////////// use std::cmp::Ordering; -use std::collections::HashMap; -use std::fs::File; -use std::io::{self, Read, Result, Seek, Write}; -use std::mem; -use std::path::Path; +use std::{ + collections::{HashMap, HashSet}, + fs::{create_dir_all, File}, + io::{BufReader, BufWriter, Cursor, Read, Result, Seek, SeekFrom, Write}, + path::Path, +}; +use std::{io, mem}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use strum::IntoEnumIterator; +use walkdir::WalkDir; use crate::fnv1a64_hash_string; use crate::io::*; use crate::kraken::*; - -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use crate::{cr2w::*, *}; #[derive(Debug, Clone, Default)] pub struct Archive { @@ -353,6 +358,579 @@ impl FromReader for LxrsFooter { } } +///////////////////////////////////////////////////////////////////////////////////////// +// ARCHIVE_FILE +// https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.zipfile?view=net-8.0#methods +// ZipFile -> namespace +// Provides static methods for creating, extracting, and opening zip archives. +// +// ZipArchive -> Archive +// Represents a package of compressed files in the zip archive format. +// +// ZipArchiveEntry -> ArchiveEntry +// Represents a compressed file within a zip archive. +///////////////////////////////////////////////////////////////////////////////////////// + +// public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); + +/// Creates an archive in the specified stream that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory( + source_directory_name: &P, + destination: W, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + W: Write + Seek, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + write_archive(source_directory_name, destination, map) +} + +// public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); + +/// Creates an archive that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory_path

( + source_directory_name: &P, + destination: &P, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + let fs: File = File::create(destination)?; + write_archive(source_directory_name, fs, map) +} + +// public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); + +/// Extracts all the files from the archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory( + source: &mut R, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + R: Read + Seek, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + extract_archive(source, destination_directory_name, overwrite_files, &map) +} + +// public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); + +/// Extracts all of the files in the specified archive to a directory on the file system. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory_path

( + source_archive_file_name: &P, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + let archive_file = File::open(source_archive_file_name)?; + let mut archive_reader = BufReader::new(archive_file); + + extract_archive( + &mut archive_reader, + destination_directory_name, + overwrite_files, + &map, + ) +} + +pub enum ArchiveMode { + Create, + Read, + Update, +} + +/* +TODO We don't support different modes for now +needs a wrapper class for archives + + +// public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); + +/// Opens an archive at the specified path and in the specified mode. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open

(archive_file_name: P, mode: ArchiveMode) -> Result +where + P: AsRef, +{ + todo!() +} + + */ + +// public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); + +/// Opens an archive for reading at the specified path. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open_read

(archive_file_name: P) -> Result +where + P: AsRef, +{ + Archive::from_file(archive_file_name) +} + +///////////////////////////////////////////////////////////////////////////////////////// +/// Lib +///////////////////////////////////////////////////////////////////////////////////////// + +/// Extracts all files from an archive and writes them to a folder +/// +/// # Panics +/// +/// Panics if file path operations fail +/// +/// # Errors +/// +/// This function will return an error if any parsing fails +fn extract_archive( + archive_reader: &mut R, + out_dir: &P, + overwrite_files: bool, + hash_map: &HashMap, +) -> Result<()> +where + P: AsRef, + R: Read + Seek, +{ + // parse archive headers + let archive = Archive::from_reader(archive_reader)?; + + for (hash, file_entry) in archive.index.file_entries.iter() { + // get filename + let mut name_or_hash: String = format!("{}.bin", hash); + if let Some(name) = hash_map.get(hash) { + name_or_hash = name.to_owned(); + } + if let Some(name) = archive.file_names.get(hash) { + name_or_hash = name.to_owned(); + } + + // name or hash is a relative path + let outfile = out_dir.as_ref().join(name_or_hash); + create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; + + // extract to stream + let mut fs = if overwrite_files { + File::create(outfile)? + } else { + File::options() + .read(true) + .write(true) + .create_new(true) + .open(outfile)? + }; + + //let mut fs = File::create(outfile)?; + let mut file_writer = BufWriter::new(&mut fs); + // decompress main file + let start_index = file_entry.segments_start; + let next_index = file_entry.segments_end; + if let Some(segment) = archive.index.file_segments.get(start_index as usize) { + // read and decompress from main archive stream + + // kraken decompress + if segment.size == segment.z_size { + // just copy over + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } else { + decompress_segment(archive_reader, segment, &mut file_writer)?; + } + } + + // extract additional buffers + for i in start_index + 1..next_index { + if let Some(segment) = archive.index.file_segments.get(i as usize) { + // do not decompress with oodle + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } + } + } + + Ok(()) +} + +/// Packs redengine 4 resource file in a folder to an archive +/// +/// # Panics +/// +/// Panics if any path conversions fail +/// +/// # Errors +/// +/// This function will return an error if any parsing or IO fails +fn write_archive(in_folder: &P, out_stream: W, hash_map: HashMap) -> Result<()> +where + P: AsRef, + W: Write + Seek, +{ + /*if !in_folder.exists() { + return Err(Error::new(ErrorKind::InvalidInput, "")); + } + + if !out_folder.exists() { + return Err(Error::new(ErrorKind::InvalidInput, "")); + } + // check extension + if !out_folder.exists() { + return Err(Error::new(ErrorKind::InvalidInput, "")); + }*/ + + // collect files + let mut included_extensions = ERedExtension::iter() + .map(|variant| variant.to_string()) + .collect::>(); + included_extensions.push(String::from("bin")); + + // get only resource files + let allfiles = WalkDir::new(in_folder) + .into_iter() + .filter_map(|e| e.ok()) + .map(|f| f.into_path()) + .filter(|p| { + if let Some(ext) = p.extension() { + if let Some(ext) = ext.to_str() { + return included_extensions.contains(&ext.to_owned()); + } + } + false + }) + .collect::>(); + + // sort by hash + let mut hashed_paths = allfiles + .iter() + .filter_map(|f| { + if let Ok(relative_path) = f.strip_prefix(in_folder) { + if let Some(path_str) = relative_path.to_str() { + let hash = fnv1a64_hash_string(&path_str.to_string()); + return Some((f.clone(), hash)); + } + } + None + }) + .collect::>(); + hashed_paths.sort_by_key(|k| k.1); + + let mut archive_writer = BufWriter::new(out_stream); + + // write temp header + let mut archive = Archive::default(); + let header = Header::default(); + header.serialize(&mut archive_writer)?; + archive_writer.write_all(&[0u8; 132])?; // some weird padding + + // write custom header + assert_eq!( + Header::HEADER_EXTENDED_SIZE, + archive_writer.stream_position()? + ); + let custom_paths = hashed_paths + .iter() + .filter(|(_p, k)| hash_map.contains_key(k)) + .filter_map(|(f, _h)| { + if let Ok(path) = f.strip_prefix(in_folder) { + return Some(path.to_string_lossy().to_string()); + } + None + }) + .collect::>(); + + let mut custom_data_length = 0; + if !custom_paths.is_empty() { + let wfooter = LxrsFooter { + files: custom_paths, + }; + wfooter.serialize(&mut archive_writer)?; + custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; + } + + // write files + let mut imports_hash_set: HashSet = HashSet::new(); + for (path, hash) in hashed_paths { + // read file + let mut file = File::open(&path)?; + let mut file_buffer = Vec::new(); + file.read_to_end(&mut file_buffer)?; + + let firstimportidx = imports_hash_set.len(); + let mut lastimportidx = imports_hash_set.len(); + let firstoffsetidx = archive.index.file_segments.len(); + let mut lastoffsetidx = 0; + let mut flags = 0; + + let mut file_cursor = Cursor::new(&file_buffer); + if let Ok(info) = read_cr2w_header(&mut file_cursor) { + // get main file + file_cursor.seek(SeekFrom::Start(0))?; + let size = info.header.objects_end; + let mut resource_buffer = vec![0; size as usize]; + file_cursor.read_exact(&mut resource_buffer[..])?; + // get archive offset before writing + let archive_offset = archive_writer.stream_position()?; + + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &resource_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + + // write compressed main file archive + // KARK header + archive_writer.write_u32::(kraken::MAGIC)?; //magic + archive_writer.write_u32::(size)?; //uncompressed buffer length + archive_writer.write_all(&compressed_buffer)?; + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset: archive_offset, + size, + z_size: zsize as u32, + }); + + // write buffers (bytes after the main file) + for buffer_info in info.buffers_table.iter() { + let mut buffer = vec![0; buffer_info.disk_size as usize]; + file_cursor.read_exact(&mut buffer[..])?; + + let bsize = buffer_info.mem_size; + let bzsize = buffer_info.disk_size; + let boffset = archive_writer.stream_position()?; + archive_writer.write_all(buffer.as_slice())?; + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset: boffset, + size: bsize, + z_size: bzsize, + }); + } + + //register imports + for import in info.imports.iter() { + // TODO fix flags + // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) + imports_hash_set.insert(import.depot_path.to_owned()); + } + + lastimportidx = imports_hash_set.len(); + lastoffsetidx = archive.index.file_segments.len(); + flags = if !info.buffers_table.is_empty() { + info.buffers_table.len() - 1 + } else { + 0 + }; + } else { + // write non-cr2w file + file_cursor.seek(SeekFrom::Start(0))?; + if let Some(os_ext) = path.extension() { + let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); + if get_aligned_file_extensions().contains(&ext) { + pad_until_page(&mut archive_writer)?; + } + + let offset = archive_writer.stream_position()?; + let size = file_buffer.len() as u32; + let mut final_zsize = file_buffer.len() as u32; + if get_uncompressed_file_extensions().contains(&ext) { + // direct copy + archive_writer.write_all(&file_buffer)?; + } else { + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &file_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + final_zsize = zsize as u32; + // write + archive_writer.write_all(&compressed_buffer)?; + } + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset, + size, + z_size: final_zsize, + }); + } + } + + // update archive metadata + let sha1_hash = sha1_hash_file(&file_buffer); + + let entry = FileEntry { + name_hash_64: hash, + timestamp: 0, // TODO proper timestamps + num_inline_buffer_segments: flags as u32, + segments_start: firstoffsetidx as u32, + segments_end: lastoffsetidx as u32, + resource_dependencies_start: firstimportidx as u32, + resource_dependencies_end: lastimportidx as u32, + sha1_hash, + }; + archive.index.file_entries.insert(hash, entry); + } + + // write footers + // padding + pad_until_page(&mut archive_writer)?; + + // write tables + let tableoffset = archive_writer.stream_position()?; + archive.index.serialize(&mut archive_writer)?; + let tablesize = archive_writer.stream_position()? - tableoffset; + + // padding + pad_until_page(&mut archive_writer)?; + let filesize = archive_writer.stream_position()?; + + // write the header again + archive.header.index_position = tableoffset; + archive.header.index_size = tablesize as u32; + archive.header.filesize = filesize; + archive_writer.seek(SeekFrom::Start(0))?; + archive.header.serialize(&mut archive_writer)?; + archive_writer.write_u32::(custom_data_length as u32)?; + + Ok(()) +} + +/// Decompresses and writes a kraken-compressed segment from an archive to a stream +/// +/// # Errors +/// +/// This function will return an error if . +fn decompress_segment( + archive_reader: &mut R, + segment: &FileSegment, + file_writer: &mut W, +) -> Result<()> { + archive_reader.seek(SeekFrom::Start(segment.offset))?; + + let magic = archive_reader.read_u32::()?; + if magic == kraken::MAGIC { + // read metadata + let mut size = segment.size; + let size_in_header = archive_reader.read_u32::()?; + if size_in_header != size { + size = size_in_header; + } + let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; + archive_reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); + + // write + file_writer.write_all(&output_buffer)?; + } else { + // incorrect data, fall back to direct copy + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + }; + + Ok(()) +} + +/// . +fn get_aligned_file_extensions() -> Vec { + let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; + files.into_iter().map(|f| f.to_owned()).collect::>() +} + +/// . +fn get_uncompressed_file_extensions() -> Vec { + let files = vec![ + ".bk2", + ".bnk", + ".opusinfo", + ".wem", + ".bin", + ".dat", + ".opuspak", + ]; + files.into_iter().map(|f| f.to_owned()).collect::>() +} + +fn pad_until_page(writer: &mut W) -> Result<()> { + let pos = writer.stream_position()?; + let modulo = pos / 4096; + let diff = ((modulo + 1) * 4096) - pos; + let padding = vec![0xD9; diff as usize]; + writer.write_all(padding.as_slice())?; + + Ok(()) +} + ///////////////////////////////////////////////////////////////////////////////////////// /// TESTS ///////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/archive_file.rs b/src/archive_file.rs deleted file mode 100644 index 2cf67d3..0000000 --- a/src/archive_file.rs +++ /dev/null @@ -1,595 +0,0 @@ -///////////////////////////////////////////////////////////////////////////////////////// -// ARCHIVE_FILE -// https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.zipfile?view=net-8.0#methods -// ZipFile -> namespace -// Provides static methods for creating, extracting, and opening zip archives. -// -// ZipArchive -> Archive -// Represents a package of compressed files in the zip archive format. -// -// ZipArchiveEntry -> ArchiveEntry -// Represents a compressed file within a zip archive. -///////////////////////////////////////////////////////////////////////////////////////// - -use std::{ - collections::{HashMap, HashSet}, - fs::{create_dir_all, File}, - io::{self, BufWriter, Read, Result, Seek, SeekFrom, Write}, - path::Path, -}; - -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use strum::IntoEnumIterator; -use walkdir::WalkDir; - -use crate::{ - archive::*, - cr2w::*, - fnv1a64_hash_string, get_red4_hashes, - kraken::{self, *}, - sha1_hash_file, ERedExtension, -}; - -// public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); - -/// Creates an archive in the specified stream that contains the files and directories from the specified directory. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn create_from_directory( - source_directory_name: &P, - destination: W, - hash_map: Option>, -) -> io::Result<()> -where - P: AsRef, - W: Write + Seek, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - write_archive(source_directory_name, destination, map) -} - -// public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); - -/// Creates an archive that contains the files and directories from the specified directory. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn create_from_directory_path

( - source_directory_name: &P, - destination: &P, - hash_map: Option>, -) -> io::Result<()> -where - P: AsRef, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - let fs: File = File::create(destination)?; - write_archive(source_directory_name, fs, map) -} - -// public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); - -/// Extracts all the files from the archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn extract_to_directory( - source: &mut R, - destination_directory_name: &P, - overwrite_files: bool, - hash_map: Option>, -) -> io::Result<()> -where - P: AsRef, - R: Read + Seek, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - extract_archive(source, destination_directory_name, overwrite_files, &map) -} - -// public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); - -/// Extracts all of the files in the specified archive to a directory on the file system. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn extract_to_directory_path

( - source_archive_file_name: &P, - destination_directory_name: &P, - overwrite_files: bool, - hash_map: Option>, -) -> io::Result<()> -where - P: AsRef, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - let archive_file = File::open(source_archive_file_name)?; - let mut archive_reader = io::BufReader::new(archive_file); - - extract_archive( - &mut archive_reader, - destination_directory_name, - overwrite_files, - &map, - ) -} - -pub enum ArchiveMode { - Create, - Read, - Update, -} - -/* -TODO We don't support different modes for now -needs a wrapper class for archives - - -// public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); - -/// Opens an archive at the specified path and in the specified mode. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn open

(archive_file_name: P, mode: ArchiveMode) -> io::Result -where - P: AsRef, -{ - todo!() -} - - */ - -// public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); - -/// Opens an archive for reading at the specified path. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn open_read

(archive_file_name: P) -> io::Result -where - P: AsRef, -{ - Archive::from_file(archive_file_name) -} - -///////////////////////////////////////////////////////////////////////////////////////// -/// Lib -///////////////////////////////////////////////////////////////////////////////////////// - -/// Extracts all files from an archive and writes them to a folder -/// -/// # Panics -/// -/// Panics if file path operations fail -/// -/// # Errors -/// -/// This function will return an error if any parsing fails -fn extract_archive( - archive_reader: &mut R, - out_dir: &P, - overwrite_files: bool, - hash_map: &HashMap, -) -> io::Result<()> -where - P: AsRef, - R: Read + Seek, -{ - // parse archive headers - let archive = Archive::from_reader(archive_reader)?; - - for (hash, file_entry) in archive.index.file_entries.iter() { - // get filename - let mut name_or_hash: String = format!("{}.bin", hash); - if let Some(name) = hash_map.get(hash) { - name_or_hash = name.to_owned(); - } - if let Some(name) = archive.file_names.get(hash) { - name_or_hash = name.to_owned(); - } - - // name or hash is a relative path - let outfile = out_dir.as_ref().join(name_or_hash); - create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; - - // extract to stream - let mut fs = if overwrite_files { - File::create(outfile)? - } else { - File::options() - .read(true) - .write(true) - .create_new(true) - .open(outfile)? - }; - - //let mut fs = File::create(outfile)?; - let mut file_writer = BufWriter::new(&mut fs); - // decompress main file - let start_index = file_entry.segments_start; - let next_index = file_entry.segments_end; - if let Some(segment) = archive.index.file_segments.get(start_index as usize) { - // read and decompress from main archive stream - - // kraken decompress - if segment.size == segment.z_size { - // just copy over - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } else { - decompress_segment(archive_reader, segment, &mut file_writer)?; - } - } - - // extract additional buffers - for i in start_index + 1..next_index { - if let Some(segment) = archive.index.file_segments.get(i as usize) { - // do not decompress with oodle - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } - } - } - - Ok(()) -} - -/// Packs redengine 4 resource file in a folder to an archive -/// -/// # Panics -/// -/// Panics if any path conversions fail -/// -/// # Errors -/// -/// This function will return an error if any parsing or IO fails -fn write_archive( - in_folder: &P, - out_stream: W, - hash_map: HashMap, -) -> io::Result<()> -where - P: AsRef, - W: Write + Seek, -{ - /*if !in_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); - } - - if !out_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); - } - // check extension - if !out_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); - }*/ - - // collect files - let mut included_extensions = ERedExtension::iter() - .map(|variant| variant.to_string()) - .collect::>(); - included_extensions.push(String::from("bin")); - - // get only resource files - let allfiles = WalkDir::new(in_folder) - .into_iter() - .filter_map(|e| e.ok()) - .map(|f| f.into_path()) - .filter(|p| { - if let Some(ext) = p.extension() { - if let Some(ext) = ext.to_str() { - return included_extensions.contains(&ext.to_owned()); - } - } - false - }) - .collect::>(); - - // sort by hash - let mut hashed_paths = allfiles - .iter() - .filter_map(|f| { - if let Ok(relative_path) = f.strip_prefix(in_folder) { - if let Some(path_str) = relative_path.to_str() { - let hash = fnv1a64_hash_string(&path_str.to_string()); - return Some((f.clone(), hash)); - } - } - None - }) - .collect::>(); - hashed_paths.sort_by_key(|k| k.1); - - let mut archive_writer = BufWriter::new(out_stream); - - // write temp header - let mut archive = Archive::default(); - let header = Header::default(); - header.serialize(&mut archive_writer)?; - archive_writer.write_all(&[0u8; 132])?; // some weird padding - - // write custom header - assert_eq!( - Header::HEADER_EXTENDED_SIZE, - archive_writer.stream_position()? - ); - let custom_paths = hashed_paths - .iter() - .filter(|(_p, k)| hash_map.contains_key(k)) - .filter_map(|(f, _h)| { - if let Ok(path) = f.strip_prefix(in_folder) { - return Some(path.to_string_lossy().to_string()); - } - None - }) - .collect::>(); - - let mut custom_data_length = 0; - if !custom_paths.is_empty() { - let wfooter = LxrsFooter { - files: custom_paths, - }; - wfooter.serialize(&mut archive_writer)?; - custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; - } - - // write files - let mut imports_hash_set: HashSet = HashSet::new(); - for (path, hash) in hashed_paths { - // read file - let mut file = File::open(&path)?; - let mut file_buffer = Vec::new(); - file.read_to_end(&mut file_buffer)?; - - let firstimportidx = imports_hash_set.len(); - let mut lastimportidx = imports_hash_set.len(); - let firstoffsetidx = archive.index.file_segments.len(); - let mut lastoffsetidx = 0; - let mut flags = 0; - - let mut file_cursor = io::Cursor::new(&file_buffer); - if let Ok(info) = read_cr2w_header(&mut file_cursor) { - // get main file - file_cursor.seek(SeekFrom::Start(0))?; - let size = info.header.objects_end; - let mut resource_buffer = vec![0; size as usize]; - file_cursor.read_exact(&mut resource_buffer[..])?; - // get archive offset before writing - let archive_offset = archive_writer.stream_position()?; - - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &resource_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - - // write compressed main file archive - // KARK header - archive_writer.write_u32::(kraken::MAGIC)?; //magic - archive_writer.write_u32::(size)?; //uncompressed buffer length - archive_writer.write_all(&compressed_buffer)?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: archive_offset, - size, - z_size: zsize as u32, - }); - - // write buffers (bytes after the main file) - for buffer_info in info.buffers_table.iter() { - let mut buffer = vec![0; buffer_info.disk_size as usize]; - file_cursor.read_exact(&mut buffer[..])?; - - let bsize = buffer_info.mem_size; - let bzsize = buffer_info.disk_size; - let boffset = archive_writer.stream_position()?; - archive_writer.write_all(buffer.as_slice())?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: boffset, - size: bsize, - z_size: bzsize, - }); - } - - //register imports - for import in info.imports.iter() { - // TODO fix flags - // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) - imports_hash_set.insert(import.depot_path.to_owned()); - } - - lastimportidx = imports_hash_set.len(); - lastoffsetidx = archive.index.file_segments.len(); - flags = if !info.buffers_table.is_empty() { - info.buffers_table.len() - 1 - } else { - 0 - }; - } else { - // write non-cr2w file - file_cursor.seek(SeekFrom::Start(0))?; - if let Some(os_ext) = path.extension() { - let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); - if get_aligned_file_extensions().contains(&ext) { - pad_until_page(&mut archive_writer)?; - } - - let offset = archive_writer.stream_position()?; - let size = file_buffer.len() as u32; - let mut final_zsize = file_buffer.len() as u32; - if get_uncompressed_file_extensions().contains(&ext) { - // direct copy - archive_writer.write_all(&file_buffer)?; - } else { - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &file_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - final_zsize = zsize as u32; - // write - archive_writer.write_all(&compressed_buffer)?; - } - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset, - size, - z_size: final_zsize, - }); - } - } - - // update archive metadata - let sha1_hash = sha1_hash_file(&file_buffer); - - let entry = FileEntry { - name_hash_64: hash, - timestamp: 0, // TODO proper timestamps - num_inline_buffer_segments: flags as u32, - segments_start: firstoffsetidx as u32, - segments_end: lastoffsetidx as u32, - resource_dependencies_start: firstimportidx as u32, - resource_dependencies_end: lastimportidx as u32, - sha1_hash, - }; - archive.index.file_entries.insert(hash, entry); - } - - // write footers - // padding - pad_until_page(&mut archive_writer)?; - - // write tables - let tableoffset = archive_writer.stream_position()?; - archive.index.serialize(&mut archive_writer)?; - let tablesize = archive_writer.stream_position()? - tableoffset; - - // padding - pad_until_page(&mut archive_writer)?; - let filesize = archive_writer.stream_position()?; - - // write the header again - archive.header.index_position = tableoffset; - archive.header.index_size = tablesize as u32; - archive.header.filesize = filesize; - archive_writer.seek(SeekFrom::Start(0))?; - archive.header.serialize(&mut archive_writer)?; - archive_writer.write_u32::(custom_data_length as u32)?; - - Ok(()) -} - -/// Decompresses and writes a kraken-compressed segment from an archive to a stream -/// -/// # Errors -/// -/// This function will return an error if . -fn decompress_segment( - archive_reader: &mut R, - segment: &FileSegment, - file_writer: &mut W, -) -> Result<()> { - archive_reader.seek(SeekFrom::Start(segment.offset))?; - - let magic = archive_reader.read_u32::()?; - if magic == kraken::MAGIC { - // read metadata - let mut size = segment.size; - let size_in_header = archive_reader.read_u32::()?; - if size_in_header != size { - size = size_in_header; - } - let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; - archive_reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); - - // write - file_writer.write_all(&output_buffer)?; - } else { - // incorrect data, fall back to direct copy - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - }; - - Ok(()) -} - -/// . -fn get_aligned_file_extensions() -> Vec { - let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; - files.into_iter().map(|f| f.to_owned()).collect::>() -} - -/// . -fn get_uncompressed_file_extensions() -> Vec { - let files = vec![ - ".bk2", - ".bnk", - ".opusinfo", - ".wem", - ".bin", - ".dat", - ".opuspak", - ]; - files.into_iter().map(|f| f.to_owned()).collect::>() -} - -fn pad_until_page(writer: &mut W) -> io::Result<()> { - let pos = writer.stream_position()?; - let modulo = pos / 4096; - let diff = ((modulo + 1) * 4096) - pos; - let padding = vec![0xD9; diff as usize]; - writer.write_all(padding.as_slice())?; - - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs index f19959b..d8ba151 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,22 +1,23 @@ #![warn(clippy::all, rust_2018_idioms)] -pub(crate) mod archive; mod cr2w; mod io; -mod kraken; -pub mod archive_file; +pub mod archive; +pub mod kraken; -use std::collections::HashMap; -use std::fs::{self}; -use std::hash::Hasher; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + hash::Hasher, + io::{BufRead, BufReader}, + path::Path, +}; use sha1::{Digest, Sha1}; use strum_macros::{Display, EnumIter}; ///////////////////////////////////////////////////////////////////////////////////////// -/// RED4 LIB +// RED4 LIB ///////////////////////////////////////////////////////////////////////////////////////// #[allow(non_camel_case_types)] @@ -165,33 +166,9 @@ enum ERedExtension { } #[warn(non_camel_case_types)] ///////////////////////////////////////////////////////////////////////////////////////// -/// HELPERS +// HELPERS ///////////////////////////////////////////////////////////////////////////////////////// -/// Get top-level files of a folder with given extension -pub fn get_files(folder_path: &Path, extension: &str) -> Vec { - let mut files = Vec::new(); - if !folder_path.exists() { - return files; - } - - if let Ok(entries) = fs::read_dir(folder_path) { - for entry in entries.flatten() { - if let Ok(file_type) = entry.file_type() { - if file_type.is_file() { - if let Some(ext) = entry.path().extension() { - if ext == extension { - files.push(entry.path()); - } - } - } - } - } - } - - files -} - /// Calculate FNV1a64 hash of a String pub fn fnv1a64_hash_string(str: &String) -> u64 { let mut hasher = fnv::FnvHasher::default(); @@ -219,8 +196,8 @@ pub fn get_red4_hashes() -> HashMap { let csv_data = include_bytes!("metadata-resources.csv"); let mut map: HashMap = HashMap::new(); - let reader = std::io::BufReader::new(&csv_data[..]); - for line in std::io::BufRead::lines(reader).flatten() { + let reader = BufReader::new(&csv_data[..]); + for line in BufRead::lines(reader).flatten() { let mut split = line.split(','); if let Some(name) = split.next() { if let Some(hash_str) = split.next() { diff --git a/tests/functional_tests.rs b/tests/functional_tests.rs index 6b61f7a..6b75544 100644 --- a/tests/functional_tests.rs +++ b/tests/functional_tests.rs @@ -6,11 +6,9 @@ mod tests { use std::fs::create_dir_all; use std::path::Path; - //use std::path::Path; use std::time::Instant; use std::{fs, path::PathBuf}; - use red4lib::archive_file::extract_to_directory_path; use red4lib::*; #[test] @@ -35,7 +33,8 @@ mod tests { assert!(fs::remove_dir_all(&dst_path).is_ok()); } - let result = extract_to_directory_path(&archive_path, &dst_path, true, Some(hashes)); + let result = + archive::extract_to_directory_path(&archive_path, &dst_path, true, Some(hashes)); assert!(result.is_ok()); // check @@ -109,7 +108,7 @@ mod tests { } create_dir_all(&dst_path).expect("Could not create folder"); - let result = archive_file::create_from_directory_path(&data_path, &dst_file, None); + let result = archive::create_from_directory_path(&data_path, &dst_file, None); assert!(result.is_ok()); // checks From ebb5a348ec67ab18959b71af3b3869ca758c2592 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Thu, 21 Dec 2023 07:38:18 +0100 Subject: [PATCH 05/14] Update archive.rs --- src/archive.rs | 1563 ++++++++++++++++++++++++------------------------ 1 file changed, 782 insertions(+), 781 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index bb518b1..5c60636 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,6 +1,7 @@ ///////////////////////////////////////////////////////////////////////////////////////// -/// ARCHIVE +// ARCHIVE ///////////////////////////////////////////////////////////////////////////////////////// + use std::cmp::Ordering; use std::{ collections::{HashMap, HashSet}, @@ -19,916 +20,916 @@ use crate::io::*; use crate::kraken::*; use crate::{cr2w::*, *}; -#[derive(Debug, Clone, Default)] -pub struct Archive { - pub header: Header, - pub index: Index, +///////////////////////////////////////////////////////////////////////////////////////// +// ARCHIVE_FILE +// https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.zipfile?view=net-8.0#methods +// ZipFile -> namespace +// Provides static methods for creating, extracting, and opening zip archives. +// +// ZipArchive -> Archive +// Represents a package of compressed files in the zip archive format. +// +// ZipArchiveEntry -> ArchiveEntry +// Represents a compressed file within a zip archive. +///////////////////////////////////////////////////////////////////////////////////////// - // custom - pub file_names: HashMap, +// public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); + +/// Creates an archive in the specified stream that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory( + source_directory_name: &P, + destination: W, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + W: Write + Seek, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + write_archive(source_directory_name, destination, map) } -impl Archive { - // Function to read a Header from a file - pub fn from_file

(file_path: P) -> Result - where - P: AsRef, - { - let mut file = File::open(file_path)?; - let mut buffer = Vec::with_capacity(mem::size_of::

()); - file.read_to_end(&mut buffer)?; +// public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); - // Ensure that the buffer has enough bytes to represent a Header - if buffer.len() < mem::size_of::
() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "File does not contain enough data to parse Header", - )); - } +/// Creates an archive that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory_path

( + source_directory_name: &P, + destination: &P, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; - let mut cursor = io::Cursor::new(&buffer); + let fs: File = File::create(destination)?; + write_archive(source_directory_name, fs, map) +} - Archive::from_reader(&mut cursor) - } +// public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); - pub fn from_reader(cursor: &mut R) -> Result - where - R: Read + Seek, - { - let header = Header::from_reader(cursor)?; +/// Extracts all the files from the archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory( + source: &mut R, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + R: Read + Seek, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; - // read custom data - let mut file_names: HashMap = HashMap::default(); - if let Ok(custom_data_length) = cursor.read_u32::() { - if custom_data_length > 0 { - cursor.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; - if let Ok(footer) = LxrsFooter::from_reader(cursor) { - // add files to hashmap - for f in footer.files { - let hash = fnv1a64_hash_string(&f); - file_names.insert(hash, f); - } - } - } - } + extract_archive(source, destination_directory_name, overwrite_files, &map) +} - // move to offset Header.IndexPosition - cursor.seek(io::SeekFrom::Start(header.index_position))?; - let index = Index::from_reader(cursor)?; +// public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); - Ok(Archive { - header, - index, - file_names, - }) - } +/// Extracts all of the files in the specified archive to a directory on the file system. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory_path

( + source_archive_file_name: &P, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; - // get filehashes - pub fn get_file_hashes(&self) -> Vec { - self.index - .file_entries - .iter() - .map(|f| f.1.name_hash_64) - .collect::>() - } -} + let archive_file = File::open(source_archive_file_name)?; + let mut archive_reader = BufReader::new(archive_file); -#[derive(Debug, Clone, Copy)] -pub struct Header { - pub magic: u32, - pub version: u32, - pub index_position: u64, - pub index_size: u32, - pub debug_position: u64, - pub debug_size: u32, - pub filesize: u64, + extract_archive( + &mut archive_reader, + destination_directory_name, + overwrite_files, + &map, + ) } -impl Header { - //static HEADER_MAGIC: u32 = 1380009042; - //static HEADER_SIZE: i32 = 40; - pub const HEADER_EXTENDED_SIZE: u64 = 0xAC; +pub enum ArchiveMode { + Create, + Read, + Update, } -impl Default for Header { - fn default() -> Self { - Self { - magic: 1380009042, - version: 12, - index_position: Default::default(), - index_size: Default::default(), - debug_position: Default::default(), - debug_size: Default::default(), - filesize: Default::default(), - } - } -} +/* +TODO We don't support different modes for now +needs a wrapper class for archives -impl FromReader for Header { - fn from_reader(reader: &mut R) -> Result { - Ok(Header { - magic: reader.read_u32::()?, - version: reader.read_u32::()?, - index_position: reader.read_u64::()?, - index_size: reader.read_u32::()?, - debug_position: reader.read_u64::()?, - debug_size: reader.read_u32::()?, - filesize: reader.read_u64::()?, - }) - } -} -impl Header { - pub fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.magic)?; - writer.write_u32::(self.version)?; - writer.write_u64::(self.index_position)?; - writer.write_u32::(self.index_size)?; - writer.write_u64::(self.debug_position)?; - writer.write_u32::(self.debug_size)?; - writer.write_u64::(self.filesize)?; - Ok(()) - } +// public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); + +/// Opens an archive at the specified path and in the specified mode. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open

(archive_file_name: P, mode: ArchiveMode) -> Result +where + P: AsRef, +{ + todo!() } -#[derive(Debug, Clone, Default)] -pub struct Index { - pub file_table_offset: u32, - pub file_table_size: u32, - pub crc: u64, - pub file_entry_count: u32, - pub file_segment_count: u32, - pub resource_dependency_count: u32, + */ - // not serialized - pub file_entries: HashMap, - pub file_segments: Vec, - pub dependencies: Vec, -} -impl Index { - pub fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.file_table_offset)?; - writer.write_u32::(self.file_table_size)?; - writer.write_u64::(self.crc)?; - writer.write_u32::(self.file_entry_count)?; - writer.write_u32::(self.file_segment_count)?; - writer.write_u32::(self.resource_dependency_count)?; +// public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); - Ok(()) - } +/// Opens an archive for reading at the specified path. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open_read

(archive_file_name: P) -> Result +where + P: AsRef, +{ + Archive::from_file(archive_file_name) } -impl FromReader for Index { - fn from_reader(cursor: &mut R) -> io::Result { - let mut index = Index { - file_table_offset: cursor.read_u32::()?, - file_table_size: cursor.read_u32::()?, - crc: cursor.read_u64::()?, - file_entry_count: cursor.read_u32::()?, - file_segment_count: cursor.read_u32::()?, - resource_dependency_count: cursor.read_u32::()?, - file_entries: HashMap::default(), - file_segments: vec![], - dependencies: vec![], - }; +/// Extracts all files from an archive and writes them to a folder +/// +/// # Panics +/// +/// Panics if file path operations fail +/// +/// # Errors +/// +/// This function will return an error if any parsing fails +fn extract_archive( + archive_reader: &mut R, + out_dir: &P, + overwrite_files: bool, + hash_map: &HashMap, +) -> Result<()> +where + P: AsRef, + R: Read + Seek, +{ + // parse archive headers + let archive = Archive::from_reader(archive_reader)?; - // read tables - for _i in 0..index.file_entry_count { - let entry = FileEntry::from_reader(cursor)?; - index.file_entries.insert(entry.name_hash_64, entry); + for (hash, file_entry) in archive.index.file_entries.iter() { + // get filename + let mut name_or_hash: String = format!("{}.bin", hash); + if let Some(name) = hash_map.get(hash) { + name_or_hash = name.to_owned(); } - - for _i in 0..index.file_segment_count { - index.file_segments.push(FileSegment::from_reader(cursor)?); + if let Some(name) = archive.file_names.get(hash) { + name_or_hash = name.to_owned(); } - for _i in 0..index.resource_dependency_count { - index.dependencies.push(Dependency::from_reader(cursor)?); - } + // name or hash is a relative path + let outfile = out_dir.as_ref().join(name_or_hash); + create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; - // ignore the rest of the archive + // extract to stream + let mut fs = if overwrite_files { + File::create(outfile)? + } else { + File::options() + .read(true) + .write(true) + .create_new(true) + .open(outfile)? + }; - Ok(index) - } -} + //let mut fs = File::create(outfile)?; + let mut file_writer = BufWriter::new(&mut fs); + // decompress main file + let start_index = file_entry.segments_start; + let next_index = file_entry.segments_end; + if let Some(segment) = archive.index.file_segments.get(start_index as usize) { + // read and decompress from main archive stream -#[derive(Debug, Clone, Copy)] -pub struct FileSegment { - pub offset: u64, - pub z_size: u32, - pub size: u32, -} + // kraken decompress + if segment.size == segment.z_size { + // just copy over + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } else { + decompress_segment(archive_reader, segment, &mut file_writer)?; + } + } -impl FromReader for FileSegment { - fn from_reader(reader: &mut R) -> io::Result { - Ok(FileSegment { - offset: reader.read_u64::()?, - z_size: reader.read_u32::()?, - size: reader.read_u32::()?, - }) + // extract additional buffers + for i in start_index + 1..next_index { + if let Some(segment) = archive.index.file_segments.get(i as usize) { + // do not decompress with oodle + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } + } } -} -#[derive(Debug, Clone, Copy)] -pub struct FileEntry { - pub name_hash_64: u64, - pub timestamp: u64, //SystemTime, - pub num_inline_buffer_segments: u32, - pub segments_start: u32, - pub segments_end: u32, - pub resource_dependencies_start: u32, - pub resource_dependencies_end: u32, - pub sha1_hash: [u8; 20], + Ok(()) } -impl FromReader for FileEntry { - fn from_reader(reader: &mut R) -> io::Result { - let mut entry = FileEntry { - name_hash_64: reader.read_u64::()?, - timestamp: reader.read_u64::()?, - num_inline_buffer_segments: reader.read_u32::()?, - segments_start: reader.read_u32::()?, - segments_end: reader.read_u32::()?, - resource_dependencies_start: reader.read_u32::()?, - resource_dependencies_end: reader.read_u32::()?, - sha1_hash: [0; 20], - }; - - reader.read_exact(&mut entry.sha1_hash[..])?; - - Ok(entry) +/// Packs redengine 4 resource file in a folder to an archive +/// +/// # Panics +/// +/// Panics if any path conversions fail +/// +/// # Errors +/// +/// This function will return an error if any parsing or IO fails +fn write_archive(in_folder: &P, out_stream: W, hash_map: HashMap) -> Result<()> +where + P: AsRef, + W: Write + Seek, +{ + /*if !in_folder.exists() { + return Err(Error::new(ErrorKind::InvalidInput, "")); } -} - -#[derive(Debug, Clone, Copy)] -pub struct Dependency { - pub hash: u64, -} -impl FromReader for Dependency { - fn from_reader(reader: &mut R) -> io::Result { - Ok(Dependency { - hash: reader.read_u64::()?, - }) + if !out_folder.exists() { + return Err(Error::new(ErrorKind::InvalidInput, "")); } -} - -#[derive(Debug, Clone)] -pub struct LxrsFooter { - pub files: Vec, -} + // check extension + if !out_folder.exists() { + return Err(Error::new(ErrorKind::InvalidInput, "")); + }*/ -impl LxrsFooter { - //const MINLEN: u32 = 20; - const MAGIC: u32 = 0x4C585253; - const VERSION: u32 = 1; + // collect files + let mut included_extensions = ERedExtension::iter() + .map(|variant| variant.to_string()) + .collect::>(); + included_extensions.push(String::from("bin")); - pub fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.files.len() as u32)?; - writer.write_u32::(LxrsFooter::VERSION)?; + // get only resource files + let allfiles = WalkDir::new(in_folder) + .into_iter() + .filter_map(|e| e.ok()) + .map(|f| f.into_path()) + .filter(|p| { + if let Some(ext) = p.extension() { + if let Some(ext) = ext.to_str() { + return included_extensions.contains(&ext.to_owned()); + } + } + false + }) + .collect::>(); - // write strings to buffer - let mut buffer: Vec = Vec::new(); - for f in &self.files { - write_null_terminated_string(&mut buffer, f.to_owned())?; - } + // sort by hash + let mut hashed_paths = allfiles + .iter() + .filter_map(|f| { + if let Ok(relative_path) = f.strip_prefix(in_folder) { + if let Some(path_str) = relative_path.to_str() { + let hash = fnv1a64_hash_string(&path_str.to_string()); + return Some((f.clone(), hash)); + } + } + None + }) + .collect::>(); + hashed_paths.sort_by_key(|k| k.1); - // compress - let size = buffer.len(); - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress(&buffer, &mut compressed_buffer, CompressionLevel::Normal); - assert!((zsize as u32) <= size as u32); - compressed_buffer.resize(zsize as usize, 0); + let mut archive_writer = BufWriter::new(out_stream); - // write to writer - writer.write_all(&compressed_buffer)?; + // write temp header + let mut archive = Archive::default(); + let header = Header::default(); + header.serialize(&mut archive_writer)?; + archive_writer.write_all(&[0u8; 132])?; // some weird padding - Ok(()) + // write custom header + assert_eq!( + Header::HEADER_EXTENDED_SIZE, + archive_writer.stream_position()? + ); + let custom_paths = hashed_paths + .iter() + .filter(|(_p, k)| hash_map.contains_key(k)) + .filter_map(|(f, _h)| { + if let Ok(path) = f.strip_prefix(in_folder) { + return Some(path.to_string_lossy().to_string()); + } + None + }) + .collect::>(); + + let mut custom_data_length = 0; + if !custom_paths.is_empty() { + let wfooter = LxrsFooter { + files: custom_paths, + }; + wfooter.serialize(&mut archive_writer)?; + custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; } -} -impl FromReader for LxrsFooter { - fn from_reader(reader: &mut R) -> io::Result { - let magic = reader.read_u32::()?; - if magic != LxrsFooter::MAGIC { - return Err(io::Error::new(io::ErrorKind::Other, "invalid magic")); - } - let _version = reader.read_u32::()?; - let size = reader.read_u32::()?; - let zsize = reader.read_u32::()?; - let count = reader.read_i32::()?; - let mut files: Vec = vec![]; - match size.cmp(&zsize) { - Ordering::Greater => { - // buffer is compressed - let mut compressed_buffer = vec![0; zsize as usize]; - reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); + // write files + let mut imports_hash_set: HashSet = HashSet::new(); + for (path, hash) in hashed_paths { + // read file + let mut file = File::open(&path)?; + let mut file_buffer = Vec::new(); + file.read_to_end(&mut file_buffer)?; - // read from buffer - let mut inner_cursor = io::Cursor::new(&output_buffer); - for _i in 0..count { - // read NullTerminatedString - if let Ok(string) = read_null_terminated_string(&mut inner_cursor) { - files.push(string); - } - } + let firstimportidx = imports_hash_set.len(); + let mut lastimportidx = imports_hash_set.len(); + let firstoffsetidx = archive.index.file_segments.len(); + let mut lastoffsetidx = 0; + let mut flags = 0; + + let mut file_cursor = Cursor::new(&file_buffer); + if let Ok(info) = read_cr2w_header(&mut file_cursor) { + // get main file + file_cursor.seek(SeekFrom::Start(0))?; + let size = info.header.objects_end; + let mut resource_buffer = vec![0; size as usize]; + file_cursor.read_exact(&mut resource_buffer[..])?; + // get archive offset before writing + let archive_offset = archive_writer.stream_position()?; + + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &resource_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + + // write compressed main file archive + // KARK header + archive_writer.write_u32::(kraken::MAGIC)?; //magic + archive_writer.write_u32::(size)?; //uncompressed buffer length + archive_writer.write_all(&compressed_buffer)?; + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset: archive_offset, + size, + z_size: zsize as u32, + }); + + // write buffers (bytes after the main file) + for buffer_info in info.buffers_table.iter() { + let mut buffer = vec![0; buffer_info.disk_size as usize]; + file_cursor.read_exact(&mut buffer[..])?; + + let bsize = buffer_info.mem_size; + let bzsize = buffer_info.disk_size; + let boffset = archive_writer.stream_position()?; + archive_writer.write_all(buffer.as_slice())?; + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset: boffset, + size: bsize, + z_size: bzsize, + }); } - Ordering::Less => { - // error - return Err(io::Error::new(io::ErrorKind::Other, "invalid buffer")); + + //register imports + for import in info.imports.iter() { + // TODO fix flags + // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) + imports_hash_set.insert(import.depot_path.to_owned()); } - Ordering::Equal => { - // no compression - for _i in 0..count { - // read NullTerminatedString - if let Ok(string) = read_null_terminated_string(reader) { - files.push(string); - } + + lastimportidx = imports_hash_set.len(); + lastoffsetidx = archive.index.file_segments.len(); + flags = if !info.buffers_table.is_empty() { + info.buffers_table.len() - 1 + } else { + 0 + }; + } else { + // write non-cr2w file + file_cursor.seek(SeekFrom::Start(0))?; + if let Some(os_ext) = path.extension() { + let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); + if get_aligned_file_extensions().contains(&ext) { + pad_until_page(&mut archive_writer)?; + } + + let offset = archive_writer.stream_position()?; + let size = file_buffer.len() as u32; + let mut final_zsize = file_buffer.len() as u32; + if get_uncompressed_file_extensions().contains(&ext) { + // direct copy + archive_writer.write_all(&file_buffer)?; + } else { + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &file_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + final_zsize = zsize as u32; + // write + archive_writer.write_all(&compressed_buffer)?; } + + // add metadata to archive + archive.index.file_segments.push(FileSegment { + offset, + size, + z_size: final_zsize, + }); } } - let footer = LxrsFooter { files }; + // update archive metadata + let sha1_hash = sha1_hash_file(&file_buffer); - Ok(footer) + let entry = FileEntry { + name_hash_64: hash, + timestamp: 0, // TODO proper timestamps + num_inline_buffer_segments: flags as u32, + segments_start: firstoffsetidx as u32, + segments_end: lastoffsetidx as u32, + resource_dependencies_start: firstimportidx as u32, + resource_dependencies_end: lastimportidx as u32, + sha1_hash, + }; + archive.index.file_entries.insert(hash, entry); } -} -///////////////////////////////////////////////////////////////////////////////////////// -// ARCHIVE_FILE -// https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.zipfile?view=net-8.0#methods -// ZipFile -> namespace -// Provides static methods for creating, extracting, and opening zip archives. -// -// ZipArchive -> Archive -// Represents a package of compressed files in the zip archive format. -// -// ZipArchiveEntry -> ArchiveEntry -// Represents a compressed file within a zip archive. -///////////////////////////////////////////////////////////////////////////////////////// + // write footers + // padding + pad_until_page(&mut archive_writer)?; -// public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); + // write tables + let tableoffset = archive_writer.stream_position()?; + archive.index.serialize(&mut archive_writer)?; + let tablesize = archive_writer.stream_position()? - tableoffset; -/// Creates an archive in the specified stream that contains the files and directories from the specified directory. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn create_from_directory( - source_directory_name: &P, - destination: W, - hash_map: Option>, -) -> Result<()> -where - P: AsRef, - W: Write + Seek, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; + // padding + pad_until_page(&mut archive_writer)?; + let filesize = archive_writer.stream_position()?; - write_archive(source_directory_name, destination, map) -} + // write the header again + archive.header.index_position = tableoffset; + archive.header.index_size = tablesize as u32; + archive.header.filesize = filesize; + archive_writer.seek(SeekFrom::Start(0))?; + archive.header.serialize(&mut archive_writer)?; + archive_writer.write_u32::(custom_data_length as u32)?; -// public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); + Ok(()) +} -/// Creates an archive that contains the files and directories from the specified directory. +/// Decompresses and writes a kraken-compressed segment from an archive to a stream /// /// # Errors /// -/// This function will return an error if any io fails. -pub fn create_from_directory_path

( - source_directory_name: &P, - destination: &P, - hash_map: Option>, -) -> Result<()> -where - P: AsRef, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - let fs: File = File::create(destination)?; - write_archive(source_directory_name, fs, map) -} +/// This function will return an error if . +fn decompress_segment( + archive_reader: &mut R, + segment: &FileSegment, + file_writer: &mut W, +) -> Result<()> { + archive_reader.seek(SeekFrom::Start(segment.offset))?; -// public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); + let magic = archive_reader.read_u32::()?; + if magic == kraken::MAGIC { + // read metadata + let mut size = segment.size; + let size_in_header = archive_reader.read_u32::()?; + if size_in_header != size { + size = size_in_header; + } + let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; + archive_reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); -/// Extracts all the files from the archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn extract_to_directory( - source: &mut R, - destination_directory_name: &P, - overwrite_files: bool, - hash_map: Option>, -) -> Result<()> -where - P: AsRef, - R: Read + Seek, -{ - let map = if let Some(hash_map) = hash_map { - hash_map + // write + file_writer.write_all(&output_buffer)?; } else { - get_red4_hashes() + // incorrect data, fall back to direct copy + archive_reader.seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; }; - extract_archive(source, destination_directory_name, overwrite_files, &map) + Ok(()) } -// public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); - -/// Extracts all of the files in the specified archive to a directory on the file system. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn extract_to_directory_path

( - source_archive_file_name: &P, - destination_directory_name: &P, - overwrite_files: bool, - hash_map: Option>, -) -> Result<()> -where - P: AsRef, -{ - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - let archive_file = File::open(source_archive_file_name)?; - let mut archive_reader = BufReader::new(archive_file); - - extract_archive( - &mut archive_reader, - destination_directory_name, - overwrite_files, - &map, - ) +/// . +fn get_aligned_file_extensions() -> Vec { + let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; + files.into_iter().map(|f| f.to_owned()).collect::>() } -pub enum ArchiveMode { - Create, - Read, - Update, +/// . +fn get_uncompressed_file_extensions() -> Vec { + let files = vec![ + ".bk2", + ".bnk", + ".opusinfo", + ".wem", + ".bin", + ".dat", + ".opuspak", + ]; + files.into_iter().map(|f| f.to_owned()).collect::>() } -/* -TODO We don't support different modes for now -needs a wrapper class for archives - - -// public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); +fn pad_until_page(writer: &mut W) -> Result<()> { + let pos = writer.stream_position()?; + let modulo = pos / 4096; + let diff = ((modulo + 1) * 4096) - pos; + let padding = vec![0xD9; diff as usize]; + writer.write_all(padding.as_slice())?; -/// Opens an archive at the specified path and in the specified mode. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn open

(archive_file_name: P, mode: ArchiveMode) -> Result -where - P: AsRef, -{ - todo!() + Ok(()) } - */ +///////////////////////////////////////////////////////////////////////////////////////// +// INTERNAL +///////////////////////////////////////////////////////////////////////////////////////// -// public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); +#[derive(Debug, Clone, Default)] +pub struct Archive { + pub header: Header, + pub index: Index, -/// Opens an archive for reading at the specified path. -/// -/// # Errors -/// -/// This function will return an error if any io fails. -pub fn open_read

(archive_file_name: P) -> Result -where - P: AsRef, -{ - Archive::from_file(archive_file_name) + // custom + pub file_names: HashMap, } -///////////////////////////////////////////////////////////////////////////////////////// -/// Lib -///////////////////////////////////////////////////////////////////////////////////////// - -/// Extracts all files from an archive and writes them to a folder -/// -/// # Panics -/// -/// Panics if file path operations fail -/// -/// # Errors -/// -/// This function will return an error if any parsing fails -fn extract_archive( - archive_reader: &mut R, - out_dir: &P, - overwrite_files: bool, - hash_map: &HashMap, -) -> Result<()> -where - P: AsRef, - R: Read + Seek, -{ - // parse archive headers - let archive = Archive::from_reader(archive_reader)?; +impl Archive { + // Function to read a Header from a file + pub fn from_file

(file_path: P) -> Result + where + P: AsRef, + { + let mut file = File::open(file_path)?; + let mut buffer = Vec::with_capacity(mem::size_of::

()); + file.read_to_end(&mut buffer)?; - for (hash, file_entry) in archive.index.file_entries.iter() { - // get filename - let mut name_or_hash: String = format!("{}.bin", hash); - if let Some(name) = hash_map.get(hash) { - name_or_hash = name.to_owned(); - } - if let Some(name) = archive.file_names.get(hash) { - name_or_hash = name.to_owned(); + // Ensure that the buffer has enough bytes to represent a Header + if buffer.len() < mem::size_of::
() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "File does not contain enough data to parse Header", + )); } - // name or hash is a relative path - let outfile = out_dir.as_ref().join(name_or_hash); - create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; + let mut cursor = io::Cursor::new(&buffer); - // extract to stream - let mut fs = if overwrite_files { - File::create(outfile)? - } else { - File::options() - .read(true) - .write(true) - .create_new(true) - .open(outfile)? - }; + Archive::from_reader(&mut cursor) + } - //let mut fs = File::create(outfile)?; - let mut file_writer = BufWriter::new(&mut fs); - // decompress main file - let start_index = file_entry.segments_start; - let next_index = file_entry.segments_end; - if let Some(segment) = archive.index.file_segments.get(start_index as usize) { - // read and decompress from main archive stream + pub fn from_reader(cursor: &mut R) -> Result + where + R: Read + Seek, + { + let header = Header::from_reader(cursor)?; - // kraken decompress - if segment.size == segment.z_size { - // just copy over - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } else { - decompress_segment(archive_reader, segment, &mut file_writer)?; + // read custom data + let mut file_names: HashMap = HashMap::default(); + if let Ok(custom_data_length) = cursor.read_u32::() { + if custom_data_length > 0 { + cursor.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; + if let Ok(footer) = LxrsFooter::from_reader(cursor) { + // add files to hashmap + for f in footer.files { + let hash = fnv1a64_hash_string(&f); + file_names.insert(hash, f); + } + } } } - // extract additional buffers - for i in start_index + 1..next_index { - if let Some(segment) = archive.index.file_segments.get(i as usize) { - // do not decompress with oodle - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } - } - } - - Ok(()) -} + // move to offset Header.IndexPosition + cursor.seek(io::SeekFrom::Start(header.index_position))?; + let index = Index::from_reader(cursor)?; -/// Packs redengine 4 resource file in a folder to an archive -/// -/// # Panics -/// -/// Panics if any path conversions fail -/// -/// # Errors -/// -/// This function will return an error if any parsing or IO fails -fn write_archive(in_folder: &P, out_stream: W, hash_map: HashMap) -> Result<()> -where - P: AsRef, - W: Write + Seek, -{ - /*if !in_folder.exists() { - return Err(Error::new(ErrorKind::InvalidInput, "")); + Ok(Archive { + header, + index, + file_names, + }) } - if !out_folder.exists() { - return Err(Error::new(ErrorKind::InvalidInput, "")); + // get filehashes + pub fn get_file_hashes(&self) -> Vec { + self.index + .file_entries + .iter() + .map(|f| f.1.name_hash_64) + .collect::>() } - // check extension - if !out_folder.exists() { - return Err(Error::new(ErrorKind::InvalidInput, "")); - }*/ - - // collect files - let mut included_extensions = ERedExtension::iter() - .map(|variant| variant.to_string()) - .collect::>(); - included_extensions.push(String::from("bin")); - - // get only resource files - let allfiles = WalkDir::new(in_folder) - .into_iter() - .filter_map(|e| e.ok()) - .map(|f| f.into_path()) - .filter(|p| { - if let Some(ext) = p.extension() { - if let Some(ext) = ext.to_str() { - return included_extensions.contains(&ext.to_owned()); - } - } - false - }) - .collect::>(); +} - // sort by hash - let mut hashed_paths = allfiles - .iter() - .filter_map(|f| { - if let Ok(relative_path) = f.strip_prefix(in_folder) { - if let Some(path_str) = relative_path.to_str() { - let hash = fnv1a64_hash_string(&path_str.to_string()); - return Some((f.clone(), hash)); - } - } - None - }) - .collect::>(); - hashed_paths.sort_by_key(|k| k.1); +#[derive(Debug, Clone, Copy)] +pub struct Header { + pub magic: u32, + pub version: u32, + pub index_position: u64, + pub index_size: u32, + pub debug_position: u64, + pub debug_size: u32, + pub filesize: u64, +} - let mut archive_writer = BufWriter::new(out_stream); +impl Header { + //static HEADER_MAGIC: u32 = 1380009042; + //static HEADER_SIZE: i32 = 40; + pub const HEADER_EXTENDED_SIZE: u64 = 0xAC; +} - // write temp header - let mut archive = Archive::default(); - let header = Header::default(); - header.serialize(&mut archive_writer)?; - archive_writer.write_all(&[0u8; 132])?; // some weird padding +impl Default for Header { + fn default() -> Self { + Self { + magic: 1380009042, + version: 12, + index_position: Default::default(), + index_size: Default::default(), + debug_position: Default::default(), + debug_size: Default::default(), + filesize: Default::default(), + } + } +} - // write custom header - assert_eq!( - Header::HEADER_EXTENDED_SIZE, - archive_writer.stream_position()? - ); - let custom_paths = hashed_paths - .iter() - .filter(|(_p, k)| hash_map.contains_key(k)) - .filter_map(|(f, _h)| { - if let Ok(path) = f.strip_prefix(in_folder) { - return Some(path.to_string_lossy().to_string()); - } - None +impl FromReader for Header { + fn from_reader(reader: &mut R) -> Result { + Ok(Header { + magic: reader.read_u32::()?, + version: reader.read_u32::()?, + index_position: reader.read_u64::()?, + index_size: reader.read_u32::()?, + debug_position: reader.read_u64::()?, + debug_size: reader.read_u32::()?, + filesize: reader.read_u64::()?, }) - .collect::>(); - - let mut custom_data_length = 0; - if !custom_paths.is_empty() { - let wfooter = LxrsFooter { - files: custom_paths, - }; - wfooter.serialize(&mut archive_writer)?; - custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; } +} +impl Header { + pub fn serialize(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.magic)?; + writer.write_u32::(self.version)?; + writer.write_u64::(self.index_position)?; + writer.write_u32::(self.index_size)?; + writer.write_u64::(self.debug_position)?; + writer.write_u32::(self.debug_size)?; + writer.write_u64::(self.filesize)?; - // write files - let mut imports_hash_set: HashSet = HashSet::new(); - for (path, hash) in hashed_paths { - // read file - let mut file = File::open(&path)?; - let mut file_buffer = Vec::new(); - file.read_to_end(&mut file_buffer)?; - - let firstimportidx = imports_hash_set.len(); - let mut lastimportidx = imports_hash_set.len(); - let firstoffsetidx = archive.index.file_segments.len(); - let mut lastoffsetidx = 0; - let mut flags = 0; - - let mut file_cursor = Cursor::new(&file_buffer); - if let Ok(info) = read_cr2w_header(&mut file_cursor) { - // get main file - file_cursor.seek(SeekFrom::Start(0))?; - let size = info.header.objects_end; - let mut resource_buffer = vec![0; size as usize]; - file_cursor.read_exact(&mut resource_buffer[..])?; - // get archive offset before writing - let archive_offset = archive_writer.stream_position()?; - - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &resource_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - - // write compressed main file archive - // KARK header - archive_writer.write_u32::(kraken::MAGIC)?; //magic - archive_writer.write_u32::(size)?; //uncompressed buffer length - archive_writer.write_all(&compressed_buffer)?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: archive_offset, - size, - z_size: zsize as u32, - }); + Ok(()) + } +} - // write buffers (bytes after the main file) - for buffer_info in info.buffers_table.iter() { - let mut buffer = vec![0; buffer_info.disk_size as usize]; - file_cursor.read_exact(&mut buffer[..])?; +#[derive(Debug, Clone, Default)] +pub struct Index { + pub file_table_offset: u32, + pub file_table_size: u32, + pub crc: u64, + pub file_entry_count: u32, + pub file_segment_count: u32, + pub resource_dependency_count: u32, - let bsize = buffer_info.mem_size; - let bzsize = buffer_info.disk_size; - let boffset = archive_writer.stream_position()?; - archive_writer.write_all(buffer.as_slice())?; + // not serialized + pub file_entries: HashMap, + pub file_segments: Vec, + pub dependencies: Vec, +} +impl Index { + pub fn serialize(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.file_table_offset)?; + writer.write_u32::(self.file_table_size)?; + writer.write_u64::(self.crc)?; + writer.write_u32::(self.file_entry_count)?; + writer.write_u32::(self.file_segment_count)?; + writer.write_u32::(self.resource_dependency_count)?; - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: boffset, - size: bsize, - z_size: bzsize, - }); - } + Ok(()) + } +} +impl FromReader for Index { + fn from_reader(cursor: &mut R) -> io::Result { + let mut index = Index { + file_table_offset: cursor.read_u32::()?, + file_table_size: cursor.read_u32::()?, + crc: cursor.read_u64::()?, + file_entry_count: cursor.read_u32::()?, + file_segment_count: cursor.read_u32::()?, + resource_dependency_count: cursor.read_u32::()?, - //register imports - for import in info.imports.iter() { - // TODO fix flags - // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) - imports_hash_set.insert(import.depot_path.to_owned()); - } + file_entries: HashMap::default(), + file_segments: vec![], + dependencies: vec![], + }; - lastimportidx = imports_hash_set.len(); - lastoffsetidx = archive.index.file_segments.len(); - flags = if !info.buffers_table.is_empty() { - info.buffers_table.len() - 1 - } else { - 0 - }; - } else { - // write non-cr2w file - file_cursor.seek(SeekFrom::Start(0))?; - if let Some(os_ext) = path.extension() { - let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); - if get_aligned_file_extensions().contains(&ext) { - pad_until_page(&mut archive_writer)?; - } + // read tables + for _i in 0..index.file_entry_count { + let entry = FileEntry::from_reader(cursor)?; + index.file_entries.insert(entry.name_hash_64, entry); + } - let offset = archive_writer.stream_position()?; - let size = file_buffer.len() as u32; - let mut final_zsize = file_buffer.len() as u32; - if get_uncompressed_file_extensions().contains(&ext) { - // direct copy - archive_writer.write_all(&file_buffer)?; - } else { - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &file_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - final_zsize = zsize as u32; - // write - archive_writer.write_all(&compressed_buffer)?; - } + for _i in 0..index.file_segment_count { + index.file_segments.push(FileSegment::from_reader(cursor)?); + } - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset, - size, - z_size: final_zsize, - }); - } + for _i in 0..index.resource_dependency_count { + index.dependencies.push(Dependency::from_reader(cursor)?); } - // update archive metadata - let sha1_hash = sha1_hash_file(&file_buffer); + // ignore the rest of the archive - let entry = FileEntry { - name_hash_64: hash, - timestamp: 0, // TODO proper timestamps - num_inline_buffer_segments: flags as u32, - segments_start: firstoffsetidx as u32, - segments_end: lastoffsetidx as u32, - resource_dependencies_start: firstimportidx as u32, - resource_dependencies_end: lastimportidx as u32, - sha1_hash, - }; - archive.index.file_entries.insert(hash, entry); + Ok(index) } +} - // write footers - // padding - pad_until_page(&mut archive_writer)?; +#[derive(Debug, Clone, Copy)] +pub struct FileSegment { + pub offset: u64, + pub z_size: u32, + pub size: u32, +} - // write tables - let tableoffset = archive_writer.stream_position()?; - archive.index.serialize(&mut archive_writer)?; - let tablesize = archive_writer.stream_position()? - tableoffset; +impl FromReader for FileSegment { + fn from_reader(reader: &mut R) -> io::Result { + Ok(FileSegment { + offset: reader.read_u64::()?, + z_size: reader.read_u32::()?, + size: reader.read_u32::()?, + }) + } +} - // padding - pad_until_page(&mut archive_writer)?; - let filesize = archive_writer.stream_position()?; +#[derive(Debug, Clone, Copy)] +pub struct FileEntry { + pub name_hash_64: u64, + pub timestamp: u64, //SystemTime, + pub num_inline_buffer_segments: u32, + pub segments_start: u32, + pub segments_end: u32, + pub resource_dependencies_start: u32, + pub resource_dependencies_end: u32, + pub sha1_hash: [u8; 20], +} - // write the header again - archive.header.index_position = tableoffset; - archive.header.index_size = tablesize as u32; - archive.header.filesize = filesize; - archive_writer.seek(SeekFrom::Start(0))?; - archive.header.serialize(&mut archive_writer)?; - archive_writer.write_u32::(custom_data_length as u32)?; +impl FromReader for FileEntry { + fn from_reader(reader: &mut R) -> io::Result { + let mut entry = FileEntry { + name_hash_64: reader.read_u64::()?, + timestamp: reader.read_u64::()?, + num_inline_buffer_segments: reader.read_u32::()?, + segments_start: reader.read_u32::()?, + segments_end: reader.read_u32::()?, + resource_dependencies_start: reader.read_u32::()?, + resource_dependencies_end: reader.read_u32::()?, + sha1_hash: [0; 20], + }; - Ok(()) + reader.read_exact(&mut entry.sha1_hash[..])?; + + Ok(entry) + } } -/// Decompresses and writes a kraken-compressed segment from an archive to a stream -/// -/// # Errors -/// -/// This function will return an error if . -fn decompress_segment( - archive_reader: &mut R, - segment: &FileSegment, - file_writer: &mut W, -) -> Result<()> { - archive_reader.seek(SeekFrom::Start(segment.offset))?; +#[derive(Debug, Clone, Copy)] +pub struct Dependency { + pub hash: u64, +} - let magic = archive_reader.read_u32::()?; - if magic == kraken::MAGIC { - // read metadata - let mut size = segment.size; - let size_in_header = archive_reader.read_u32::()?; - if size_in_header != size { - size = size_in_header; +impl FromReader for Dependency { + fn from_reader(reader: &mut R) -> io::Result { + Ok(Dependency { + hash: reader.read_u64::()?, + }) + } +} + +#[derive(Debug, Clone)] +pub struct LxrsFooter { + pub files: Vec, +} + +impl LxrsFooter { + //const MINLEN: u32 = 20; + const MAGIC: u32 = 0x4C585253; + const VERSION: u32 = 1; + + pub fn serialize(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.files.len() as u32)?; + writer.write_u32::(LxrsFooter::VERSION)?; + + // write strings to buffer + let mut buffer: Vec = Vec::new(); + for f in &self.files { + write_null_terminated_string(&mut buffer, f.to_owned())?; } - let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; - archive_reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); - // write - file_writer.write_all(&output_buffer)?; - } else { - // incorrect data, fall back to direct copy - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - }; + // compress + let size = buffer.len(); + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress(&buffer, &mut compressed_buffer, CompressionLevel::Normal); + assert!((zsize as u32) <= size as u32); + compressed_buffer.resize(zsize as usize, 0); - Ok(()) -} + // write to writer + writer.write_all(&compressed_buffer)?; -/// . -fn get_aligned_file_extensions() -> Vec { - let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; - files.into_iter().map(|f| f.to_owned()).collect::>() + Ok(()) + } } +impl FromReader for LxrsFooter { + fn from_reader(reader: &mut R) -> io::Result { + let magic = reader.read_u32::()?; + if magic != LxrsFooter::MAGIC { + return Err(io::Error::new(io::ErrorKind::Other, "invalid magic")); + } + let _version = reader.read_u32::()?; + let size = reader.read_u32::()?; + let zsize = reader.read_u32::()?; + let count = reader.read_i32::()?; -/// . -fn get_uncompressed_file_extensions() -> Vec { - let files = vec![ - ".bk2", - ".bnk", - ".opusinfo", - ".wem", - ".bin", - ".dat", - ".opuspak", - ]; - files.into_iter().map(|f| f.to_owned()).collect::>() -} + let mut files: Vec = vec![]; + match size.cmp(&zsize) { + Ordering::Greater => { + // buffer is compressed + let mut compressed_buffer = vec![0; zsize as usize]; + reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); -fn pad_until_page(writer: &mut W) -> Result<()> { - let pos = writer.stream_position()?; - let modulo = pos / 4096; - let diff = ((modulo + 1) * 4096) - pos; - let padding = vec![0xD9; diff as usize]; - writer.write_all(padding.as_slice())?; + // read from buffer + let mut inner_cursor = io::Cursor::new(&output_buffer); + for _i in 0..count { + // read NullTerminatedString + if let Ok(string) = read_null_terminated_string(&mut inner_cursor) { + files.push(string); + } + } + } + Ordering::Less => { + // error + return Err(io::Error::new(io::ErrorKind::Other, "invalid buffer")); + } + Ordering::Equal => { + // no compression + for _i in 0..count { + // read NullTerminatedString + if let Ok(string) = read_null_terminated_string(reader) { + files.push(string); + } + } + } + } - Ok(()) + let footer = LxrsFooter { files }; + + Ok(footer) + } } ///////////////////////////////////////////////////////////////////////////////////////// From 6a0c246960cd9ce91b2baef5496f183379c2ee76 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Fri, 22 Dec 2023 11:53:08 +0100 Subject: [PATCH 06/14] start read modes --- src/archive.rs | 122 +++++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 54 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 5c60636..9efd881 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -142,12 +142,6 @@ where ) } -pub enum ArchiveMode { - Create, - Read, - Update, -} - /* TODO We don't support different modes for now needs a wrapper class for archives @@ -176,11 +170,17 @@ where /// # Errors /// /// This function will return an error if any io fails. -pub fn open_read

(archive_file_name: P) -> Result +pub fn open_read

(archive_file_name: P) -> Result where P: AsRef, { - Archive::from_file(archive_file_name) + let ar = Archive::from_file(archive_file_name)?; + + let a: ZipArchive = ZipArchive { + mode: ArchiveMode::Read, + }; + + Ok(a) } /// Extracts all files from an archive and writes them to a folder @@ -589,22 +589,36 @@ fn pad_until_page(writer: &mut W) -> Result<()> { Ok(()) } +///////////////////////////////////////////////////////////////////////////////////////// +// TODO API +///////////////////////////////////////////////////////////////////////////////////////// + +pub enum ArchiveMode { + Create, + Read, + Update, +} + +pub struct ZipArchive { + mode: ArchiveMode, +} + ///////////////////////////////////////////////////////////////////////////////////////// // INTERNAL ///////////////////////////////////////////////////////////////////////////////////////// #[derive(Debug, Clone, Default)] -pub struct Archive { - pub header: Header, - pub index: Index, +struct Archive { + header: Header, + index: Index, // custom - pub file_names: HashMap, + file_names: HashMap, } impl Archive { // Function to read a Header from a file - pub fn from_file

(file_path: P) -> Result + fn from_file

(file_path: P) -> Result where P: AsRef, { @@ -625,7 +639,7 @@ impl Archive { Archive::from_reader(&mut cursor) } - pub fn from_reader(cursor: &mut R) -> Result + fn from_reader(cursor: &mut R) -> Result where R: Read + Seek, { @@ -658,7 +672,7 @@ impl Archive { } // get filehashes - pub fn get_file_hashes(&self) -> Vec { + fn get_file_hashes(&self) -> Vec { self.index .file_entries .iter() @@ -668,20 +682,20 @@ impl Archive { } #[derive(Debug, Clone, Copy)] -pub struct Header { - pub magic: u32, - pub version: u32, - pub index_position: u64, - pub index_size: u32, - pub debug_position: u64, - pub debug_size: u32, - pub filesize: u64, +struct Header { + magic: u32, + version: u32, + index_position: u64, + index_size: u32, + debug_position: u64, + debug_size: u32, + filesize: u64, } impl Header { //static HEADER_MAGIC: u32 = 1380009042; //static HEADER_SIZE: i32 = 40; - pub const HEADER_EXTENDED_SIZE: u64 = 0xAC; + const HEADER_EXTENDED_SIZE: u64 = 0xAC; } impl Default for Header { @@ -712,7 +726,7 @@ impl FromReader for Header { } } impl Header { - pub fn serialize(&self, writer: &mut W) -> Result<()> { + fn serialize(&self, writer: &mut W) -> Result<()> { writer.write_u32::(self.magic)?; writer.write_u32::(self.version)?; writer.write_u64::(self.index_position)?; @@ -726,21 +740,21 @@ impl Header { } #[derive(Debug, Clone, Default)] -pub struct Index { - pub file_table_offset: u32, - pub file_table_size: u32, - pub crc: u64, - pub file_entry_count: u32, - pub file_segment_count: u32, - pub resource_dependency_count: u32, +struct Index { + file_table_offset: u32, + file_table_size: u32, + crc: u64, + file_entry_count: u32, + file_segment_count: u32, + resource_dependency_count: u32, // not serialized - pub file_entries: HashMap, - pub file_segments: Vec, - pub dependencies: Vec, + file_entries: HashMap, + file_segments: Vec, + dependencies: Vec, } impl Index { - pub fn serialize(&self, writer: &mut W) -> Result<()> { + fn serialize(&self, writer: &mut W) -> Result<()> { writer.write_u32::(self.file_table_offset)?; writer.write_u32::(self.file_table_size)?; writer.write_u64::(self.crc)?; @@ -787,10 +801,10 @@ impl FromReader for Index { } #[derive(Debug, Clone, Copy)] -pub struct FileSegment { - pub offset: u64, - pub z_size: u32, - pub size: u32, +struct FileSegment { + offset: u64, + z_size: u32, + size: u32, } impl FromReader for FileSegment { @@ -804,15 +818,15 @@ impl FromReader for FileSegment { } #[derive(Debug, Clone, Copy)] -pub struct FileEntry { - pub name_hash_64: u64, - pub timestamp: u64, //SystemTime, - pub num_inline_buffer_segments: u32, - pub segments_start: u32, - pub segments_end: u32, - pub resource_dependencies_start: u32, - pub resource_dependencies_end: u32, - pub sha1_hash: [u8; 20], +struct FileEntry { + name_hash_64: u64, + timestamp: u64, //SystemTime, + num_inline_buffer_segments: u32, + segments_start: u32, + segments_end: u32, + resource_dependencies_start: u32, + resource_dependencies_end: u32, + sha1_hash: [u8; 20], } impl FromReader for FileEntry { @@ -835,8 +849,8 @@ impl FromReader for FileEntry { } #[derive(Debug, Clone, Copy)] -pub struct Dependency { - pub hash: u64, +struct Dependency { + hash: u64, } impl FromReader for Dependency { @@ -848,8 +862,8 @@ impl FromReader for Dependency { } #[derive(Debug, Clone)] -pub struct LxrsFooter { - pub files: Vec, +struct LxrsFooter { + files: Vec, } impl LxrsFooter { @@ -857,7 +871,7 @@ impl LxrsFooter { const MAGIC: u32 = 0x4C585253; const VERSION: u32 = 1; - pub fn serialize(&self, writer: &mut W) -> Result<()> { + fn serialize(&self, writer: &mut W) -> Result<()> { writer.write_u32::(self.files.len() as u32)?; writer.write_u32::(LxrsFooter::VERSION)?; From 349d5d356cafa6cc99609862865d7c2e91589752 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Sat, 23 Dec 2023 10:10:34 +0100 Subject: [PATCH 07/14] lifetime --- src/archive.rs | 522 ++++++++++++++++++++++++-------------- tests/functional_tests.rs | 10 +- 2 files changed, 343 insertions(+), 189 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 9efd881..b0baaa2 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -6,10 +6,10 @@ use std::cmp::Ordering; use std::{ collections::{HashMap, HashSet}, fs::{create_dir_all, File}, - io::{BufReader, BufWriter, Cursor, Read, Result, Seek, SeekFrom, Write}, + io::{BufWriter, Cursor, Read, Result, Seek, SeekFrom, Write}, path::Path, }; -use std::{io, mem}; +use std::{io, vec}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use strum::IntoEnumIterator; @@ -98,15 +98,10 @@ pub fn extract_to_directory( ) -> Result<()> where P: AsRef, - R: Read + Seek, + R: Read + Seek + 'static, { - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - extract_archive(source, destination_directory_name, overwrite_files, &map) + let mut archive = ZipArchive::from_reader(source, ArchiveMode::Read)?; + archive.extract_to_directory(destination_directory_name, overwrite_files, hash_map) } // public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); @@ -116,7 +111,7 @@ where /// # Errors /// /// This function will return an error if any io fails. -pub fn extract_to_directory_path

( +pub fn extract_to_directory_path( source_archive_file_name: &P, destination_directory_name: &P, overwrite_files: bool, @@ -124,29 +119,13 @@ pub fn extract_to_directory_path

( ) -> Result<()> where P: AsRef, + R: Read + Seek, { - let map = if let Some(hash_map) = hash_map { - hash_map - } else { - get_red4_hashes() - }; - - let archive_file = File::open(source_archive_file_name)?; - let mut archive_reader = BufReader::new(archive_file); - - extract_archive( - &mut archive_reader, - destination_directory_name, - overwrite_files, - &map, - ) + let mut file = File::open(source_archive_file_name)?; + let mut archive = ZipArchive::from_reader(&mut file, ArchiveMode::Read)?; + archive.extract_to_directory(destination_directory_name, overwrite_files, hash_map) } -/* -TODO We don't support different modes for now -needs a wrapper class for archives - - // public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); /// Opens an archive at the specified path and in the specified mode. @@ -154,14 +133,20 @@ needs a wrapper class for archives /// # Errors /// /// This function will return an error if any io fails. -pub fn open

(archive_file_name: P, mode: ArchiveMode) -> Result +pub fn open(archive_file_name: P, mode: ArchiveMode) -> Result> where P: AsRef, { - todo!() -} + let archive = match mode { + ArchiveMode::Create => todo!(), + //ArchiveMode::Read => ZipArchive::from_file(archive_file_name, mode)?, + ArchiveMode::Read => todo!(), + //ArchiveMode::Update => ZipArchive::from_file(archive_file_name, mode)?, + ArchiveMode::Update => todo!(), + }; - */ + //Ok(archive) +} // public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); @@ -170,99 +155,12 @@ where /// # Errors /// /// This function will return an error if any io fails. -pub fn open_read

(archive_file_name: P) -> Result -where - P: AsRef, -{ - let ar = Archive::from_file(archive_file_name)?; - - let a: ZipArchive = ZipArchive { - mode: ArchiveMode::Read, - }; - - Ok(a) -} - -/// Extracts all files from an archive and writes them to a folder -/// -/// # Panics -/// -/// Panics if file path operations fail -/// -/// # Errors -/// -/// This function will return an error if any parsing fails -fn extract_archive( - archive_reader: &mut R, - out_dir: &P, - overwrite_files: bool, - hash_map: &HashMap, -) -> Result<()> +pub fn open_read

(archive_file_name: P) -> Result> where P: AsRef, - R: Read + Seek, { - // parse archive headers - let archive = Archive::from_reader(archive_reader)?; - - for (hash, file_entry) in archive.index.file_entries.iter() { - // get filename - let mut name_or_hash: String = format!("{}.bin", hash); - if let Some(name) = hash_map.get(hash) { - name_or_hash = name.to_owned(); - } - if let Some(name) = archive.file_names.get(hash) { - name_or_hash = name.to_owned(); - } - - // name or hash is a relative path - let outfile = out_dir.as_ref().join(name_or_hash); - create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; - - // extract to stream - let mut fs = if overwrite_files { - File::create(outfile)? - } else { - File::options() - .read(true) - .write(true) - .create_new(true) - .open(outfile)? - }; - - //let mut fs = File::create(outfile)?; - let mut file_writer = BufWriter::new(&mut fs); - // decompress main file - let start_index = file_entry.segments_start; - let next_index = file_entry.segments_end; - if let Some(segment) = archive.index.file_segments.get(start_index as usize) { - // read and decompress from main archive stream - - // kraken decompress - if segment.size == segment.z_size { - // just copy over - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } else { - decompress_segment(archive_reader, segment, &mut file_writer)?; - } - } - - // extract additional buffers - for i in start_index + 1..next_index { - if let Some(segment) = archive.index.file_segments.get(i as usize) { - // do not decompress with oodle - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } - } - } - - Ok(()) + let file = File::open(archive_file_name)?; + ZipArchive::from_reader_consume(file, ArchiveMode::Read) } /// Packs redengine 4 resource file in a folder to an archive @@ -281,14 +179,6 @@ where { /*if !in_folder.exists() { return Err(Error::new(ErrorKind::InvalidInput, "")); - } - - if !out_folder.exists() { - return Err(Error::new(ErrorKind::InvalidInput, "")); - } - // check extension - if !out_folder.exists() { - return Err(Error::new(ErrorKind::InvalidInput, "")); }*/ // collect files @@ -590,67 +480,233 @@ fn pad_until_page(writer: &mut W) -> Result<()> { } ///////////////////////////////////////////////////////////////////////////////////////// -// TODO API +// API ///////////////////////////////////////////////////////////////////////////////////////// +#[derive(Debug, Clone, Default, PartialEq)] pub enum ArchiveMode { + #[default] Create, Read, Update, } -pub struct ZipArchive { +#[derive(Debug)] +pub struct ZipArchive<'a, S> { + /// wraps an Archive internally + internal_archive: Archive, + /// wraps a stream + stream: &'a mut S, + + /// The read-write mode of the archive mode: ArchiveMode, + /// The files inside an archive + entries: HashMap, } -///////////////////////////////////////////////////////////////////////////////////////// -// INTERNAL -///////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone, Default)] -struct Archive { - header: Header, - index: Index, +/*impl<'a> ZipArchive<'a, File> { + /// Opens an archive from a file in a specific mode + fn from_file

(file_path: P, mode: ArchiveMode) -> Result> + where + P: AsRef, + { + // checks + if mode == ArchiveMode::Create { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Can only construct an archive in read-only mode.", + )); + } - // custom - file_names: HashMap, -} + let mut file = File::open(file_path)?; + ZipArchive::from_reader(&mut file, mode) + } +}*/ -impl Archive { - // Function to read a Header from a file - fn from_file

(file_path: P) -> Result +impl ZipArchive<'_, R> +where + R: Read + Seek, +{ + pub fn extract_to_directory

( + &mut self, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, + ) -> Result<()> where P: AsRef, { - let mut file = File::open(file_path)?; - let mut buffer = Vec::with_capacity(mem::size_of::

()); - file.read_to_end(&mut buffer)?; + let hash_map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + // collect info + let mut temp: Vec<(String, FileSegment, Vec)> = vec![]; + for (hash, file_entry) in self.entries.iter() { + // get filename + let resolved = if let Some(name) = &file_entry.name { + name.to_owned() + } else { + let mut name_or_hash: String = format!("{}.bin", hash); + // check vanilla hashes + if let Some(name) = hash_map.get(&hash) { + name_or_hash = name.to_owned(); + } + name_or_hash + }; + + let start_index = file_entry.segments_start(); + let next_index = file_entry.segments_end(); + + if let Some(segment) = self.get_file_segment(start_index as usize) { + let mut buffers: Vec = vec![]; + for i in start_index + 1..next_index { + if let Some(segment) = self.get_file_segment(i as usize) { + buffers.push(*segment); + } + } + temp.push((resolved, *segment, buffers)); + } + } + + // extract + for (resolved, segment, buffers) in temp { + // name or hash is a relative path + let outfile = destination_directory_name.as_ref().join(resolved); + create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; + + // extract to stream + let mut fs = if overwrite_files { + File::create(outfile)? + } else { + File::options() + .read(true) + .write(true) + .create_new(true) + .open(outfile)? + }; + + //let mut fs = File::create(outfile)?; + let mut file_writer = BufWriter::new(&mut fs); + // decompress main file + //let start_index = file_entry.segments_start(); + //let next_index = file_entry.segments_end(); + //if let Some(segment) = self.get_file_segment(start_index as usize) { + // read and decompress from main archive stream + + // kraken decompress + if segment.size == segment.z_size { + // just copy + self.reader_mut().seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + self.reader_mut().read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + } else { + decompress_segment(self.reader_mut(), &segment, &mut file_writer)?; + } + //} + + // extract additional buffers + //for i in start_index + 1..next_index { + for segment in buffers { + //if let Some(segment) = self.get_file_segment(i as usize) { + // do not decompress with oodle + self.reader_mut().seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + self.reader_mut().read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + //} + } + } + + Ok(()) + } - // Ensure that the buffer has enough bytes to represent a Header - if buffer.len() < mem::size_of::
() { + fn reader_mut(&mut self) -> &mut R { + self.stream + } + + fn reader(&self) -> &R { + &self.stream + } + + /// Opens an archive from, needs to be read-only + fn from_reader_consume<'a>(mut reader: R, mode: ArchiveMode) -> Result> { + // checks + if mode == ArchiveMode::Create { return Err(io::Error::new( io::ErrorKind::InvalidData, - "File does not contain enough data to parse Header", + "Can only construct an archive in read-only mode.", )); } - let mut cursor = io::Cursor::new(&buffer); + // read header + let header = Header::from_reader(&mut reader)?; - Archive::from_reader(&mut cursor) + // read custom data + let mut file_names: HashMap = HashMap::default(); + if let Ok(custom_data_length) = reader.read_u32::() { + if custom_data_length > 0 { + reader.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; + if let Ok(footer) = LxrsFooter::from_reader(&mut reader) { + // add files to hashmap + for f in footer.files { + let hash = fnv1a64_hash_string(&f); + file_names.insert(hash, f); + } + } + } + } + + // read index + // move to offset Header.IndexPosition + reader.seek(io::SeekFrom::Start(header.index_position))?; + let index = Index::from_reader(&mut reader)?; + + // construct wrapper + let internal_archive = Archive { header, index }; + let mut entries: HashMap = HashMap::default(); + for (hash, entry) in internal_archive.index.file_entries.iter() { + let resolved = if let Some(name) = file_names.get(hash) { + Some(name.to_owned()) + } else { + None + }; + + let zip_entry = ZipEntry::new(*hash, resolved, *entry); + entries.insert(*hash, zip_entry); + } + + let archive = ZipArchive::<'a, R> { + stream: &mut reader, + internal_archive, + mode, + entries, + }; + Ok(archive) } - fn from_reader(cursor: &mut R) -> Result - where - R: Read + Seek, - { - let header = Header::from_reader(cursor)?; + /// Opens an archive from, needs to be read-only + fn from_reader<'a>(mut reader: &'a mut R, mode: ArchiveMode) -> Result> { + // checks + if mode == ArchiveMode::Create { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Can only construct an archive in read-only mode.", + )); + } + + // read header + let header = Header::from_reader(&mut reader)?; // read custom data let mut file_names: HashMap = HashMap::default(); - if let Ok(custom_data_length) = cursor.read_u32::() { + if let Ok(custom_data_length) = reader.read_u32::() { if custom_data_length > 0 { - cursor.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; - if let Ok(footer) = LxrsFooter::from_reader(cursor) { + reader.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; + if let Ok(footer) = LxrsFooter::from_reader(&mut reader) { // add files to hashmap for f in footer.files { let hash = fnv1a64_hash_string(&f); @@ -660,25 +716,113 @@ impl Archive { } } + // read index // move to offset Header.IndexPosition - cursor.seek(io::SeekFrom::Start(header.index_position))?; - let index = Index::from_reader(cursor)?; + reader.seek(io::SeekFrom::Start(header.index_position))?; + let index = Index::from_reader(&mut reader)?; + + // construct wrapper + let internal_archive = Archive { header, index }; + let mut entries: HashMap = HashMap::default(); + for (hash, entry) in internal_archive.index.file_entries.iter() { + let resolved = if let Some(name) = file_names.get(hash) { + Some(name.to_owned()) + } else { + None + }; - Ok(Archive { - header, - index, - file_names, - }) + let zip_entry = ZipEntry::new(*hash, resolved, *entry); + entries.insert(*hash, zip_entry); + } + + let archive = ZipArchive:: { + stream: reader, + internal_archive, + mode, + entries, + }; + Ok(archive) + } +} + +impl ZipArchive<'_, W> {} + +impl ZipArchive<'_, S> { + /// Compresses and adds a file to the archive. + /// + /// # Errors + /// + /// This function will return an error if compression or io fails, or if the mode is Read. + pub fn create_entry>( + &self, + file_path: P, + compression_level: CompressionLevel, + ) -> Result { + if self.mode == ArchiveMode::Read { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Archive is in read-only mode.", + )); + } + + todo!() + } + + /// Get an entry in the archive by resource path. + pub fn get_entry(&self, name: &str) -> Option<&ZipEntry> { + self.entries.get(&fnv1a64_hash_string(&name.to_owned())) + } + + /// Get an entry in the archive by hash (FNV1a64 of resource path). + pub fn get_entry_by_hash(&self, hash: &u64) -> Option<&ZipEntry> { + self.entries.get(hash) + } + + // private methods + + fn get_file_segment(&self, i: usize) -> Option<&FileSegment> { + self.internal_archive.index.file_segments.get(i) + } +} + +#[derive(Debug, Clone)] +pub struct ZipEntry { + /// FNV1a64 hash of the entry name + hash: u64, + /// Resolved resource path of that entry, this may not be available + name: Option, + + /// wrapped internal struct + entry: FileEntry, +} + +impl ZipEntry { + fn new(hash: u64, name: Option, entry: FileEntry) -> Self { + Self { hash, name, entry } + } + + // getters + + fn segments_start(&self) -> u32 { + self.entry.segments_start } - // get filehashes - fn get_file_hashes(&self) -> Vec { - self.index - .file_entries - .iter() - .map(|f| f.1.name_hash_64) - .collect::>() + fn segments_end(&self) -> u32 { + self.entry.segments_end } + + // methods + pub fn delete() {} +} + +///////////////////////////////////////////////////////////////////////////////////////// +// INTERNAL +///////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Default)] +struct Archive { + header: Header, + index: Index, } #[derive(Debug, Clone, Copy)] @@ -953,13 +1097,15 @@ impl FromReader for LxrsFooter { #[cfg(test)] mod integration_tests { use std::{ - fs, + fs::{self, File}, io::{self, Read}, path::PathBuf, }; + use crate::archive::{open_read, ArchiveMode, ZipArchive}; + + use super::FromReader; use super::LxrsFooter; - use super::{Archive, FromReader}; #[test] fn read_srxl() { @@ -976,25 +1122,29 @@ mod integration_tests { #[test] fn read_archive() { let archive_path = PathBuf::from("tests").join("test1.archive"); - let result = Archive::from_file(archive_path); + let result = open_read(archive_path); assert!(result.is_ok()); } #[test] fn read_archive2() { let archive_path = PathBuf::from("tests").join("nci.archive"); - let result = Archive::from_file(archive_path); + let mut file = File::open(archive_path).expect("Could not parse archive"); + let result = ZipArchive::from_reader(&mut file, ArchiveMode::Read); assert!(result.is_ok()); } #[test] fn read_custom_data() { let archive_path = PathBuf::from("tests").join("test1.archive"); - let archive = Archive::from_file(archive_path).expect("Could not parse archive"); + let mut file = File::open(archive_path).expect("Could not parse archive"); + let archive = + ZipArchive::from_reader(&mut file, ArchiveMode::Read).expect("Could not parse archive"); let mut file_names = archive - .file_names + .entries .values() - .map(|f| f.to_owned()) + .map(|f| f.name.to_owned()) + .flatten() .collect::>(); file_names.sort(); diff --git a/tests/functional_tests.rs b/tests/functional_tests.rs index 6b75544..e81032f 100644 --- a/tests/functional_tests.rs +++ b/tests/functional_tests.rs @@ -4,7 +4,7 @@ #[cfg(test)] mod tests { - use std::fs::create_dir_all; + use std::fs::{create_dir_all, File}; use std::path::Path; use std::time::Instant; use std::{fs, path::PathBuf}; @@ -33,8 +33,12 @@ mod tests { assert!(fs::remove_dir_all(&dst_path).is_ok()); } - let result = - archive::extract_to_directory_path(&archive_path, &dst_path, true, Some(hashes)); + let result = archive::extract_to_directory_path::( + &archive_path, + &dst_path, + true, + Some(hashes), + ); assert!(result.is_ok()); // check From 22d7c793fe452031abefdb57b9d1c1fd7b0041d2 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Sun, 24 Dec 2023 10:50:14 +0100 Subject: [PATCH 08/14] add more implementations --- src/archive.rs | 471 +++++++++++++++++++++++++++---------------------- 1 file changed, 260 insertions(+), 211 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index b0baaa2..1dddb00 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -2,14 +2,14 @@ // ARCHIVE ///////////////////////////////////////////////////////////////////////////////////////// -use std::cmp::Ordering; use std::{ + borrow::BorrowMut, + cmp::Ordering, collections::{HashMap, HashSet}, fs::{create_dir_all, File}, - io::{BufWriter, Cursor, Read, Result, Seek, SeekFrom, Write}, + io::{self, BufWriter, Cursor, Read, Result, Seek, SeekFrom, Write}, path::Path, }; -use std::{io, vec}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use strum::IntoEnumIterator; @@ -100,7 +100,7 @@ where P: AsRef, R: Read + Seek + 'static, { - let mut archive = ZipArchive::from_reader(source, ArchiveMode::Read)?; + let mut archive = ZipArchive::from_reader_consume(source, ArchiveMode::Read)?; archive.extract_to_directory(destination_directory_name, overwrite_files, hash_map) } @@ -121,8 +121,7 @@ where P: AsRef, R: Read + Seek, { - let mut file = File::open(source_archive_file_name)?; - let mut archive = ZipArchive::from_reader(&mut file, ArchiveMode::Read)?; + let mut archive = open_read(source_archive_file_name)?; archive.extract_to_directory(destination_directory_name, overwrite_files, hash_map) } @@ -133,19 +132,21 @@ where /// # Errors /// /// This function will return an error if any io fails. -pub fn open(archive_file_name: P, mode: ArchiveMode) -> Result> +pub fn open

(archive_file_name: P, mode: ArchiveMode) -> Result> where P: AsRef, { - let archive = match mode { - ArchiveMode::Create => todo!(), - //ArchiveMode::Read => ZipArchive::from_file(archive_file_name, mode)?, - ArchiveMode::Read => todo!(), - //ArchiveMode::Update => ZipArchive::from_file(archive_file_name, mode)?, - ArchiveMode::Update => todo!(), - }; - - //Ok(archive) + match mode { + ArchiveMode::Create => { + let file = File::create(archive_file_name)?; + ZipArchive::from_reader_consume(file, mode) + } + ArchiveMode::Read => open_read(archive_file_name), + ArchiveMode::Update => { + let file = File::open(archive_file_name)?; + ZipArchive::from_reader_consume(file, mode) + } + } } // public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); @@ -155,7 +156,7 @@ where /// # Errors /// /// This function will return an error if any io fails. -pub fn open_read

(archive_file_name: P) -> Result> +pub fn open_read

(archive_file_name: P) -> Result> where P: AsRef, { @@ -492,11 +493,11 @@ pub enum ArchiveMode { } #[derive(Debug)] -pub struct ZipArchive<'a, S> { +pub struct ZipArchive { /// wraps an Archive internally internal_archive: Archive, /// wraps a stream - stream: &'a mut S, + stream: S, /// The read-write mode of the archive mode: ArchiveMode, @@ -504,38 +505,155 @@ pub struct ZipArchive<'a, S> { entries: HashMap, } -/*impl<'a> ZipArchive<'a, File> { - /// Opens an archive from a file in a specific mode - fn from_file

(file_path: P, mode: ArchiveMode) -> Result> - where - P: AsRef, - { - // checks - if mode == ArchiveMode::Create { +impl ZipArchive { + /// Get an entry in the archive by resource path. + pub fn get_entry(&self, name: &str) -> Option<&ZipEntry> { + self.entries.get(&fnv1a64_hash_string(&name.to_owned())) + } + + /// Get an entry in the archive by hash (FNV1a64 of resource path). + pub fn get_entry_by_hash(&self, hash: &u64) -> Option<&ZipEntry> { + self.entries.get(hash) + } + + // private methods + fn get_file_segment(&self, i: usize) -> Option<&FileSegment> { + self.internal_archive.index.file_segments.get(i) + } +} + +type EntryExtractInfo = (String, FileSegment, Vec); + +impl ZipArchive +where + R: Read + Seek, +{ + /// Extracts a single entry to a directory path. + /// + /// # Errors + /// + /// This function will return an error if the entry cannot be found or any io fails. + pub fn extract_entry>( + &mut self, + entry: ZipEntry, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: &HashMap, + ) -> Result<()> { + let Some(info) = self.get_extract_info(&entry, &hash_map) else { return Err(io::Error::new( io::ErrorKind::InvalidData, - "Can only construct an archive in read-only mode.", + "Could not get entry info from archive.", + )); + }; + + // name or hash is a relative path + let outfile = destination_directory_name.as_ref().join(info.0); + create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; + + // extract to stream + let mut fs = if overwrite_files { + File::create(outfile)? + } else { + File::options() + .read(true) + .write(true) + .create_new(true) + .open(outfile)? + }; + + let writer = BufWriter::new(&mut fs); + self.extract_segments(info.1, info.2, writer)?; + + Ok(()) + } + + /// Extracts a single entry by hash to a directory path. + /// + /// # Errors + /// + /// This function will return an error if the entry cannot be found or any io fails. + pub fn extract_entry_by_hash>( + &mut self, + hash: u64, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: &HashMap, + ) -> Result<()> { + if let Some(entry) = self.get_entry_by_hash(&hash) { + self.extract_entry( + entry.clone(), + destination_directory_name, + overwrite_files, + hash_map, + ) + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Could not find entry.", )); } + } - let mut file = File::open(file_path)?; - ZipArchive::from_reader(&mut file, mode) + /// Extracts a single entry by resource path to a directory path. + /// + /// # Errors + /// + /// This function will return an error if the entry cannot be found or any io fails. + pub fn extract_entry_by_name>( + &mut self, + name: String, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: &HashMap, + ) -> Result<()> { + if let Some(entry) = self.get_entry(&name) { + self.extract_entry( + entry.clone(), + destination_directory_name, + overwrite_files, + hash_map, + ) + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Could not find entry.", + )); + } } -}*/ -impl ZipArchive<'_, R> -where - R: Read + Seek, -{ - pub fn extract_to_directory

( + /// Returns an open read stream to an entry of this [`ZipArchive`]. + pub fn open_entry( + &mut self, + entry: ZipEntry, + hash_map: &HashMap, + writer: W, + ) -> Result<()> { + let Some(info) = self.get_extract_info(&entry, &hash_map) else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Could not get entry info from archive.", + )); + }; + + let segment = info.1; + let buffers = info.2; + self.extract_segments(segment, buffers, writer)?; + + Ok(()) + } + + /// Extracts all entries to the given directory. + /// + /// # Errors + /// + /// This function will return an error if io fails. + pub fn extract_to_directory>( &mut self, destination_directory_name: &P, overwrite_files: bool, hash_map: Option>, - ) -> Result<()> - where - P: AsRef, - { + ) -> Result<()> { let hash_map = if let Some(hash_map) = hash_map { hash_map } else { @@ -543,159 +661,105 @@ where }; // collect info - let mut temp: Vec<(String, FileSegment, Vec)> = vec![]; - for (hash, file_entry) in self.entries.iter() { - // get filename - let resolved = if let Some(name) = &file_entry.name { - name.to_owned() - } else { - let mut name_or_hash: String = format!("{}.bin", hash); - // check vanilla hashes - if let Some(name) = hash_map.get(&hash) { - name_or_hash = name.to_owned(); - } - name_or_hash - }; - - let start_index = file_entry.segments_start(); - let next_index = file_entry.segments_end(); - - if let Some(segment) = self.get_file_segment(start_index as usize) { - let mut buffers: Vec = vec![]; - for i in start_index + 1..next_index { - if let Some(segment) = self.get_file_segment(i as usize) { - buffers.push(*segment); - } - } - temp.push((resolved, *segment, buffers)); - } + let mut entries: Vec = vec![]; + for (_hash, entry) in &self.entries { + entries.push(entry.clone()); } - // extract - for (resolved, segment, buffers) in temp { - // name or hash is a relative path - let outfile = destination_directory_name.as_ref().join(resolved); - create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; - - // extract to stream - let mut fs = if overwrite_files { - File::create(outfile)? - } else { - File::options() - .read(true) - .write(true) - .create_new(true) - .open(outfile)? - }; - - //let mut fs = File::create(outfile)?; - let mut file_writer = BufWriter::new(&mut fs); - // decompress main file - //let start_index = file_entry.segments_start(); - //let next_index = file_entry.segments_end(); - //if let Some(segment) = self.get_file_segment(start_index as usize) { - // read and decompress from main archive stream - - // kraken decompress - if segment.size == segment.z_size { - // just copy - self.reader_mut().seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - self.reader_mut().read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } else { - decompress_segment(self.reader_mut(), &segment, &mut file_writer)?; - } - //} - - // extract additional buffers - //for i in start_index + 1..next_index { - for segment in buffers { - //if let Some(segment) = self.get_file_segment(i as usize) { - // do not decompress with oodle - self.reader_mut().seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - self.reader_mut().read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - //} - } + for entry in entries { + self.extract_entry( + entry, + destination_directory_name, + overwrite_files, + &hash_map, + )?; } Ok(()) } + // getters + fn reader_mut(&mut self) -> &mut R { - self.stream + self.stream.borrow_mut() } - fn reader(&self) -> &R { - &self.stream - } + // methods - /// Opens an archive from, needs to be read-only - fn from_reader_consume<'a>(mut reader: R, mode: ArchiveMode) -> Result> { - // checks - if mode == ArchiveMode::Create { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Can only construct an archive in read-only mode.", - )); + /// Extracts segments to a writer, expects correct offset info. + /// + /// # Errors + /// + /// This function will return an error if io fails + fn extract_segments( + &mut self, + segment: FileSegment, + buffers: Vec, + mut writer: W, + ) -> Result<()> { + if segment.size == segment.z_size { + // just copy + self.reader_mut().seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + self.reader_mut().read_exact(&mut buffer[..])?; + writer.write_all(&buffer)?; + } else { + decompress_segment(self.reader_mut(), &segment, &mut writer)?; + } + for segment in buffers { + self.reader_mut().seek(SeekFrom::Start(segment.offset))?; + let mut buffer = vec![0; segment.z_size as usize]; + self.reader_mut().read_exact(&mut buffer[..])?; + writer.write_all(&buffer)?; } - // read header - let header = Header::from_reader(&mut reader)?; + Ok(()) + } - // read custom data - let mut file_names: HashMap = HashMap::default(); - if let Ok(custom_data_length) = reader.read_u32::() { - if custom_data_length > 0 { - reader.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; - if let Ok(footer) = LxrsFooter::from_reader(&mut reader) { - // add files to hashmap - for f in footer.files { - let hash = fnv1a64_hash_string(&f); - file_names.insert(hash, f); - } - } + /// Gets offset info from the underlying archive for a given entry. + fn get_extract_info( + &self, + file_entry: &ZipEntry, + hash_map: &HashMap, + ) -> Option<(String, FileSegment, Vec)> { + // get filename + let resolved = if let Some(name) = &file_entry.name { + name.to_owned() + } else { + let mut name_or_hash: String = format!("{}.bin", file_entry.hash); + // check vanilla hashes + if let Some(name) = hash_map.get(&file_entry.hash) { + name_or_hash = name.to_owned(); } - } - - // read index - // move to offset Header.IndexPosition - reader.seek(io::SeekFrom::Start(header.index_position))?; - let index = Index::from_reader(&mut reader)?; + name_or_hash + }; - // construct wrapper - let internal_archive = Archive { header, index }; - let mut entries: HashMap = HashMap::default(); - for (hash, entry) in internal_archive.index.file_entries.iter() { - let resolved = if let Some(name) = file_names.get(hash) { - Some(name.to_owned()) - } else { - None - }; + let start_index = file_entry.segments_start(); + let next_index = file_entry.segments_end(); + let mut info: Option = None; + if let Some(segment) = self.get_file_segment(start_index as usize) { + let mut buffers: Vec = vec![]; + for i in start_index + 1..next_index { + if let Some(segment) = self.get_file_segment(i as usize) { + buffers.push(*segment); + } + } - let zip_entry = ZipEntry::new(*hash, resolved, *entry); - entries.insert(*hash, zip_entry); + info = Some((resolved, *segment, buffers)); } - - let archive = ZipArchive::<'a, R> { - stream: &mut reader, - internal_archive, - mode, - entries, - }; - Ok(archive) + info } - /// Opens an archive from, needs to be read-only - fn from_reader<'a>(mut reader: &'a mut R, mode: ArchiveMode) -> Result> { + /// Opens an archive, needs to be read-only + fn from_reader_consume(mut reader: R, mode: ArchiveMode) -> Result> { // checks if mode == ArchiveMode::Create { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Can only construct an archive in read-only mode.", - )); + return Ok(ZipArchive:: { + stream: reader, + internal_archive: Archive::default(), + mode, + entries: HashMap::default(), + }); } // read header @@ -731,23 +795,24 @@ where None }; - let zip_entry = ZipEntry::new(*hash, resolved, *entry); + let zip_entry = ZipEntry { + hash: *hash, + name: resolved, + entry: *entry, + }; entries.insert(*hash, zip_entry); } - let archive = ZipArchive:: { + Ok(ZipArchive:: { stream: reader, internal_archive, mode, entries, - }; - Ok(archive) + }) } } -impl ZipArchive<'_, W> {} - -impl ZipArchive<'_, S> { +impl ZipArchive { /// Compresses and adds a file to the archive. /// /// # Errors @@ -755,8 +820,8 @@ impl ZipArchive<'_, S> { /// This function will return an error if compression or io fails, or if the mode is Read. pub fn create_entry>( &self, - file_path: P, - compression_level: CompressionLevel, + _file_path: P, + _compression_level: CompressionLevel, ) -> Result { if self.mode == ArchiveMode::Read { return Err(io::Error::new( @@ -768,20 +833,12 @@ impl ZipArchive<'_, S> { todo!() } - /// Get an entry in the archive by resource path. - pub fn get_entry(&self, name: &str) -> Option<&ZipEntry> { - self.entries.get(&fnv1a64_hash_string(&name.to_owned())) - } - - /// Get an entry in the archive by hash (FNV1a64 of resource path). - pub fn get_entry_by_hash(&self, hash: &u64) -> Option<&ZipEntry> { - self.entries.get(hash) - } - - // private methods + /// Deletes an entry from the archive + pub fn delete_entry(&mut self, hash: &u64) -> Option { + let removed = self.entries.remove(hash); - fn get_file_segment(&self, i: usize) -> Option<&FileSegment> { - self.internal_archive.index.file_segments.get(i) + // todo write? update offsets? + removed } } @@ -797,12 +854,7 @@ pub struct ZipEntry { } impl ZipEntry { - fn new(hash: u64, name: Option, entry: FileEntry) -> Self { - Self { hash, name, entry } - } - // getters - fn segments_start(&self) -> u32 { self.entry.segments_start } @@ -810,9 +862,6 @@ impl ZipEntry { fn segments_end(&self) -> u32 { self.entry.segments_end } - - // methods - pub fn delete() {} } ///////////////////////////////////////////////////////////////////////////////////////// @@ -960,7 +1009,7 @@ impl FromReader for FileSegment { }) } } - +#[allow(dead_code)] #[derive(Debug, Clone, Copy)] struct FileEntry { name_hash_64: u64, @@ -972,6 +1021,7 @@ struct FileEntry { resource_dependencies_end: u32, sha1_hash: [u8; 20], } +#[warn(dead_code)] impl FromReader for FileEntry { fn from_reader(reader: &mut R) -> io::Result { @@ -992,10 +1042,12 @@ impl FromReader for FileEntry { } } +#[allow(dead_code)] #[derive(Debug, Clone, Copy)] struct Dependency { hash: u64, } +#[warn(dead_code)] impl FromReader for Dependency { fn from_reader(reader: &mut R) -> io::Result { @@ -1097,12 +1149,12 @@ impl FromReader for LxrsFooter { #[cfg(test)] mod integration_tests { use std::{ - fs::{self, File}, + fs::{self}, io::{self, Read}, path::PathBuf, }; - use crate::archive::{open_read, ArchiveMode, ZipArchive}; + use crate::archive::open_read; use super::FromReader; use super::LxrsFooter; @@ -1128,18 +1180,15 @@ mod integration_tests { #[test] fn read_archive2() { - let archive_path = PathBuf::from("tests").join("nci.archive"); - let mut file = File::open(archive_path).expect("Could not parse archive"); - let result = ZipArchive::from_reader(&mut file, ArchiveMode::Read); + let file = PathBuf::from("tests").join("nci.archive"); + let result = open_read(file); assert!(result.is_ok()); } #[test] fn read_custom_data() { - let archive_path = PathBuf::from("tests").join("test1.archive"); - let mut file = File::open(archive_path).expect("Could not parse archive"); - let archive = - ZipArchive::from_reader(&mut file, ArchiveMode::Read).expect("Could not parse archive"); + let file = PathBuf::from("tests").join("test1.archive"); + let archive = open_read(file).expect("Could not parse archive"); let mut file_names = archive .entries .values() From 283ddd6b7f39505b3f6192e99dd201ae6dd7524a Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Tue, 26 Dec 2023 17:43:13 +0100 Subject: [PATCH 09/14] better writing --- Cargo.lock | 7 + Cargo.toml | 1 + src/archive.rs | 484 +++++++++++++++++++++++++++++-------------------- 3 files changed, 296 insertions(+), 196 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3cd3114..edd048f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,6 +120,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2707e3afba5e19b75d582d88bc79237418f2a2a2d673d01cf9b03633b46e98f3" + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -298,6 +304,7 @@ version = "0.2.0" dependencies = [ "byteorder", "cmake", + "crc64", "fnv", "sha1", "strum", diff --git a/Cargo.toml b/Cargo.toml index 23a8393..58631b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" byteorder = "1.5" fnv = "1.0" sha1 = "0.10" +crc64 = "2.0" strum = "0.25" strum_macros = "0.25" walkdir = "2.4" diff --git a/src/archive.rs b/src/archive.rs index 1dddb00..971b5fd 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -7,8 +7,8 @@ use std::{ cmp::Ordering, collections::{HashMap, HashSet}, fs::{create_dir_all, File}, - io::{self, BufWriter, Cursor, Read, Result, Seek, SeekFrom, Write}, - path::Path, + io::{self, BufWriter, Cursor, Error, ErrorKind, Read, Result, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, }; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; @@ -178,33 +178,17 @@ where P: AsRef, W: Write + Seek, { - /*if !in_folder.exists() { - return Err(Error::new(ErrorKind::InvalidInput, "")); - }*/ - - // collect files - let mut included_extensions = ERedExtension::iter() - .map(|variant| variant.to_string()) - .collect::>(); - included_extensions.push(String::from("bin")); - - // get only resource files - let allfiles = WalkDir::new(in_folder) - .into_iter() - .filter_map(|e| e.ok()) - .map(|f| f.into_path()) - .filter(|p| { - if let Some(ext) = p.extension() { - if let Some(ext) = ext.to_str() { - return included_extensions.contains(&ext.to_owned()); - } - } - false - }) - .collect::>(); + if !in_folder.as_ref().exists() { + return Err(Error::new( + ErrorKind::InvalidInput, + "Input folder does not exist", + )); + } + // get files + let resources = collect_resource_files(in_folder); - // sort by hash - let mut hashed_paths = allfiles + // get paths and sort by hash + let mut file_info = resources .iter() .filter_map(|f| { if let Ok(relative_path) = f.strip_prefix(in_folder) { @@ -216,22 +200,9 @@ where None }) .collect::>(); - hashed_paths.sort_by_key(|k| k.1); + file_info.sort_by_key(|k| k.1); - let mut archive_writer = BufWriter::new(out_stream); - - // write temp header - let mut archive = Archive::default(); - let header = Header::default(); - header.serialize(&mut archive_writer)?; - archive_writer.write_all(&[0u8; 132])?; // some weird padding - - // write custom header - assert_eq!( - Header::HEADER_EXTENDED_SIZE, - archive_writer.stream_position()? - ); - let custom_paths = hashed_paths + let custom_paths = file_info .iter() .filter(|(_p, k)| hash_map.contains_key(k)) .filter_map(|(f, _h)| { @@ -242,18 +213,31 @@ where }) .collect::>(); + // start write + + let mut archive_writer = BufWriter::new(out_stream); + + // write empty header + let header = Header::default(); + header.serialize(&mut archive_writer)?; + archive_writer.write_all(&[0u8; 132])?; // padding + + // write custom header let mut custom_data_length = 0; if !custom_paths.is_empty() { let wfooter = LxrsFooter { files: custom_paths, }; - wfooter.serialize(&mut archive_writer)?; + wfooter.write(&mut archive_writer)?; custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; } // write files + let mut file_segments_cnt = 0; + let mut entries = HashMap::default(); let mut imports_hash_set: HashSet = HashSet::new(); - for (path, hash) in hashed_paths { + + for (path, hash) in file_info { // read file let mut file = File::open(&path)?; let mut file_buffer = Vec::new(); @@ -261,11 +245,14 @@ where let firstimportidx = imports_hash_set.len(); let mut lastimportidx = imports_hash_set.len(); - let firstoffsetidx = archive.index.file_segments.len(); + let firstoffsetidx = file_segments_cnt; let mut lastoffsetidx = 0; let mut flags = 0; let mut file_cursor = Cursor::new(&file_buffer); + let mut segment: Option = None; + let mut buffers = vec![]; + if let Ok(info) = read_cr2w_header(&mut file_cursor) { // get main file file_cursor.seek(SeekFrom::Start(0))?; @@ -293,11 +280,12 @@ where archive_writer.write_all(&compressed_buffer)?; // add metadata to archive - archive.index.file_segments.push(FileSegment { + segment = Some(FileSegment { offset: archive_offset, size, z_size: zsize as u32, }); + file_segments_cnt += 1; // write buffers (bytes after the main file) for buffer_info in info.buffers_table.iter() { @@ -310,11 +298,12 @@ where archive_writer.write_all(buffer.as_slice())?; // add metadata to archive - archive.index.file_segments.push(FileSegment { + buffers.push(FileSegment { offset: boffset, size: bsize, z_size: bzsize, }); + file_segments_cnt += 1; } //register imports @@ -325,7 +314,7 @@ where } lastimportidx = imports_hash_set.len(); - lastoffsetidx = archive.index.file_segments.len(); + lastoffsetidx = file_segments_cnt; flags = if !info.buffers_table.is_empty() { info.buffers_table.len() - 1 } else { @@ -342,10 +331,11 @@ where let offset = archive_writer.stream_position()?; let size = file_buffer.len() as u32; - let mut final_zsize = file_buffer.len() as u32; + let final_zsize; if get_uncompressed_file_extensions().contains(&ext) { // direct copy archive_writer.write_all(&file_buffer)?; + final_zsize = size; } else { // kark file let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); @@ -363,11 +353,12 @@ where } // add metadata to archive - archive.index.file_segments.push(FileSegment { + segment = Some(FileSegment { offset, size, z_size: final_zsize, }); + file_segments_cnt += 1; } } @@ -384,16 +375,39 @@ where resource_dependencies_end: lastimportidx as u32, sha1_hash, }; - archive.index.file_entries.insert(hash, entry); + + if let Some(segment) = segment { + let wrapped_entry = ZipEntry { + hash, + name: None, + entry, + segment, + buffers, + }; + entries.insert(hash, wrapped_entry); + } } // write footers + let archive = ZipArchive { + stream: todo!(), + mode: ArchiveMode::Create, + entries, + dirty: false, + dependencies: imports_hash_set + .iter() + .map(|e| Dependency { + hash: fnv1a64_hash_string(e), + }) + .collect::>(), + }; + // padding pad_until_page(&mut archive_writer)?; // write tables let tableoffset = archive_writer.stream_position()?; - archive.index.serialize(&mut archive_writer)?; + archive.write_index(&mut archive_writer)?; let tablesize = archive_writer.stream_position()? - tableoffset; // padding @@ -401,16 +415,40 @@ where let filesize = archive_writer.stream_position()?; // write the header again - archive.header.index_position = tableoffset; - archive.header.index_size = tablesize as u32; - archive.header.filesize = filesize; + header.index_position = tableoffset; + header.index_size = tablesize as u32; + header.filesize = filesize; archive_writer.seek(SeekFrom::Start(0))?; - archive.header.serialize(&mut archive_writer)?; + header.serialize(&mut archive_writer)?; archive_writer.write_u32::(custom_data_length as u32)?; Ok(()) } +fn collect_resource_files>(in_folder: &P) -> Vec { + // collect files + let mut included_extensions = ERedExtension::iter() + .map(|variant| variant.to_string()) + .collect::>(); + included_extensions.push(String::from("bin")); + + // get only resource files + let allfiles = WalkDir::new(in_folder) + .into_iter() + .filter_map(|e| e.ok()) + .map(|f| f.into_path()) + .filter(|p| { + if let Some(ext) = p.extension() { + if let Some(ext) = ext.to_str() { + return included_extensions.contains(&ext.to_owned()); + } + } + false + }) + .collect::>(); + allfiles +} + /// Decompresses and writes a kraken-compressed segment from an archive to a stream /// /// # Errors @@ -494,15 +532,24 @@ pub enum ArchiveMode { #[derive(Debug)] pub struct ZipArchive { - /// wraps an Archive internally - internal_archive: Archive, /// wraps a stream stream: S, /// The read-write mode of the archive mode: ArchiveMode, + dirty: bool, /// The files inside an archive entries: HashMap, + dependencies: Vec, +} + +impl Drop for ZipArchive { + fn drop(&mut self) { + // in update mode, we write on drop + if self.mode == ArchiveMode::Update { + todo!() + } + } } impl ZipArchive { @@ -515,15 +562,8 @@ impl ZipArchive { pub fn get_entry_by_hash(&self, hash: &u64) -> Option<&ZipEntry> { self.entries.get(hash) } - - // private methods - fn get_file_segment(&self, i: usize) -> Option<&FileSegment> { - self.internal_archive.index.file_segments.get(i) - } } -type EntryExtractInfo = (String, FileSegment, Vec); - impl ZipArchive where R: Read + Seek, @@ -540,7 +580,7 @@ where overwrite_files: bool, hash_map: &HashMap, ) -> Result<()> { - let Some(info) = self.get_extract_info(&entry, &hash_map) else { + let Some(info) = entry.get_resolved_name(&hash_map) else { return Err(io::Error::new( io::ErrorKind::InvalidData, "Could not get entry info from archive.", @@ -548,7 +588,7 @@ where }; // name or hash is a relative path - let outfile = destination_directory_name.as_ref().join(info.0); + let outfile = destination_directory_name.as_ref().join(info); create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; // extract to stream @@ -563,7 +603,7 @@ where }; let writer = BufWriter::new(&mut fs); - self.extract_segments(info.1, info.2, writer)?; + self.extract_segments(&entry, writer)?; Ok(()) } @@ -623,22 +663,8 @@ where } /// Returns an open read stream to an entry of this [`ZipArchive`]. - pub fn open_entry( - &mut self, - entry: ZipEntry, - hash_map: &HashMap, - writer: W, - ) -> Result<()> { - let Some(info) = self.get_extract_info(&entry, &hash_map) else { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Could not get entry info from archive.", - )); - }; - - let segment = info.1; - let buffers = info.2; - self.extract_segments(segment, buffers, writer)?; + pub fn open_entry(&mut self, entry: ZipEntry, writer: W) -> Result<()> { + self.extract_segments(&entry, writer)?; Ok(()) } @@ -691,12 +717,10 @@ where /// # Errors /// /// This function will return an error if io fails - fn extract_segments( - &mut self, - segment: FileSegment, - buffers: Vec, - mut writer: W, - ) -> Result<()> { + fn extract_segments(&mut self, entry: &ZipEntry, mut writer: W) -> Result<()> { + let segment = entry.segment; + let buffers = entry.buffers.clone(); + if segment.size == segment.z_size { // just copy self.reader_mut().seek(SeekFrom::Start(segment.offset))?; @@ -716,49 +740,16 @@ where Ok(()) } - /// Gets offset info from the underlying archive for a given entry. - fn get_extract_info( - &self, - file_entry: &ZipEntry, - hash_map: &HashMap, - ) -> Option<(String, FileSegment, Vec)> { - // get filename - let resolved = if let Some(name) = &file_entry.name { - name.to_owned() - } else { - let mut name_or_hash: String = format!("{}.bin", file_entry.hash); - // check vanilla hashes - if let Some(name) = hash_map.get(&file_entry.hash) { - name_or_hash = name.to_owned(); - } - name_or_hash - }; - - let start_index = file_entry.segments_start(); - let next_index = file_entry.segments_end(); - let mut info: Option = None; - if let Some(segment) = self.get_file_segment(start_index as usize) { - let mut buffers: Vec = vec![]; - for i in start_index + 1..next_index { - if let Some(segment) = self.get_file_segment(i as usize) { - buffers.push(*segment); - } - } - - info = Some((resolved, *segment, buffers)); - } - info - } - /// Opens an archive, needs to be read-only fn from_reader_consume(mut reader: R, mode: ArchiveMode) -> Result> { // checks if mode == ArchiveMode::Create { return Ok(ZipArchive:: { stream: reader, - internal_archive: Archive::default(), mode, + dirty: true, entries: HashMap::default(), + dependencies: Vec::default(), }); } @@ -785,34 +776,122 @@ where reader.seek(io::SeekFrom::Start(header.index_position))?; let index = Index::from_reader(&mut reader)?; + // read tables + let mut file_entries: HashMap = HashMap::default(); + for _i in 0..index.file_entry_count { + let entry = FileEntry::from_reader(&mut reader)?; + file_entries.insert(entry.name_hash_64, entry); + } + + let mut file_segments = Vec::default(); + for _i in 0..index.file_segment_count { + file_segments.push(FileSegment::from_reader(&mut reader)?); + } + + // dependencies can't be connected to individual files anymore + let mut dependencies = Vec::default(); + for _i in 0..index.resource_dependency_count { + dependencies.push(Dependency::from_reader(&mut reader)?); + } + // construct wrapper - let internal_archive = Archive { header, index }; - let mut entries: HashMap = HashMap::default(); - for (hash, entry) in internal_archive.index.file_entries.iter() { + let mut entries = HashMap::default(); + for (hash, entry) in file_entries.iter() { let resolved = if let Some(name) = file_names.get(hash) { Some(name.to_owned()) } else { None }; - let zip_entry = ZipEntry { - hash: *hash, - name: resolved, - entry: *entry, - }; - entries.insert(*hash, zip_entry); + let start_index = entry.segments_start; + let next_index = entry.segments_end; + if let Some(segment) = file_segments.get(start_index as usize) { + let mut buffers: Vec = vec![]; + for i in start_index + 1..next_index { + if let Some(buffer) = file_segments.get(i as usize) { + buffers.push(*buffer); + } + } + + let zip_entry = ZipEntry { + hash: *hash, + name: resolved, + entry: *entry, + segment: *segment, + buffers, + }; + entries.insert(*hash, zip_entry); + } } - Ok(ZipArchive:: { + let archive = ZipArchive:: { stream: reader, - internal_archive, mode, entries, - }) + dependencies, + dirty: false, + }; + Ok(archive) } } impl ZipArchive { + fn write(&mut self) { + todo!() + } + + fn write_index(&mut self, writer: &mut W) -> Result<()> { + let file_entry_count = self.entries.len() as u32; + let buffer_counts = self.entries.iter().map(|e| e.1.buffers.len() + 1); + let file_segment_count = buffer_counts.sum::() as u32; + let resource_dependency_count = self.dependencies.len() as u32; + + // todo write table to buffer + let mut buffer: Vec = Vec::new(); + //let mut table_writer = Cursor::new(buffer); + buffer.write_u32::(file_entry_count)?; + buffer.write_u32::(file_segment_count)?; + buffer.write_u32::(resource_dependency_count)?; + let mut entries = self.entries.values().collect::>(); + entries.sort_by_key(|e| e.hash); + // write entries + let mut segments = Vec::default(); + for entry in entries { + entry.entry.write(&mut buffer)?; + // collect offsets + segments.push(entry.segment); + for buffer in &entry.buffers { + segments.push(buffer.clone()); + } + } + // write segments + for segment in segments { + segment.write(&mut buffer)?; + } + + // write dependencies + for dependency in &self.dependencies { + dependency.write(&mut buffer)?; + } + + // write to out stream + let crc = crc64::crc64(0, buffer.as_slice()); + let index = Index { + file_table_offset: 8, + file_table_size: buffer.len() as u32 + 8, + crc, + file_entry_count, + file_segment_count, + resource_dependency_count, + }; + writer.write_u32::(index.file_table_offset)?; + writer.write_u32::(index.file_table_size)?; + writer.write_u64::(index.crc)?; + writer.write_all(buffer.as_slice())?; + + Ok(()) + } + /// Compresses and adds a file to the archive. /// /// # Errors @@ -823,22 +902,38 @@ impl ZipArchive { _file_path: P, _compression_level: CompressionLevel, ) -> Result { - if self.mode == ArchiveMode::Read { + // can only add entries in update mode + if self.mode != ArchiveMode::Update { return Err(io::Error::new( io::ErrorKind::InvalidData, "Archive is in read-only mode.", )); } + // todo write? + + // update offsets? + todo!() } /// Deletes an entry from the archive - pub fn delete_entry(&mut self, hash: &u64) -> Option { - let removed = self.entries.remove(hash); + pub fn delete_entry(&mut self, hash: &u64) -> Result { + // can only delete entries in update mode + if self.mode != ArchiveMode::Update { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Archive is in read-only mode.", + )); + } + + // update internally + let _removed = self.entries.remove(hash); + + todo!() // todo write? update offsets? - removed + //removed } } @@ -851,16 +946,25 @@ pub struct ZipEntry { /// wrapped internal struct entry: FileEntry, + segment: FileSegment, + buffers: Vec, } impl ZipEntry { - // getters - fn segments_start(&self) -> u32 { - self.entry.segments_start - } + fn get_resolved_name(&self, hash_map: &HashMap) -> Option { + // get filename + let resolved = if let Some(name) = &self.name { + name.to_owned() + } else { + let mut name_or_hash: String = format!("{}.bin", self.hash); + // check vanilla hashes + if let Some(name) = hash_map.get(&self.hash) { + name_or_hash = name.to_owned(); + } + name_or_hash + }; - fn segments_end(&self) -> u32 { - self.entry.segments_end + Some(resolved) } } @@ -868,12 +972,6 @@ impl ZipEntry { // INTERNAL ///////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, Default)] -struct Archive { - header: Header, - index: Index, -} - #[derive(Debug, Clone, Copy)] struct Header { magic: u32, @@ -932,63 +1030,29 @@ impl Header { } } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] struct Index { + /// Offset from the beginning of this struct, should be 8 file_table_offset: u32, + /// byte size of the table file_table_size: u32, crc: u64, file_entry_count: u32, file_segment_count: u32, resource_dependency_count: u32, - - // not serialized - file_entries: HashMap, - file_segments: Vec, - dependencies: Vec, -} -impl Index { - fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.file_table_offset)?; - writer.write_u32::(self.file_table_size)?; - writer.write_u64::(self.crc)?; - writer.write_u32::(self.file_entry_count)?; - writer.write_u32::(self.file_segment_count)?; - writer.write_u32::(self.resource_dependency_count)?; - - Ok(()) - } } +impl Index {} impl FromReader for Index { fn from_reader(cursor: &mut R) -> io::Result { - let mut index = Index { + let index = Index { file_table_offset: cursor.read_u32::()?, file_table_size: cursor.read_u32::()?, crc: cursor.read_u64::()?, file_entry_count: cursor.read_u32::()?, file_segment_count: cursor.read_u32::()?, resource_dependency_count: cursor.read_u32::()?, - - file_entries: HashMap::default(), - file_segments: vec![], - dependencies: vec![], }; - // read tables - for _i in 0..index.file_entry_count { - let entry = FileEntry::from_reader(cursor)?; - index.file_entries.insert(entry.name_hash_64, entry); - } - - for _i in 0..index.file_segment_count { - index.file_segments.push(FileSegment::from_reader(cursor)?); - } - - for _i in 0..index.resource_dependency_count { - index.dependencies.push(Dependency::from_reader(cursor)?); - } - - // ignore the rest of the archive - Ok(index) } } @@ -1000,6 +1064,15 @@ struct FileSegment { size: u32, } +impl FileSegment { + fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.offset)?; + writer.write_u32::(self.z_size)?; + writer.write_u32::(self.size)?; + Ok(()) + } +} + impl FromReader for FileSegment { fn from_reader(reader: &mut R) -> io::Result { Ok(FileSegment { @@ -1009,7 +1082,6 @@ impl FromReader for FileSegment { }) } } -#[allow(dead_code)] #[derive(Debug, Clone, Copy)] struct FileEntry { name_hash_64: u64, @@ -1021,7 +1093,20 @@ struct FileEntry { resource_dependencies_end: u32, sha1_hash: [u8; 20], } -#[warn(dead_code)] + +impl FileEntry { + fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.name_hash_64)?; + writer.write_u64::(self.timestamp)?; + writer.write_u32::(self.num_inline_buffer_segments)?; + writer.write_u32::(self.segments_start)?; + writer.write_u32::(self.segments_end)?; + writer.write_u32::(self.resource_dependencies_start)?; + writer.write_u32::(self.resource_dependencies_end)?; + writer.write_all(self.sha1_hash.as_slice())?; + Ok(()) + } +} impl FromReader for FileEntry { fn from_reader(reader: &mut R) -> io::Result { @@ -1047,6 +1132,13 @@ impl FromReader for FileEntry { struct Dependency { hash: u64, } + +impl Dependency { + fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.hash)?; + Ok(()) + } +} #[warn(dead_code)] impl FromReader for Dependency { @@ -1067,7 +1159,7 @@ impl LxrsFooter { const MAGIC: u32 = 0x4C585253; const VERSION: u32 = 1; - fn serialize(&self, writer: &mut W) -> Result<()> { + fn write(&self, writer: &mut W) -> Result<()> { writer.write_u32::(self.files.len() as u32)?; writer.write_u32::(LxrsFooter::VERSION)?; From c2f2f4d2ad1f1856dd63d2e438bd518dd772c2b0 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Tue, 26 Dec 2023 22:53:45 +0100 Subject: [PATCH 10/14] refactor --- src/archive/dependency.rs | 31 ++ src/archive/file_entry.rs | 84 ++++++ src/archive/file_segment.rs | 51 ++++ src/archive/header.rs | 86 ++++++ src/archive/index.rs | 46 +++ src/archive/lxrs.rs | 101 +++++++ src/{archive.rs => archive/mod.rs} | 466 ++++++----------------------- src/io.rs | 2 +- 8 files changed, 491 insertions(+), 376 deletions(-) create mode 100644 src/archive/dependency.rs create mode 100644 src/archive/file_entry.rs create mode 100644 src/archive/file_segment.rs create mode 100644 src/archive/header.rs create mode 100644 src/archive/index.rs create mode 100644 src/archive/lxrs.rs rename src/{archive.rs => archive/mod.rs} (68%) diff --git a/src/archive/dependency.rs b/src/archive/dependency.rs new file mode 100644 index 0000000..e41c51e --- /dev/null +++ b/src/archive/dependency.rs @@ -0,0 +1,31 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy)] +pub(crate) struct Dependency { + hash: u64, +} + +impl Dependency { + pub(crate) fn new(hash: u64) -> Self { + Self { hash } + } + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.hash)?; + Ok(()) + } +} +#[warn(dead_code)] + +impl FromReader for Dependency { + fn from_reader(reader: &mut R) -> Result { + Ok(Dependency { + hash: reader.read_u64::()?, + }) + } +} diff --git a/src/archive/file_entry.rs b/src/archive/file_entry.rs new file mode 100644 index 0000000..0506f4a --- /dev/null +++ b/src/archive/file_entry.rs @@ -0,0 +1,84 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct FileEntry { + name_hash_64: u64, + timestamp: u64, //SystemTime, + num_inline_buffer_segments: u32, + segments_start: u32, + segments_end: u32, + resource_dependencies_start: u32, + resource_dependencies_end: u32, + sha1_hash: [u8; 20], +} + +impl FileEntry { + pub(crate) fn new( + name_hash_64: u64, + timestamp: u64, + num_inline_buffer_segments: u32, + segments_start: u32, + segments_end: u32, + resource_dependencies_start: u32, + resource_dependencies_end: u32, + sha1_hash: [u8; 20], + ) -> Self { + Self { + name_hash_64, + timestamp, + num_inline_buffer_segments, + segments_start, + segments_end, + resource_dependencies_start, + resource_dependencies_end, + sha1_hash, + } + } + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.name_hash_64)?; + writer.write_u64::(self.timestamp)?; + writer.write_u32::(self.num_inline_buffer_segments)?; + writer.write_u32::(self.segments_start)?; + writer.write_u32::(self.segments_end)?; + writer.write_u32::(self.resource_dependencies_start)?; + writer.write_u32::(self.resource_dependencies_end)?; + writer.write_all(self.sha1_hash.as_slice())?; + Ok(()) + } + + pub(crate) fn name_hash_64(&self) -> u64 { + self.name_hash_64 + } + + pub(crate) fn segments_start(&self) -> u32 { + self.segments_start + } + + pub(crate) fn segments_end(&self) -> u32 { + self.segments_end + } +} + +impl FromReader for FileEntry { + fn from_reader(reader: &mut R) -> Result { + let mut entry = FileEntry { + name_hash_64: reader.read_u64::()?, + timestamp: reader.read_u64::()?, + num_inline_buffer_segments: reader.read_u32::()?, + segments_start: reader.read_u32::()?, + segments_end: reader.read_u32::()?, + resource_dependencies_start: reader.read_u32::()?, + resource_dependencies_end: reader.read_u32::()?, + sha1_hash: [0; 20], + }; + + reader.read_exact(&mut entry.sha1_hash[..])?; + + Ok(entry) + } +} diff --git a/src/archive/file_segment.rs b/src/archive/file_segment.rs new file mode 100644 index 0000000..e065a74 --- /dev/null +++ b/src/archive/file_segment.rs @@ -0,0 +1,51 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct FileSegment { + offset: u64, + z_size: u32, + size: u32, +} + +impl FileSegment { + pub(crate) fn new(offset: u64, z_size: u32, size: u32) -> Self { + Self { + offset, + z_size, + size, + } + } + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.offset)?; + writer.write_u32::(self.z_size)?; + writer.write_u32::(self.size)?; + Ok(()) + } + + pub(crate) fn offset(&self) -> u64 { + self.offset + } + + pub(crate) fn z_size(&self) -> u32 { + self.z_size + } + + pub(crate) fn size(&self) -> u32 { + self.size + } +} + +impl FromReader for FileSegment { + fn from_reader(reader: &mut R) -> Result { + Ok(FileSegment { + offset: reader.read_u64::()?, + z_size: reader.read_u32::()?, + size: reader.read_u32::()?, + }) + } +} diff --git a/src/archive/header.rs b/src/archive/header.rs new file mode 100644 index 0000000..33d37f3 --- /dev/null +++ b/src/archive/header.rs @@ -0,0 +1,86 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Header { + magic: u32, + version: u32, + index_position: u64, + index_size: u32, + debug_position: u64, + debug_size: u32, + filesize: u64, +} + +impl Header { + pub(crate) fn new( + index_position: u64, + index_size: u32, + debug_position: u64, + debug_size: u32, + filesize: u64, + ) -> Self { + Self { + magic: Header::HEADER_MAGIC, + version: Header::HEADER_VERSION, + index_position, + index_size, + debug_position, + debug_size, + filesize, + } + } + + pub(crate) const HEADER_MAGIC: u32 = 1380009042; + pub(crate) const HEADER_VERSION: u32 = 12; + pub(crate) const HEADER_SIZE: usize = 40; + pub(crate) const HEADER_EXTENDED_SIZE: u64 = 0xAC; + + pub(crate) fn index_position(&self) -> u64 { + self.index_position + } +} + +// impl Default for Header { +// fn default() -> Self { +// Self { +// magic: 1380009042, +// version: 12, +// index_position: Default::default(), +// index_size: Default::default(), +// debug_position: Default::default(), +// debug_size: Default::default(), +// filesize: Default::default(), +// } +// } +// } + +impl FromReader for Header { + fn from_reader(reader: &mut R) -> Result { + Ok(Header { + magic: reader.read_u32::()?, + version: reader.read_u32::()?, + index_position: reader.read_u64::()?, + index_size: reader.read_u32::()?, + debug_position: reader.read_u64::()?, + debug_size: reader.read_u32::()?, + filesize: reader.read_u64::()?, + }) + } +} +impl Header { + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.magic)?; + writer.write_u32::(self.version)?; + writer.write_u64::(self.index_position)?; + writer.write_u32::(self.index_size)?; + writer.write_u64::(self.debug_position)?; + writer.write_u32::(self.debug_size)?; + writer.write_u64::(self.filesize)?; + + Ok(()) + } +} diff --git a/src/archive/index.rs b/src/archive/index.rs new file mode 100644 index 0000000..a4e8796 --- /dev/null +++ b/src/archive/index.rs @@ -0,0 +1,46 @@ +use std::io::{Read, Result}; + +use byteorder::{LittleEndian, ReadBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone)] +pub(crate) struct Index { + /// Offset from the beginning of this struct, should be 8 + file_table_offset: u32, + /// byte size of the table + file_table_size: u32, + crc: u64, + file_entry_count: u32, + file_segment_count: u32, + resource_dependency_count: u32, +} + +impl Index { + pub(crate) fn file_entry_count(&self) -> u32 { + self.file_entry_count + } + + pub(crate) fn file_segment_count(&self) -> u32 { + self.file_segment_count + } + + pub(crate) fn resource_dependency_count(&self) -> u32 { + self.resource_dependency_count + } +} + +impl FromReader for Index { + fn from_reader(cursor: &mut R) -> Result { + let index = Index { + file_table_offset: cursor.read_u32::()?, + file_table_size: cursor.read_u32::()?, + crc: cursor.read_u64::()?, + file_entry_count: cursor.read_u32::()?, + file_segment_count: cursor.read_u32::()?, + resource_dependency_count: cursor.read_u32::()?, + }; + + Ok(index) + } +} diff --git a/src/archive/lxrs.rs b/src/archive/lxrs.rs new file mode 100644 index 0000000..1ab2636 --- /dev/null +++ b/src/archive/lxrs.rs @@ -0,0 +1,101 @@ +use std::{ + cmp::Ordering, + io::{Cursor, Error, ErrorKind, Read, Result, Write}, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::{io::*, kraken::*}; + +#[derive(Debug, Clone)] +pub(crate) struct LxrsFooter { + files: Vec, +} + +impl LxrsFooter { + pub(crate) fn new(files: Vec) -> Self { + Self { files } + } + + //const MINLEN: u32 = 20; + const MAGIC: u32 = 0x4C585253; + const VERSION: u32 = 1; + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.files.len() as u32)?; + writer.write_u32::(LxrsFooter::VERSION)?; + + // write strings to buffer + let mut buffer: Vec = Vec::new(); + for f in &self.files { + write_null_terminated_string(&mut buffer, f.to_owned())?; + } + + // compress + let size = buffer.len(); + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress(&buffer, &mut compressed_buffer, CompressionLevel::Normal); + assert!((zsize as u32) <= size as u32); + compressed_buffer.resize(zsize as usize, 0); + + // write to writer + writer.write_all(&compressed_buffer)?; + + Ok(()) + } + + pub(crate) fn files(&self) -> &[String] { + self.files.as_ref() + } +} +impl FromReader for LxrsFooter { + fn from_reader(reader: &mut R) -> Result { + let magic = reader.read_u32::()?; + if magic != LxrsFooter::MAGIC { + return Err(Error::new(ErrorKind::Other, "invalid magic")); + } + let _version = reader.read_u32::()?; + let size = reader.read_u32::()?; + let zsize = reader.read_u32::()?; + let count = reader.read_i32::()?; + + let mut files: Vec = vec![]; + match size.cmp(&zsize) { + Ordering::Greater => { + // buffer is compressed + let mut compressed_buffer = vec![0; zsize as usize]; + reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); + + // read from buffer + let mut inner_cursor = Cursor::new(&output_buffer); + for _i in 0..count { + // read NullTerminatedString + if let Ok(string) = read_null_terminated_string(&mut inner_cursor) { + files.push(string); + } + } + } + Ordering::Less => { + // error + return Err(Error::new(ErrorKind::Other, "invalid buffer")); + } + Ordering::Equal => { + // no compression + for _i in 0..count { + // read NullTerminatedString + if let Ok(string) = read_null_terminated_string(reader) { + files.push(string); + } + } + } + } + + let footer = LxrsFooter { files }; + + Ok(footer) + } +} diff --git a/src/archive.rs b/src/archive/mod.rs similarity index 68% rename from src/archive.rs rename to src/archive/mod.rs index 971b5fd..2ffb154 100644 --- a/src/archive.rs +++ b/src/archive/mod.rs @@ -4,7 +4,6 @@ use std::{ borrow::BorrowMut, - cmp::Ordering, collections::{HashMap, HashSet}, fs::{create_dir_all, File}, io::{self, BufWriter, Cursor, Error, ErrorKind, Read, Result, Seek, SeekFrom, Write}, @@ -15,10 +14,18 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use strum::IntoEnumIterator; use walkdir::WalkDir; -use crate::fnv1a64_hash_string; -use crate::io::*; use crate::kraken::*; use crate::{cr2w::*, *}; +use crate::{fnv1a64_hash_string, io::FromReader}; + +use self::{dependency::*, file_entry::*, file_segment::*, header::*, index::*, lxrs::*}; + +mod dependency; +mod file_entry; +mod file_segment; +mod header; +mod index; +mod lxrs; ///////////////////////////////////////////////////////////////////////////////////////// // ARCHIVE_FILE @@ -218,16 +225,13 @@ where let mut archive_writer = BufWriter::new(out_stream); // write empty header - let header = Header::default(); - header.serialize(&mut archive_writer)?; + archive_writer.write_all(&[0u8; Header::HEADER_SIZE])?; //write empty header archive_writer.write_all(&[0u8; 132])?; // padding // write custom header let mut custom_data_length = 0; if !custom_paths.is_empty() { - let wfooter = LxrsFooter { - files: custom_paths, - }; + let wfooter = LxrsFooter::new(custom_paths); wfooter.write(&mut archive_writer)?; custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; } @@ -280,11 +284,7 @@ where archive_writer.write_all(&compressed_buffer)?; // add metadata to archive - segment = Some(FileSegment { - offset: archive_offset, - size, - z_size: zsize as u32, - }); + segment = Some(FileSegment::new(archive_offset, zsize as u32, size)); file_segments_cnt += 1; // write buffers (bytes after the main file) @@ -298,11 +298,7 @@ where archive_writer.write_all(buffer.as_slice())?; // add metadata to archive - buffers.push(FileSegment { - offset: boffset, - size: bsize, - z_size: bzsize, - }); + buffers.push(FileSegment::new(boffset, bzsize, bsize)); file_segments_cnt += 1; } @@ -353,11 +349,7 @@ where } // add metadata to archive - segment = Some(FileSegment { - offset, - size, - z_size: final_zsize, - }); + segment = Some(FileSegment::new(offset, final_zsize, size)); file_segments_cnt += 1; } } @@ -365,16 +357,16 @@ where // update archive metadata let sha1_hash = sha1_hash_file(&file_buffer); - let entry = FileEntry { - name_hash_64: hash, - timestamp: 0, // TODO proper timestamps - num_inline_buffer_segments: flags as u32, - segments_start: firstoffsetidx as u32, - segments_end: lastoffsetidx as u32, - resource_dependencies_start: firstimportidx as u32, - resource_dependencies_end: lastimportidx as u32, + let entry = FileEntry::new( + hash, + 0, + flags as u32, + firstoffsetidx as u32, + lastoffsetidx as u32, + firstimportidx as u32, + lastimportidx as u32, sha1_hash, - }; + ); if let Some(segment) = segment { let wrapped_entry = ZipEntry { @@ -389,37 +381,27 @@ where } // write footers - let archive = ZipArchive { - stream: todo!(), - mode: ArchiveMode::Create, - entries, - dirty: false, - dependencies: imports_hash_set - .iter() - .map(|e| Dependency { - hash: fnv1a64_hash_string(e), - }) - .collect::>(), - }; + let dependencies = imports_hash_set + .iter() + .map(|e| Dependency::new(fnv1a64_hash_string(e))) + .collect::>(); // padding pad_until_page(&mut archive_writer)?; // write tables let tableoffset = archive_writer.stream_position()?; - archive.write_index(&mut archive_writer)?; + write_index(&mut archive_writer, &entries, &dependencies)?; let tablesize = archive_writer.stream_position()? - tableoffset; // padding pad_until_page(&mut archive_writer)?; - let filesize = archive_writer.stream_position()?; // write the header again - header.index_position = tableoffset; - header.index_size = tablesize as u32; - header.filesize = filesize; + let filesize = archive_writer.stream_position()?; + let header = Header::new(tableoffset, tablesize as u32, 0, 0, filesize); archive_writer.seek(SeekFrom::Start(0))?; - header.serialize(&mut archive_writer)?; + header.write(&mut archive_writer)?; archive_writer.write_u32::(custom_data_length as u32)?; Ok(()) @@ -459,17 +441,17 @@ fn decompress_segment( segment: &FileSegment, file_writer: &mut W, ) -> Result<()> { - archive_reader.seek(SeekFrom::Start(segment.offset))?; + archive_reader.seek(SeekFrom::Start(segment.offset()))?; let magic = archive_reader.read_u32::()?; if magic == kraken::MAGIC { // read metadata - let mut size = segment.size; + let mut size = segment.size(); let size_in_header = archive_reader.read_u32::()?; if size_in_header != size { size = size_in_header; } - let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; + let mut compressed_buffer = vec![0; segment.z_size() as usize - 8]; archive_reader.read_exact(&mut compressed_buffer[..])?; let mut output_buffer = vec![]; let result = decompress(compressed_buffer, &mut output_buffer, size as usize); @@ -479,8 +461,8 @@ fn decompress_segment( file_writer.write_all(&output_buffer)?; } else { // incorrect data, fall back to direct copy - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; + archive_reader.seek(SeekFrom::Start(segment.offset()))?; + let mut buffer = vec![0; segment.z_size() as usize]; archive_reader.read_exact(&mut buffer[..])?; file_writer.write_all(&buffer)?; }; @@ -721,18 +703,18 @@ where let segment = entry.segment; let buffers = entry.buffers.clone(); - if segment.size == segment.z_size { + if segment.size() == segment.z_size() { // just copy - self.reader_mut().seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; + self.reader_mut().seek(SeekFrom::Start(segment.offset()))?; + let mut buffer = vec![0; segment.z_size() as usize]; self.reader_mut().read_exact(&mut buffer[..])?; writer.write_all(&buffer)?; } else { decompress_segment(self.reader_mut(), &segment, &mut writer)?; } for segment in buffers { - self.reader_mut().seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; + self.reader_mut().seek(SeekFrom::Start(segment.offset()))?; + let mut buffer = vec![0; segment.z_size() as usize]; self.reader_mut().read_exact(&mut buffer[..])?; writer.write_all(&buffer)?; } @@ -763,9 +745,9 @@ where reader.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; if let Ok(footer) = LxrsFooter::from_reader(&mut reader) { // add files to hashmap - for f in footer.files { - let hash = fnv1a64_hash_string(&f); - file_names.insert(hash, f); + for f in footer.files() { + let hash = fnv1a64_hash_string(f); + file_names.insert(hash, f.to_owned()); } } } @@ -773,24 +755,24 @@ where // read index // move to offset Header.IndexPosition - reader.seek(io::SeekFrom::Start(header.index_position))?; + reader.seek(io::SeekFrom::Start(header.index_position()))?; let index = Index::from_reader(&mut reader)?; // read tables let mut file_entries: HashMap = HashMap::default(); - for _i in 0..index.file_entry_count { + for _i in 0..index.file_entry_count() { let entry = FileEntry::from_reader(&mut reader)?; - file_entries.insert(entry.name_hash_64, entry); + file_entries.insert(entry.name_hash_64(), entry); } let mut file_segments = Vec::default(); - for _i in 0..index.file_segment_count { + for _i in 0..index.file_segment_count() { file_segments.push(FileSegment::from_reader(&mut reader)?); } // dependencies can't be connected to individual files anymore let mut dependencies = Vec::default(); - for _i in 0..index.resource_dependency_count { + for _i in 0..index.resource_dependency_count() { dependencies.push(Dependency::from_reader(&mut reader)?); } @@ -803,8 +785,8 @@ where None }; - let start_index = entry.segments_start; - let next_index = entry.segments_end; + let start_index = entry.segments_start(); + let next_index = entry.segments_end(); if let Some(segment) = file_segments.get(start_index as usize) { let mut buffers: Vec = vec![]; for i in start_index + 1..next_index { @@ -840,58 +822,6 @@ impl ZipArchive { todo!() } - fn write_index(&mut self, writer: &mut W) -> Result<()> { - let file_entry_count = self.entries.len() as u32; - let buffer_counts = self.entries.iter().map(|e| e.1.buffers.len() + 1); - let file_segment_count = buffer_counts.sum::() as u32; - let resource_dependency_count = self.dependencies.len() as u32; - - // todo write table to buffer - let mut buffer: Vec = Vec::new(); - //let mut table_writer = Cursor::new(buffer); - buffer.write_u32::(file_entry_count)?; - buffer.write_u32::(file_segment_count)?; - buffer.write_u32::(resource_dependency_count)?; - let mut entries = self.entries.values().collect::>(); - entries.sort_by_key(|e| e.hash); - // write entries - let mut segments = Vec::default(); - for entry in entries { - entry.entry.write(&mut buffer)?; - // collect offsets - segments.push(entry.segment); - for buffer in &entry.buffers { - segments.push(buffer.clone()); - } - } - // write segments - for segment in segments { - segment.write(&mut buffer)?; - } - - // write dependencies - for dependency in &self.dependencies { - dependency.write(&mut buffer)?; - } - - // write to out stream - let crc = crc64::crc64(0, buffer.as_slice()); - let index = Index { - file_table_offset: 8, - file_table_size: buffer.len() as u32 + 8, - crc, - file_entry_count, - file_segment_count, - resource_dependency_count, - }; - writer.write_u32::(index.file_table_offset)?; - writer.write_u32::(index.file_table_size)?; - writer.write_u64::(index.crc)?; - writer.write_all(buffer.as_slice())?; - - Ok(()) - } - /// Compresses and adds a file to the archive. /// /// # Errors @@ -972,266 +902,52 @@ impl ZipEntry { // INTERNAL ///////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, Copy)] -struct Header { - magic: u32, - version: u32, - index_position: u64, - index_size: u32, - debug_position: u64, - debug_size: u32, - filesize: u64, -} - -impl Header { - //static HEADER_MAGIC: u32 = 1380009042; - //static HEADER_SIZE: i32 = 40; - const HEADER_EXTENDED_SIZE: u64 = 0xAC; -} - -impl Default for Header { - fn default() -> Self { - Self { - magic: 1380009042, - version: 12, - index_position: Default::default(), - index_size: Default::default(), - debug_position: Default::default(), - debug_size: Default::default(), - filesize: Default::default(), +fn write_index( + writer: &mut W, + entries: &HashMap, + dependencies: &[Dependency], +) -> Result<()> { + let file_entry_count = entries.len() as u32; + let buffer_counts = entries.iter().map(|e| e.1.buffers.len() + 1); + let file_segment_count = buffer_counts.sum::() as u32; + let resource_dependency_count = dependencies.len() as u32; + + // todo write table to buffer + let mut buffer: Vec = Vec::new(); + //let mut table_writer = Cursor::new(buffer); + buffer.write_u32::(file_entry_count)?; + buffer.write_u32::(file_segment_count)?; + buffer.write_u32::(resource_dependency_count)?; + let mut entries = entries.values().collect::>(); + entries.sort_by_key(|e| e.hash); + // write entries + let mut segments = Vec::default(); + for entry in entries { + entry.entry.write(&mut buffer)?; + // collect offsets + segments.push(entry.segment); + for buffer in &entry.buffers { + segments.push(buffer.clone()); } } -} - -impl FromReader for Header { - fn from_reader(reader: &mut R) -> Result { - Ok(Header { - magic: reader.read_u32::()?, - version: reader.read_u32::()?, - index_position: reader.read_u64::()?, - index_size: reader.read_u32::()?, - debug_position: reader.read_u64::()?, - debug_size: reader.read_u32::()?, - filesize: reader.read_u64::()?, - }) - } -} -impl Header { - fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.magic)?; - writer.write_u32::(self.version)?; - writer.write_u64::(self.index_position)?; - writer.write_u32::(self.index_size)?; - writer.write_u64::(self.debug_position)?; - writer.write_u32::(self.debug_size)?; - writer.write_u64::(self.filesize)?; - - Ok(()) - } -} - -#[derive(Debug, Clone)] -struct Index { - /// Offset from the beginning of this struct, should be 8 - file_table_offset: u32, - /// byte size of the table - file_table_size: u32, - crc: u64, - file_entry_count: u32, - file_segment_count: u32, - resource_dependency_count: u32, -} -impl Index {} -impl FromReader for Index { - fn from_reader(cursor: &mut R) -> io::Result { - let index = Index { - file_table_offset: cursor.read_u32::()?, - file_table_size: cursor.read_u32::()?, - crc: cursor.read_u64::()?, - file_entry_count: cursor.read_u32::()?, - file_segment_count: cursor.read_u32::()?, - resource_dependency_count: cursor.read_u32::()?, - }; - - Ok(index) - } -} - -#[derive(Debug, Clone, Copy)] -struct FileSegment { - offset: u64, - z_size: u32, - size: u32, -} - -impl FileSegment { - fn write(&self, writer: &mut W) -> Result<()> { - writer.write_u64::(self.offset)?; - writer.write_u32::(self.z_size)?; - writer.write_u32::(self.size)?; - Ok(()) - } -} - -impl FromReader for FileSegment { - fn from_reader(reader: &mut R) -> io::Result { - Ok(FileSegment { - offset: reader.read_u64::()?, - z_size: reader.read_u32::()?, - size: reader.read_u32::()?, - }) - } -} -#[derive(Debug, Clone, Copy)] -struct FileEntry { - name_hash_64: u64, - timestamp: u64, //SystemTime, - num_inline_buffer_segments: u32, - segments_start: u32, - segments_end: u32, - resource_dependencies_start: u32, - resource_dependencies_end: u32, - sha1_hash: [u8; 20], -} - -impl FileEntry { - fn write(&self, writer: &mut W) -> Result<()> { - writer.write_u64::(self.name_hash_64)?; - writer.write_u64::(self.timestamp)?; - writer.write_u32::(self.num_inline_buffer_segments)?; - writer.write_u32::(self.segments_start)?; - writer.write_u32::(self.segments_end)?; - writer.write_u32::(self.resource_dependencies_start)?; - writer.write_u32::(self.resource_dependencies_end)?; - writer.write_all(self.sha1_hash.as_slice())?; - Ok(()) - } -} - -impl FromReader for FileEntry { - fn from_reader(reader: &mut R) -> io::Result { - let mut entry = FileEntry { - name_hash_64: reader.read_u64::()?, - timestamp: reader.read_u64::()?, - num_inline_buffer_segments: reader.read_u32::()?, - segments_start: reader.read_u32::()?, - segments_end: reader.read_u32::()?, - resource_dependencies_start: reader.read_u32::()?, - resource_dependencies_end: reader.read_u32::()?, - sha1_hash: [0; 20], - }; - - reader.read_exact(&mut entry.sha1_hash[..])?; - - Ok(entry) - } -} - -#[allow(dead_code)] -#[derive(Debug, Clone, Copy)] -struct Dependency { - hash: u64, -} - -impl Dependency { - fn write(&self, writer: &mut W) -> Result<()> { - writer.write_u64::(self.hash)?; - Ok(()) - } -} -#[warn(dead_code)] - -impl FromReader for Dependency { - fn from_reader(reader: &mut R) -> io::Result { - Ok(Dependency { - hash: reader.read_u64::()?, - }) + // write segments + for segment in segments { + segment.write(&mut buffer)?; } -} - -#[derive(Debug, Clone)] -struct LxrsFooter { - files: Vec, -} - -impl LxrsFooter { - //const MINLEN: u32 = 20; - const MAGIC: u32 = 0x4C585253; - const VERSION: u32 = 1; - fn write(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.files.len() as u32)?; - writer.write_u32::(LxrsFooter::VERSION)?; - - // write strings to buffer - let mut buffer: Vec = Vec::new(); - for f in &self.files { - write_null_terminated_string(&mut buffer, f.to_owned())?; - } - - // compress - let size = buffer.len(); - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress(&buffer, &mut compressed_buffer, CompressionLevel::Normal); - assert!((zsize as u32) <= size as u32); - compressed_buffer.resize(zsize as usize, 0); - - // write to writer - writer.write_all(&compressed_buffer)?; - - Ok(()) + // write dependencies + for dependency in dependencies { + dependency.write(&mut buffer)?; } -} -impl FromReader for LxrsFooter { - fn from_reader(reader: &mut R) -> io::Result { - let magic = reader.read_u32::()?; - if magic != LxrsFooter::MAGIC { - return Err(io::Error::new(io::ErrorKind::Other, "invalid magic")); - } - let _version = reader.read_u32::()?; - let size = reader.read_u32::()?; - let zsize = reader.read_u32::()?; - let count = reader.read_i32::()?; - - let mut files: Vec = vec![]; - match size.cmp(&zsize) { - Ordering::Greater => { - // buffer is compressed - let mut compressed_buffer = vec![0; zsize as usize]; - reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); - - // read from buffer - let mut inner_cursor = io::Cursor::new(&output_buffer); - for _i in 0..count { - // read NullTerminatedString - if let Ok(string) = read_null_terminated_string(&mut inner_cursor) { - files.push(string); - } - } - } - Ordering::Less => { - // error - return Err(io::Error::new(io::ErrorKind::Other, "invalid buffer")); - } - Ordering::Equal => { - // no compression - for _i in 0..count { - // read NullTerminatedString - if let Ok(string) = read_null_terminated_string(reader) { - files.push(string); - } - } - } - } - let footer = LxrsFooter { files }; + // write to out stream + let crc = crc64::crc64(0, buffer.as_slice()); + writer.write_u32::(8)?; + writer.write_u32::(buffer.len() as u32 + 8)?; + writer.write_u64::(crc)?; + writer.write_all(buffer.as_slice())?; - Ok(footer) - } + Ok(()) } ///////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/io.rs b/src/io.rs index 4bf5194..cacd754 100644 --- a/src/io.rs +++ b/src/io.rs @@ -5,7 +5,7 @@ use std::io::{self, Read, Write}; use byteorder::WriteBytesExt; -pub trait FromReader: Sized { +pub(crate) trait FromReader: Sized { fn from_reader(reader: &mut R) -> io::Result; } From be74b037775b4e4d08eeff48b57ac54a1bc2c879 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Fri, 12 Jan 2024 18:48:57 +0100 Subject: [PATCH 11/14] fix archive dependecy list --- src/archive/index.rs | 3 +++ src/archive/mod.rs | 21 ++++++--------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/archive/index.rs b/src/archive/index.rs index a4e8796..0b436c3 100644 --- a/src/archive/index.rs +++ b/src/archive/index.rs @@ -4,6 +4,7 @@ use byteorder::{LittleEndian, ReadBytesExt}; use crate::io::FromReader; +#[allow(dead_code)] #[derive(Debug, Clone)] pub(crate) struct Index { /// Offset from the beginning of this struct, should be 8 @@ -16,6 +17,8 @@ pub(crate) struct Index { resource_dependency_count: u32, } +#[warn(dead_code)] + impl Index { pub(crate) fn file_entry_count(&self) -> u32 { self.file_entry_count diff --git a/src/archive/mod.rs b/src/archive/mod.rs index 2ffb154..0ae1920 100644 --- a/src/archive/mod.rs +++ b/src/archive/mod.rs @@ -239,7 +239,7 @@ where // write files let mut file_segments_cnt = 0; let mut entries = HashMap::default(); - let mut imports_hash_set: HashSet = HashSet::new(); + let imports_hash_set: HashSet = HashSet::new(); for (path, hash) in file_info { // read file @@ -303,11 +303,11 @@ where } //register imports - for import in info.imports.iter() { - // TODO fix flags - // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) - imports_hash_set.insert(import.depot_path.to_owned()); - } + // NOTE don't use a dependency list for mods + //for import in info.imports.iter() { + // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) + //imports_hash_set.insert(import.depot_path.to_owned()); + //} lastimportidx = imports_hash_set.len(); lastoffsetidx = file_segments_cnt; @@ -525,15 +525,6 @@ pub struct ZipArchive { dependencies: Vec, } -impl Drop for ZipArchive { - fn drop(&mut self) { - // in update mode, we write on drop - if self.mode == ArchiveMode::Update { - todo!() - } - } -} - impl ZipArchive { /// Get an entry in the archive by resource path. pub fn get_entry(&self, name: &str) -> Option<&ZipEntry> { From 29965665bf3e0ecc096e0c297393d0ef1b90b065 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Fri, 12 Jan 2024 19:26:10 +0100 Subject: [PATCH 12/14] start add entry --- src/archive/dependency.rs | 24 +++++------ src/archive/file_entry.rs | 8 ++++ src/archive/mod.rs | 89 +++++++++++++++++++-------------------- 3 files changed, 63 insertions(+), 58 deletions(-) diff --git a/src/archive/dependency.rs b/src/archive/dependency.rs index e41c51e..3ef29e0 100644 --- a/src/archive/dependency.rs +++ b/src/archive/dependency.rs @@ -1,25 +1,25 @@ -use std::io::{Read, Result, Write}; +use std::io::{Read, Result}; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use byteorder::{LittleEndian, ReadBytesExt}; use crate::io::FromReader; #[allow(dead_code)] #[derive(Debug, Clone, Copy)] -pub(crate) struct Dependency { +pub struct Dependency { hash: u64, } -impl Dependency { - pub(crate) fn new(hash: u64) -> Self { - Self { hash } - } +// impl Dependency { +// pub(crate) fn new(hash: u64) -> Self { +// Self { hash } +// } - pub(crate) fn write(&self, writer: &mut W) -> Result<()> { - writer.write_u64::(self.hash)?; - Ok(()) - } -} +// pub(crate) fn write(&self, writer: &mut W) -> Result<()> { +// writer.write_u64::(self.hash)?; +// Ok(()) +// } +// } #[warn(dead_code)] impl FromReader for Dependency { diff --git a/src/archive/file_entry.rs b/src/archive/file_entry.rs index 0506f4a..372765a 100644 --- a/src/archive/file_entry.rs +++ b/src/archive/file_entry.rs @@ -62,6 +62,14 @@ impl FileEntry { pub(crate) fn segments_end(&self) -> u32 { self.segments_end } + + pub(crate) fn set_segments_start(&mut self, segments_start: u32) { + self.segments_start = segments_start; + } + + pub(crate) fn set_segments_end(&mut self, segments_end: u32) { + self.segments_end = segments_end; + } } impl FromReader for FileEntry { diff --git a/src/archive/mod.rs b/src/archive/mod.rs index 0ae1920..cf44220 100644 --- a/src/archive/mod.rs +++ b/src/archive/mod.rs @@ -4,7 +4,7 @@ use std::{ borrow::BorrowMut, - collections::{HashMap, HashSet}, + collections::HashMap, fs::{create_dir_all, File}, io::{self, BufWriter, Cursor, Error, ErrorKind, Read, Result, Seek, SeekFrom, Write}, path::{Path, PathBuf}, @@ -237,23 +237,19 @@ where } // write files - let mut file_segments_cnt = 0; + //let imports_hash_set: HashSet = HashSet::new(); let mut entries = HashMap::default(); - let imports_hash_set: HashSet = HashSet::new(); - for (path, hash) in file_info { // read file let mut file = File::open(&path)?; let mut file_buffer = Vec::new(); file.read_to_end(&mut file_buffer)?; - let firstimportidx = imports_hash_set.len(); - let mut lastimportidx = imports_hash_set.len(); - let firstoffsetidx = file_segments_cnt; - let mut lastoffsetidx = 0; - let mut flags = 0; - let mut file_cursor = Cursor::new(&file_buffer); + + //let firstimportidx = imports_hash_set.len(); + //let mut lastimportidx = imports_hash_set.len(); + let mut flags = 0; let mut segment: Option = None; let mut buffers = vec![]; @@ -285,7 +281,6 @@ where // add metadata to archive segment = Some(FileSegment::new(archive_offset, zsize as u32, size)); - file_segments_cnt += 1; // write buffers (bytes after the main file) for buffer_info in info.buffers_table.iter() { @@ -299,7 +294,6 @@ where // add metadata to archive buffers.push(FileSegment::new(boffset, bzsize, bsize)); - file_segments_cnt += 1; } //register imports @@ -309,8 +303,8 @@ where //imports_hash_set.insert(import.depot_path.to_owned()); //} - lastimportidx = imports_hash_set.len(); - lastoffsetidx = file_segments_cnt; + //lastimportidx = imports_hash_set.len(); + flags = if !info.buffers_table.is_empty() { info.buffers_table.len() - 1 } else { @@ -350,7 +344,6 @@ where // add metadata to archive segment = Some(FileSegment::new(offset, final_zsize, size)); - file_segments_cnt += 1; } } @@ -361,10 +354,10 @@ where hash, 0, flags as u32, - firstoffsetidx as u32, - lastoffsetidx as u32, - firstimportidx as u32, - lastimportidx as u32, + 0, //firstoffsetidx as u32, + 0, //lastoffsetidx as u32, + 0, //firstimportidx as u32, + 0, //lastimportidx as u32, sha1_hash, ); @@ -380,18 +373,28 @@ where } } + // run through entries again and enumerate the segments + let mut file_segments_cnt = 0; + for (_hash, entry) in entries.iter_mut() { + let firstoffsetidx = file_segments_cnt; + file_segments_cnt += entry.buffers.len() + 1; + let lastoffsetidx = file_segments_cnt; + entry.entry.set_segments_start(firstoffsetidx as u32); + entry.entry.set_segments_end(lastoffsetidx as u32); + } + // write footers - let dependencies = imports_hash_set - .iter() - .map(|e| Dependency::new(fnv1a64_hash_string(e))) - .collect::>(); + // let dependencies = imports_hash_set + // .iter() + // .map(|e| Dependency::new(fnv1a64_hash_string(e))) + // .collect::>(); // padding pad_until_page(&mut archive_writer)?; // write tables let tableoffset = archive_writer.stream_position()?; - write_index(&mut archive_writer, &entries, &dependencies)?; + write_index(&mut archive_writer, &entries /*, &dependencies */)?; let tablesize = archive_writer.stream_position()? - tableoffset; // padding @@ -522,7 +525,7 @@ pub struct ZipArchive { dirty: bool, /// The files inside an archive entries: HashMap, - dependencies: Vec, + pub dependencies: Vec, } impl ZipArchive { @@ -819,7 +822,7 @@ impl ZipArchive { /// /// This function will return an error if compression or io fails, or if the mode is Read. pub fn create_entry>( - &self, + &mut self, _file_path: P, _compression_level: CompressionLevel, ) -> Result { @@ -831,30 +834,25 @@ impl ZipArchive { )); } - // todo write? + // write? - // update offsets? + // set dirty + self.dirty = true; todo!() } /// Deletes an entry from the archive - pub fn delete_entry(&mut self, hash: &u64) -> Result { + pub fn delete_entry(&mut self, hash: &u64) -> Option { // can only delete entries in update mode if self.mode != ArchiveMode::Update { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Archive is in read-only mode.", - )); + return None; } - // update internally - let _removed = self.entries.remove(hash); + // Set dirty + self.dirty = true; - todo!() - - // todo write? update offsets? - //removed + self.entries.remove(hash) } } @@ -896,16 +894,15 @@ impl ZipEntry { fn write_index( writer: &mut W, entries: &HashMap, - dependencies: &[Dependency], + //dependencies: &[Dependency], ) -> Result<()> { let file_entry_count = entries.len() as u32; let buffer_counts = entries.iter().map(|e| e.1.buffers.len() + 1); let file_segment_count = buffer_counts.sum::() as u32; - let resource_dependency_count = dependencies.len() as u32; + let resource_dependency_count = 0; //dependencies.len() as u32; - // todo write table to buffer + // write table to buffer let mut buffer: Vec = Vec::new(); - //let mut table_writer = Cursor::new(buffer); buffer.write_u32::(file_entry_count)?; buffer.write_u32::(file_segment_count)?; buffer.write_u32::(resource_dependency_count)?; @@ -927,9 +924,9 @@ fn write_index( } // write dependencies - for dependency in dependencies { - dependency.write(&mut buffer)?; - } + // for dependency in dependencies { + // dependency.write(&mut buffer)?; + // } // write to out stream let crc = crc64::crc64(0, buffer.as_slice()); From 350d7b4365d5f6feb5073556a9a54898b9c27b53 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Fri, 12 Jan 2024 19:31:09 +0100 Subject: [PATCH 13/14] before refactor --- src/archive/mod.rs | 82 ++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/src/archive/mod.rs b/src/archive/mod.rs index cf44220..9e7899c 100644 --- a/src/archive/mod.rs +++ b/src/archive/mod.rs @@ -250,7 +250,7 @@ where //let firstimportidx = imports_hash_set.len(); //let mut lastimportidx = imports_hash_set.len(); let mut flags = 0; - let mut segment: Option = None; + let mut segment: FileSegment; let mut buffers = vec![]; if let Ok(info) = read_cr2w_header(&mut file_cursor) { @@ -280,7 +280,7 @@ where archive_writer.write_all(&compressed_buffer)?; // add metadata to archive - segment = Some(FileSegment::new(archive_offset, zsize as u32, size)); + segment = FileSegment::new(archive_offset, zsize as u32, size); // write buffers (bytes after the main file) for buffer_info in info.buffers_table.iter() { @@ -313,38 +313,37 @@ where } else { // write non-cr2w file file_cursor.seek(SeekFrom::Start(0))?; - if let Some(os_ext) = path.extension() { - let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); - if get_aligned_file_extensions().contains(&ext) { - pad_until_page(&mut archive_writer)?; - } - - let offset = archive_writer.stream_position()?; - let size = file_buffer.len() as u32; - let final_zsize; - if get_uncompressed_file_extensions().contains(&ext) { - // direct copy - archive_writer.write_all(&file_buffer)?; - final_zsize = size; - } else { - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &file_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - final_zsize = zsize as u32; - // write - archive_writer.write_all(&compressed_buffer)?; - } + let os_ext = path.extension().unwrap(); + let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); + if get_aligned_file_extensions().contains(&ext) { + pad_until_page(&mut archive_writer)?; + } - // add metadata to archive - segment = Some(FileSegment::new(offset, final_zsize, size)); + let offset = archive_writer.stream_position()?; + let size = file_buffer.len() as u32; + let final_zsize; + if get_uncompressed_file_extensions().contains(&ext) { + // direct copy + archive_writer.write_all(&file_buffer)?; + final_zsize = size; + } else { + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &file_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + final_zsize = zsize as u32; + // write + archive_writer.write_all(&compressed_buffer)?; } + + // add metadata to archive + segment = FileSegment::new(offset, final_zsize, size); } // update archive metadata @@ -361,16 +360,15 @@ where sha1_hash, ); - if let Some(segment) = segment { - let wrapped_entry = ZipEntry { - hash, - name: None, - entry, - segment, - buffers, - }; - entries.insert(hash, wrapped_entry); - } + let wrapped_entry = ZipEntry { + hash, + name: None, + entry, + segment, + buffers, + }; + + entries.insert(hash, wrapped_entry); } // run through entries again and enumerate the segments From 736e3db8d5156ebd7b0e4fefbc7b700c1476c092 Mon Sep 17 00:00:00 2001 From: Moritz Baron Date: Sat, 13 Jan 2024 12:34:00 +0100 Subject: [PATCH 14/14] Update mod.rs --- src/archive/mod.rs | 255 +++++++++++++++++++++++---------------------- 1 file changed, 128 insertions(+), 127 deletions(-) diff --git a/src/archive/mod.rs b/src/archive/mod.rs index 9e7899c..0e4d6c0 100644 --- a/src/archive/mod.rs +++ b/src/archive/mod.rs @@ -240,133 +240,7 @@ where //let imports_hash_set: HashSet = HashSet::new(); let mut entries = HashMap::default(); for (path, hash) in file_info { - // read file - let mut file = File::open(&path)?; - let mut file_buffer = Vec::new(); - file.read_to_end(&mut file_buffer)?; - - let mut file_cursor = Cursor::new(&file_buffer); - - //let firstimportidx = imports_hash_set.len(); - //let mut lastimportidx = imports_hash_set.len(); - let mut flags = 0; - let mut segment: FileSegment; - let mut buffers = vec![]; - - if let Ok(info) = read_cr2w_header(&mut file_cursor) { - // get main file - file_cursor.seek(SeekFrom::Start(0))?; - let size = info.header.objects_end; - let mut resource_buffer = vec![0; size as usize]; - file_cursor.read_exact(&mut resource_buffer[..])?; - // get archive offset before writing - let archive_offset = archive_writer.stream_position()?; - - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &resource_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - - // write compressed main file archive - // KARK header - archive_writer.write_u32::(kraken::MAGIC)?; //magic - archive_writer.write_u32::(size)?; //uncompressed buffer length - archive_writer.write_all(&compressed_buffer)?; - - // add metadata to archive - segment = FileSegment::new(archive_offset, zsize as u32, size); - - // write buffers (bytes after the main file) - for buffer_info in info.buffers_table.iter() { - let mut buffer = vec![0; buffer_info.disk_size as usize]; - file_cursor.read_exact(&mut buffer[..])?; - - let bsize = buffer_info.mem_size; - let bzsize = buffer_info.disk_size; - let boffset = archive_writer.stream_position()?; - archive_writer.write_all(buffer.as_slice())?; - - // add metadata to archive - buffers.push(FileSegment::new(boffset, bzsize, bsize)); - } - - //register imports - // NOTE don't use a dependency list for mods - //for import in info.imports.iter() { - // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) - //imports_hash_set.insert(import.depot_path.to_owned()); - //} - - //lastimportidx = imports_hash_set.len(); - - flags = if !info.buffers_table.is_empty() { - info.buffers_table.len() - 1 - } else { - 0 - }; - } else { - // write non-cr2w file - file_cursor.seek(SeekFrom::Start(0))?; - let os_ext = path.extension().unwrap(); - let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); - if get_aligned_file_extensions().contains(&ext) { - pad_until_page(&mut archive_writer)?; - } - - let offset = archive_writer.stream_position()?; - let size = file_buffer.len() as u32; - let final_zsize; - if get_uncompressed_file_extensions().contains(&ext) { - // direct copy - archive_writer.write_all(&file_buffer)?; - final_zsize = size; - } else { - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &file_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - final_zsize = zsize as u32; - // write - archive_writer.write_all(&compressed_buffer)?; - } - - // add metadata to archive - segment = FileSegment::new(offset, final_zsize, size); - } - - // update archive metadata - let sha1_hash = sha1_hash_file(&file_buffer); - - let entry = FileEntry::new( - hash, - 0, - flags as u32, - 0, //firstoffsetidx as u32, - 0, //lastoffsetidx as u32, - 0, //firstimportidx as u32, - 0, //lastimportidx as u32, - sha1_hash, - ); - - let wrapped_entry = ZipEntry { - hash, - name: None, - entry, - segment, - buffers, - }; + let wrapped_entry = make_entry(path, &mut archive_writer, hash)?; entries.insert(hash, wrapped_entry); } @@ -408,6 +282,133 @@ where Ok(()) } +fn make_entry( + path: PathBuf, + archive_writer: &mut BufWriter, + hash: u64, +) -> Result { + let mut file = File::open(&path)?; + let mut file_buffer = Vec::new(); + file.read_to_end(&mut file_buffer)?; + let mut file_cursor = Cursor::new(&file_buffer); + + let mut flags = 0; + let segment: FileSegment; + let mut buffers = vec![]; + + if let Ok(info) = read_cr2w_header(&mut file_cursor) { + // get main file + file_cursor.seek(SeekFrom::Start(0))?; + let size = info.header.objects_end; + let mut resource_buffer = vec![0; size as usize]; + file_cursor.read_exact(&mut resource_buffer[..])?; + // get archive offset before writing + let archive_offset = archive_writer.stream_position()?; + + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &resource_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + + // write compressed main file archive + // KARK header + archive_writer.write_u32::(kraken::MAGIC)?; //magic + archive_writer.write_u32::(size)?; //uncompressed buffer length + archive_writer.write_all(&compressed_buffer)?; + + // add metadata to archive + segment = FileSegment::new(archive_offset, zsize as u32, size); + + // write buffers (bytes after the main file) + for buffer_info in info.buffers_table.iter() { + let mut buffer = vec![0; buffer_info.disk_size as usize]; + file_cursor.read_exact(&mut buffer[..])?; + + let bsize = buffer_info.mem_size; + let bzsize = buffer_info.disk_size; + let boffset = archive_writer.stream_position()?; + archive_writer.write_all(buffer.as_slice())?; + + // add metadata to archive + buffers.push(FileSegment::new(boffset, bzsize, bsize)); + } + + //register imports + // NOTE don't use a dependency list for mods + //for import in info.imports.iter() { + // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) + //imports_hash_set.insert(import.depot_path.to_owned()); + //} + + //lastimportidx = imports_hash_set.len(); + + flags = if !info.buffers_table.is_empty() { + info.buffers_table.len() - 1 + } else { + 0 + }; + } else { + // write non-cr2w file + file_cursor.seek(SeekFrom::Start(0))?; + let os_ext = path.extension().unwrap(); + let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); + if get_aligned_file_extensions().contains(&ext) { + pad_until_page(archive_writer)?; + } + + let offset = archive_writer.stream_position()?; + let size = file_buffer.len() as u32; + let final_zsize; + if get_uncompressed_file_extensions().contains(&ext) { + // direct copy + archive_writer.write_all(&file_buffer)?; + final_zsize = size; + } else { + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &file_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + final_zsize = zsize as u32; + // write + archive_writer.write_all(&compressed_buffer)?; + } + + // add metadata to archive + segment = FileSegment::new(offset, final_zsize, size); + } + let sha1_hash = sha1_hash_file(&file_buffer); + let entry = FileEntry::new( + hash, + 0, + flags as u32, + 0, //firstoffsetidx as u32, + 0, //lastoffsetidx as u32, + 0, //firstimportidx as u32, + 0, //lastimportidx as u32, + sha1_hash, + ); + let wrapped_entry = ZipEntry { + hash, + name: None, + entry, + segment, + buffers, + }; + Ok(wrapped_entry) +} + fn collect_resource_files>(in_folder: &P) -> Vec { // collect files let mut included_extensions = ERedExtension::iter()