Make chunk size configurable

.gitattributes

@@ -1 +1 @@
-tests/data/* filter=lfs diff=lfs merge=lfs -text
+tests/_data/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -6,6 +6,6 @@ dist/
 *.pyc
 __pycache__/
 .pytest_cache/
-tests/docs/api
-tests/docs/build
+tests/_docs/api
+tests/_docs/build
 .tox/

dissect/archive/wim.py

@@ -151,7 +151,9 @@
             decompressor = DECOMPRESSOR_MAP.get(compression_flags)
             if decompressor is None:
                 raise NotImplementedError(f"Compression algorithm not yet supported: {compression_flags}")
-            return CompressedStream(self.wim.fh, self.offset, self.size, self.original_size, decompressor)
+            return CompressedStream(
+                self.wim.fh, self.offset, self.size, self.original_size, decompressor, self.wim.header.CompressionSize
+            )
 
         return RelativeStream(self.wim.fh, self.offset, self.size)
 
@@ -434,16 +436,18 @@
         compressed_size: int,
         original_size: int,
         decompressor: Callable[[bytes], bytes],
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
     ):
         self.fh = fh
         self.offset = offset
         self.compressed_size = compressed_size
         self.original_size = original_size
         self.decompressor = decompressor
+        self.chunk_size = chunk_size
 
         # Read the chunk table in advance
         fh.seek(self.offset)
-        num_chunks = (original_size + DEFAULT_CHUNK_SIZE - 1) // DEFAULT_CHUNK_SIZE - 1
+        num_chunks = (original_size + self.chunk_size - 1) // self.chunk_size - 1
         if num_chunks == 0:
             self._chunks = (0,)
         else:
@@ -460,7 +464,7 @@
         result = []
 
         num_chunks = len(self._chunks)
-        chunk, offset_in_chunk = divmod(offset, DEFAULT_CHUNK_SIZE)
+        chunk, offset_in_chunk = divmod(offset, self.chunk_size)
 
         while length:
             if chunk >= num_chunks:
@@ -470,10 +474,10 @@
             chunk_offset = self._chunks[chunk]
             if chunk < num_chunks - 1:
                 next_chunk_offset = self._chunks[chunk + 1]
-                chunk_remaining = DEFAULT_CHUNK_SIZE - offset_in_chunk
+                chunk_remaining = self.chunk_size - offset_in_chunk
             else:
                 next_chunk_offset = self.compressed_size
-                chunk_remaining = (self.original_size - (chunk * DEFAULT_CHUNK_SIZE)) - offset_in_chunk
+                chunk_remaining = (self.original_size - (chunk * self.chunk_size)) - offset_in_chunk
 
             read_length = min(chunk_remaining, length)
 

tests/_docs/Makefile

@@ -0,0 +1,24 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?= -jauto -w $(BUILDDIR)/warnings.log --fail-on-warning
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: clean help Makefile
+
+clean: Makefile
+	rm -rf api
+	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

tests/_docs/conf.py

@@ -0,0 +1,41 @@
+project = "dissect.archive"
+
+extensions = [
+    "autoapi.extension",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.doctest",
+    "sphinx.ext.napoleon",
+    "sphinx_argparse_cli",
+]
+
+exclude_patterns = []
+
+html_theme = "furo"
+
+autoapi_type = "python"
+autoapi_dirs = ["../../dissect/"]
+autoapi_ignore = ["*tests*", "*.tox*", "*venv*", "*examples*"]
+autoapi_python_use_implicit_namespaces = True
+autoapi_add_toctree_entry = False
+autoapi_root = "api"
+autoapi_options = [
+    "members",
+    "undoc-members",
+    "show-inheritance",
+    "show-module-summary",
+    "special-members",
+    "imported-members",
+]
+autoapi_keep_files = True
+autoapi_template_dir = "_templates/autoapi"
+
+autodoc_typehints = "signature"
+autodoc_member_order = "groupwise"
+
+autosectionlabel_prefix_document = True
+
+suppress_warnings = [
+    # https://github.com/readthedocs/sphinx-autoapi/issues/285
+    "autoapi.python_import_resolution",
+]

tests/_docs/index.rst

@@ -0,0 +1,8 @@
+API Reference
+=============
+
+.. toctree::
+    :maxdepth: 1
+    :glob:
+
+    /api/*/*/index

tests/conftest.py

@@ -25,20 +25,35 @@ def open_file_gz(name: str, mode: str = "rb") -> Iterator[BinaryIO]:
 
 
 @pytest.fixture
-def basic_wim() -> Iterator[BinaryIO]:
-    yield from open_file_gz("data/basic.wim.gz")
+def basic_wim_4k() -> Iterator[BinaryIO]:
+    yield from open_file_gz("_data/basic4k.wim.gz")
+
+
+@pytest.fixture
+def basic_wim_8k() -> Iterator[BinaryIO]:
+    yield from open_file_gz("_data/basic8k.wim.gz")
+
+
+@pytest.fixture
+def basic_wim_16k() -> Iterator[BinaryIO]:
+    yield from open_file_gz("_data/basic16k.wim.gz")
+
+
+@pytest.fixture
+def basic_wim_32k() -> Iterator[BinaryIO]:
+    yield from open_file_gz("_data/basic32k.wim.gz")
 
 
 @pytest.fixture
 def basic_vma() -> Iterator[BinaryIO]:
-    yield from open_file_gz("data/test.vma.gz")
+    yield from open_file_gz("_data/test.vma.gz")
 
 
 @pytest.fixture
 def vbk9() -> Iterator[BinaryIO]:
-    yield from open_file_gz("data/test9.vbk.gz")
+    yield from open_file_gz("_data/test9.vbk.gz")
 
 
 @pytest.fixture
 def vbk13() -> Iterator[BinaryIO]:
-    yield from open_file_gz("data/test13.vbk.gz")
+    yield from open_file_gz("_data/test13.vbk.gz")

tests/test_wim.py

@@ -3,11 +3,32 @@
 import hashlib
 from typing import BinaryIO
 
-from dissect.archive.wim import WIM
+import pytest
+from dissect.util.compression.lzxpress_huffman import decompress
 
+from dissect.archive.wim import WIM, CompressedStream
 
-def test_wim(basic_wim: BinaryIO) -> None:
-    wim = WIM(basic_wim)
+
+@pytest.mark.parametrize(
+    ("fixture", "chunk_size"),
+    [
+        ("basic_wim_4k", 0x1000),
+        ("basic_wim_8k", 0x2000),
+        ("basic_wim_16k", 0x4000),
+        ("basic_wim_32k", 0x8000),
+    ],
+)
+def test_wim(fixture: BinaryIO, chunk_size: int, request: pytest.FixtureRequest) -> None:
+    value = request.getfixturevalue(fixture)
+    wim = WIM(value)
+    assert wim.header.CompressionSize == chunk_size
+
+    resource = next(iter(wim.resources.values()))
+    assert resource.open().chunk_size == chunk_size
+
+    stream = CompressedStream(wim.fh, resource.offset, resource.size, resource.original_size, decompress, chunk_size)
+    assert resource.wim.header.CompressionSize == stream.chunk_size
+    assert resource.open().read() == stream.read()
 
     images = list(wim.images())
     assert len(images) == 1

tox.ini

@@ -62,12 +62,12 @@ deps =
     sphinx-design
     furo
 commands =
-    make -C tests/docs clean
-    make -C tests/docs html
+    make -C tests/_docs clean
+    make -C tests/_docs html
 
 [testenv:docs-linkcheck]
 allowlist_externals = make
 deps = {[testenv:docs-build]deps}
 commands =
-    make -C tests/docs clean
-    make -C tests/docs linkcheck
+    make -C tests/_docs clean
+    make -C tests/_docs linkcheck