moving shared.PakFile to valve.source.PakFile

bsp_tool/branches/nexon/cso2.py

@@ -1,8 +1,6 @@
 """2013-2017 format"""
 # https://git.sr.ht/~leite/cso2-bsp-converter/tree/master/item/src/bsptypes.hpp
 import enum
-import io
-import zipfile
 
 from .. import base
 from ..valve import source
@@ -100,22 +98,10 @@ class LumpHeader(base.MappedArray):
 
 
 # special lump classes, in alphabetical order:
-class PakFile(zipfile.ZipFile):  # WIP
-    """CSO2 PakFiles have a custom .zip format"""
-    # NOTE: it's not as simple as changing the FILE_MAGIC
-    # -- this appears to be a unique implementation of .zip
-    # b"CS" file magic & different header format?
-    def __init__(self, raw_zip: bytes):
-        # TODO: translate header to b"PK\x03\x04..."
-        raw_zip = b"".join([b"PK", raw_zip[2:]])  # not that easy
-        self._buffer = io.BytesIO(raw_zip)
-        super(PakFile, self).__init__(self._buffer)
-
-    def as_bytes(self) -> bytes:
-        # TODO: translate header to b"CS\x03\x04..."
-        raw_zip = self._buffer.getvalue()
-        raw_zip = b"".join([b"CS", raw_zip[2:]])  # not that easy
-        return raw_zip
+# TODO: PakFile
+# -- struct magics use b"CS" instead of b"PK"
+# -- changes may go deeper than his
+# -- hopefully just a reskin of PK/3/4 ZIP_STORE
 
 
 # {"LUMP_NAME": {version: LumpClass}}

bsp_tool/branches/respawn/titanfall.py

@@ -1068,7 +1068,7 @@ def as_bytes(self) -> bytes:
                         "ENTITIES":                  {0: shared.Entities},
                         # NOTE: .ent files are handled directly by the RespawnBsp class
                         "LEVEL_INFO":                {0: LevelInfo},
-                        "PAKFILE":                   {0: shared.PakFile},
+                        "PAKFILE":                   {0: source.PakFile},
                         # "PHYSICS_COLLIDE":           {0: physics.CollideLump},  # BROKEN .as_bytes()
                         "TEXTURE_DATA_STRING_DATA":  {0: shared.TextureDataStringData}}
 # TODO: LightProbeParentInfos/BspNodes/RefIds & StaticPropLightProbeIndices may all be Special

bsp_tool/branches/shared.py

@@ -1,8 +1,6 @@
-import io
 import math
 import re
-import zipfile
-from typing import Any, Dict, List
+from typing import Dict, List
 
 
 # Basic Lump Classes
@@ -115,38 +113,6 @@ def from_bytes(cls, raw_lump: bytes):
         return cls(entities)
 
 
-class PakFile(zipfile.ZipFile):
-    _buffer: io.BytesIO
-
-    def __init__(self, file_: Any = None, mode: str = "a", **kwargs):
-        """always a read-only copy of the lump"""
-        if file_ is None:
-            empty_zip = [b"PK\x05\x06", b"\x00" * 16, b"\x20\x00XZP1\x20\x30", b"\x00" * 26]
-            self._buffer = io.BytesIO(b"".join(empty_zip))
-        elif isinstance(file_, io.BytesIO):  # BspClass will take this route via .from_bytes()
-            self._buffer = file_
-        elif isinstance(file_, str):
-            self._buffer = io.BytesIO(open(file_, "rb").read())
-        else:
-            raise TypeError(f"Cannot create {self.__class__.__name__} from type '{type(file_)}'")
-        super().__init__(self._buffer, mode=mode, **kwargs)
-
-    def as_bytes(self) -> bytes:
-        # write ending records if edits were made (adapted from ZipFile.close)
-        if self.mode in "wxa" and self._didModify and self.fp is not None:
-            with self._lock:
-                if self._seekable:
-                    self.fp.seek(self.start_dir)
-                self._write_end_record()
-        self._didModify = False  # don't double up when .close() is called
-        # NOTE: .close() can get funky but it's OK because ._buffer isn't a real file
-        return self._buffer.getvalue()
-
-    @classmethod
-    def from_bytes(cls, raw_lump: bytes):
-        return cls(io.BytesIO(raw_lump))
-
-
 class TextureDataStringData(list):
     def __init__(self, iterable: List[str] = tuple()):
         super().__init__(iterable)

bsp_tool/branches/valve/orange_box_x360.py

@@ -85,6 +85,8 @@ class MAX(enum.Enum):
 
 
 # special lump classes, in alphabetical order:
+# TODO: PakFile_x360
+# TODO: PhysicsCollide_x360
 # class PhysicsDisplacement_x360(physics.Displacement):
 #     _int = UnsignedShort_x360
 
@@ -134,12 +136,8 @@ class GameLump_SPRPv10_x360(orange_box.GameLump_SPRPv10):  # sprp GAME LUMP (LUM
     globals()[LumpClass_name] = LumpClass
 del LumpClass_name, LumpClass
 
-
 SPECIAL_LUMP_CLASSES = {"ENTITIES":                 {0: shared.Entities},
-                        # "PAKFILE":                  {0: shared.PakFile_x360},
-                        # "PHYSICS_DISPLACEMENT":     {0: PhysicsDisplacement_x360},
                         "TEXTURE_DATA_STRING_DATA": {0: shared.TextureDataStringData}}
-# TODO: PhysicsCollide_x360
 
 GAME_LUMP_HEADER = x360.make_big_endian(orange_box.GAME_LUMP_HEADER)
 

bsp_tool/branches/valve/source.py

@@ -6,7 +6,8 @@
 import enum
 import io
 import struct
-from typing import List
+from typing import Any, List
+import zipfile
 
 from ... import lumps
 from .. import base
@@ -918,6 +919,42 @@ class GameLump_SPRPv7(GameLump_SPRPv6):  # sprp GameLump (LUMP 35)
     StaticPropClass = StaticPropv7
 
 
+class PakFile(zipfile.ZipFile):  # LUMP 40
+    _buffer: io.BytesIO
+
+    def __init__(self, file_: Any = None, mode: str = "a", **kwargs):
+        """always a read-only copy of the lump"""
+        if file_ is None:
+            empty_zip = [b"PK\x05\x06", b"\x00" * 16, b"\x20\x00XZP1 0", b"\x00" * 26]
+            self._buffer = io.BytesIO(b"".join(empty_zip))
+        elif isinstance(file_, io.BytesIO):  # BspClass will take this route via .from_bytes()
+            self._buffer = file_
+        elif isinstance(file_, str):
+            self._buffer = io.BytesIO(open(file_, "rb").read())
+        else:
+            raise TypeError(f"Cannot create {self.__class__.__name__} from type '{type(file_)}'")
+        super().__init__(self._buffer, mode=mode, **kwargs)
+
+    def __repr__(self) -> str:
+        dev_branch_class = ".".join([*self.__module__.split(".")[-2:], self.__class__.__name__])
+        return f"<{dev_branch_class} {len(self.namelist())} files mode='{self.mode}' @ 0x{id(self):016X}>"
+
+    def as_bytes(self) -> bytes:
+        # write ending records if edits were made (adapted from ZipFile.close)
+        if self.mode in "wxa" and self._didModify and self.fp is not None:
+            with self._lock:
+                if self._seekable:
+                    self.fp.seek(self.start_dir)
+                self._write_end_record()
+        self._didModify = False  # don't double up when .close() is called
+        # NOTE: .close() can get funky but it's OK because ._buffer isn't a real file
+        return self._buffer.getvalue()
+
+    @classmethod
+    def from_bytes(cls, raw_lump: bytes):
+        return cls(io.BytesIO(raw_lump))
+
+
 # {"LUMP_NAME": {version: LumpClass}}
 BASIC_LUMP_CLASSES = {"DISPLACEMENT_TRIANGLES":    {0: DisplacementTriangle},
                       "FACE_IDS":                  {0: shared.UnsignedShorts},
@@ -963,7 +1000,7 @@ class GameLump_SPRPv7(GameLump_SPRPv6):  # sprp GameLump (LUMP 35)
 
 SPECIAL_LUMP_CLASSES = {"ENTITIES":                 {0: shared.Entities},
                         "TEXTURE_DATA_STRING_DATA": {0: shared.TextureDataStringData},
-                        "PAKFILE":                  {0: shared.PakFile},
+                        "PAKFILE":                  {0: PakFile},
                         # "PHYSICS_COLLIDE":          {0: physics.CollideLump},  # BROKEN .as_bytes()
                         "PHYSICS_DISPLACEMENT":     {0: physics.Displacement},
                         "VISIBILITY":               {0: quake2.Visibility}}

bsp_tool/extensions/diff/__init__.py

@@ -3,13 +3,15 @@
 
 from . import base
 from . import shared
+from .valve import source
 
 from bsp_tool import branches
 from bsp_tool.base import Bsp
 from bsp_tool.lumps import BasicBspLump, RawBspLump, ExternalRawBspLump
 
 
 def diff_lumps(old_lump: Any, new_lump: Any) -> base.Diff:
+    """lookup table & intialiser"""
     LumpClasses = set()
     for lump in (old_lump, new_lump):
         if issubclass(lump.__class__, BasicBspLump):
@@ -24,8 +26,8 @@ def diff_lumps(old_lump: Any, new_lump: Any) -> base.Diff:
         raise NotImplementedError("Cannot diff lumps of differring LumpClass")
     if LumpClasses == {branches.shared.Entities}:
         DiffClass = shared.EntitiesDiff
-    elif LumpClasses == {branches.shared.PakFile}:
-        DiffClass = shared.PakFileDiff
+    elif LumpClasses == {branches.valve.source.PakFile}:
+        DiffClass = source.PakFileDiff
     elif RawBspLump in LumpClasses or ExternalRawBspLump in LumpClasses:
         # TODO: core.xxd diff
         raise NotImplementedError("Cannot diff raw lumps")

bsp_tool/extensions/diff/shared.py

@@ -1,10 +1,7 @@
 import difflib
-import io
-import os
-from typing import Dict, Generator, List, Tuple
+from typing import Dict, Generator, List
 
 from . import base
-from . import core
 
 
 class EntitiesDiff(base.Diff):
@@ -64,57 +61,3 @@ def long_repr(entity: Dict[str, str]) -> List[str]:
             else:
                 out.append(f'"{key}" "{value}"')
         return ["{", *out, "}"]
-
-
-class PakFileDiff(base.Diff):
-    """Works on any ValveBsp based .bsp (except CS:O2)"""
-
-    def short_stats(self) -> str:
-        """quick & dirty namelist check"""
-        old = set(self.old.namelist())
-        new = set(self.new.namelist())
-        added = len(new.difference(old))
-        removed = len(old.difference(new))
-        return f"{added} insertions(+) {removed} deletions(-)"
-
-    def unified_diff(self) -> Generator[str, None, None]:
-        old_filelist = [f"{fn}\n" for fn in self.old.namelist()]
-        new_filelist = [f"{fn}\n" for fn in self.new.namelist()]
-        meta_diff = difflib.ndiff(old_filelist, new_filelist)
-        for meta_line in meta_diff:
-            if meta_line.startswith(" "):  # maybe a match
-                filename = meta_line.lstrip(" ").rstrip("\n")
-                old_file = self.old.read(filename)
-                new_file = self.new.read(filename)
-                if old_file == new_file:
-                    yield meta_line
-                else:
-                    for line in self.diff_file(filename):
-                        yield f"  {line}"
-            else:
-                yield meta_line
-
-    def diff_file(self, filename: str) -> Generator[str, None, None]:
-        old_file = self.old.read(filename)
-        new_file = self.new.read(filename)
-        _, ext = os.path.splitext(filename)
-        # TODO: check MegaTest .bsps for other common plaintext file formats
-        if ext in (".log", ".vmt", ".txt"):
-            try:
-                old = io.StringIO(old_file.decode()).readlines()
-                new = io.StringIO(new_file.decode()).readlines()
-            except UnicodeDecodeError:  # not pure utf-8
-                old = list(core.xxd(old_file))
-                new = list(core.xxd(new_file))
-        else:  # binary diff
-            old = list(core.xxd(old_file))
-            new = list(core.xxd(new_file))
-        old_time = self.formatted_date_time(self.old.getinfo(filename).date_time)
-        new_time = self.formatted_date_time(self.new.getinfo(filename).date_time)
-        for line in difflib.unified_diff(old, new, filename, filename, old_time, new_time):
-            yield line
-
-    @staticmethod
-    def formatted_date_time(zipinfo_date_time: Tuple[int]) -> str:
-        year, month, day, hour, minute, second = zipinfo_date_time
-        return f"{year:04d}-{month:02d}-{day:02d} {hour:02d}:{minute:02d}:{second:02d}"

bsp_tool/extensions/diff/valve/source.py

@@ -0,0 +1,62 @@
+import difflib
+import io
+import os
+
+from typing import Generator, Tuple
+
+from .. import base
+from .. import core
+
+
+class PakFileDiff(base.Diff):
+    """Works on any ValveBsp based .bsp (except CS:O2)"""
+
+    def short_stats(self) -> str:
+        """quick & dirty namelist check"""
+        old = set(self.old.namelist())
+        new = set(self.new.namelist())
+        added = len(new.difference(old))
+        removed = len(old.difference(new))
+        return f"{added} insertions(+) {removed} deletions(-)"
+
+    def unified_diff(self) -> Generator[str, None, None]:
+        old_filelist = [f"{fn}\n" for fn in self.old.namelist()]
+        new_filelist = [f"{fn}\n" for fn in self.new.namelist()]
+        meta_diff = difflib.ndiff(old_filelist, new_filelist)
+        for meta_line in meta_diff:
+            if meta_line.startswith(" "):  # maybe a match
+                filename = meta_line.lstrip(" ").rstrip("\n")
+                old_file = self.old.read(filename)
+                new_file = self.new.read(filename)
+                if old_file == new_file:
+                    yield meta_line
+                else:
+                    for line in self.diff_file(filename):
+                        yield f"  {line}"
+            else:
+                yield meta_line
+
+    def diff_file(self, filename: str) -> Generator[str, None, None]:
+        old_file = self.old.read(filename)
+        new_file = self.new.read(filename)
+        _, ext = os.path.splitext(filename)
+        # TODO: check MegaTest .bsps for other common plaintext file formats
+        if ext in (".log", ".vmt", ".txt"):
+            try:
+                old = io.StringIO(old_file.decode()).readlines()
+                new = io.StringIO(new_file.decode()).readlines()
+            except UnicodeDecodeError:  # not pure utf-8
+                old = list(core.xxd(old_file))
+                new = list(core.xxd(new_file))
+        else:  # binary diff
+            old = list(core.xxd(old_file))
+            new = list(core.xxd(new_file))
+        old_time = self.formatted_date_time(self.old.getinfo(filename).date_time)
+        new_time = self.formatted_date_time(self.new.getinfo(filename).date_time)
+        for line in difflib.unified_diff(old, new, filename, filename, old_time, new_time):
+            yield line
+
+    @staticmethod
+    def formatted_date_time(zipinfo_date_time: Tuple[int]) -> str:
+        year, month, day, hour, minute, second = zipinfo_date_time
+        return f"{year:04d}-{month:02d}-{day:02d} {hour:02d}:{minute:02d}:{second:02d}"