(#112) fixing all LumpClasses except DisplacementInfo

bsp_tool/branches/arkane/dark_messiah_sp.py

@@ -2,6 +2,7 @@
 from typing import List
 
 from .. import base
+from .. import vector
 from ..valve import orange_box
 from ..valve import source
 
@@ -26,14 +27,15 @@ class Model(base.Struct):  # LUMP 14
     bounds: List[float]
     # bounds.mins: List[float]  # xyz
     # bounds.maxs: List[float]  # xyz
-    origin: List[float]
+    origin: List[float]  # xyz
     unknown: int
-    head_node: int   # index into Node lump
-    first_face: int  # index into Face lump
+    head_node: int   # top-level Node of this Model
+    first_face: int  # first Face of this Model
     num_faces: int   # number of Faces after first_face in this Model
     __slots__ = ["bounds", "origin", "unknown", "head_node", "first_face", "num_faces"]
     _format = "9f4i"
-    _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}}
+    _arrays = {"bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}, "origin": [*"xyz"]}
+    _classes = {"bounds.mins": vector.vec3, "bounds.maxs": vector.vec3, "origin": vector.vec3}
 
 
 class TextureInfo(base.Struct):  # LUMP 6
@@ -43,11 +45,14 @@ class TextureInfo(base.Struct):  # LUMP 6
     unknown: bytes
     flags: int  # Surface flags
     texture_data: int  # index of TextureData
-    __slots__ = ["texture", "lightmap", "flags", "texture_data"]
+    __slots__ = ["texture", "lightmap", "unknown", "flags", "texture_data"]
     _format = "16f24s2i"
     _arrays = {"texture": {"s": [*"xyz", "offset"], "t": [*"xyz", "offset"]},
                "lightmap": {"s": [*"xyz", "offset"], "t": [*"xyz", "offset"]}}
-    # ^ nested MappedArrays; texture.s.x, texture.t.x
+    _classes = {**{"{g}.{v}.xyz": vector.vec3 for g in ("texture", "lightmap") for v in "st"},
+                "flags": source.Surface}
+    # TODO: TextureVector class from vmf_tool
+    # TODO: .uv_at(position: vector.vec3) TextureVector projection
 
 
 # classes for special lumps, in alphabetical order:

bsp_tool/branches/id_software/qfusion.py

@@ -168,17 +168,19 @@ def from_tuple(cls, _tuple):
         return out
 
 
-class Vertex(base.MappedArray):  # LUMP 10
+class Vertex(base.Struct):  # LUMP 10
     position: List[float]
     uv0: List[float]
-    lightmap_uv: List[List[float]]
+    lightmap_uv: List[List[float]]  # 4 uvs for 4 styles
     normal: List[float]
-    color: List[List[int]]
-    __slots__ = ["position", "uv0", "lightmap_uv", "normal"]
+    color: List[List[int]]  # 4 colours for 4 styles
+    __slots__ = ["position", "uv0", "lightmap_uv", "normal", "colour"]
     _format = "16f16B"
-    _arrays = {"position": [*"xyz"], "uv0": [*"uv"],
+    _arrays = {"position": [*"xyz"],
+               "uv0": [*"uv"],
                "lightmap_uv": {s: [*"uv"] for s in "ABCD"},
-               "normal": [*"xyz"], "color": {s: 4 for s in "ABCD"}}
+               "normal": [*"xyz"],
+               "colour": {s: 4 for s in "ABCD"}}
 
 
 # {"LUMP_NAME": LumpClass}

bsp_tool/branches/id_software/quake2.py

@@ -147,16 +147,18 @@ class Surface(enum.IntFlag):  # qcommon/qfiles.h
 
 # classes for lumps, in alphabetical order:
 class Brush(base.MappedArray):  # LUMP 14
-    first_side: int
-    num_sides: int
+    first_brush_side: int  # first BrushSide of this Brush
+    num_brush_sides: int  # number of BrushSides after first_brush_side on this Brush
     contents: int
-    _mapping = ["first_side", "num_sides", "contents"]
+    _mapping = ["first_brush_side", "num_brush_sides", "contents"]
     _format = "3i"
+    _classes = {"contents": Contents}
 
 
 class BrushSide(base.MappedArray):  # LUMP 15
-    plane: int
-    texture_info: int
+    plane: int  # Plane this BrushSide lies on
+    texture_info: int  # TextureInfo of this BrushSide
+    _mapping = ["plane", "texture_info"]
     _format = "Hh"
 
 
@@ -194,7 +196,7 @@ class Node(base.Struct):  # LUMP 4
     # NOTE: bounds are generous, rounding up to the nearest 16 units
     first_face: int  # index of the first Face in this Node
     num_faces: int  # number of Faces in this Node after first_face
-    # __slots__ = ["plane", "children", "bounds", "first_face", "num_faces"]
+    __slots__ = ["plane", "children", "bounds", "first_face", "num_faces"]
     _format = "I2i8h"
     _arrays = {"children": ["front", "back"],
                "bounds": {"mins": [*"xyz"], "maxs": [*"xyz"]}}

bsp_tool/branches/infinity_ward/call_of_duty1_demo.py

@@ -97,7 +97,7 @@ class BrushSide(base.Struct):  # LUMP 3
     plane: int   # Plane this BrushSide lies on
     # NOTE: in some cases the plane index is a distance instead (float)
     # "first 6 entries indicated by an entry in lump 6 [brushes] are distances (float), rest is plane ID's"
-    texture: int  # index into Texture lump
+    texture: int  # Texture of this BrushSide
     __slots__ = ["plane", "texture"]
     _format = "2I"
 
@@ -107,13 +107,15 @@ class Cell(base.Struct):  # LUMP 17
     unknown: List[int]
     __slots__ = ["unknown"]
     _format = "13i"
+    _arrays = {"unknown": 13}
 
 
 class CullGroup(base.Struct):  # LUMP 9
     # indices & aabb tree stuff?
     unknown: List[int]
     __slots__ = ["unknown"]
     _format = "8i"
+    _arrays = {"unknown": 8}
 
 
 class Leaf(base.Struct):  # LUMP 21

bsp_tool/branches/infinity_ward/call_of_duty2.py

@@ -5,6 +5,7 @@
 from typing import List
 
 from .. import base
+from .. import vector
 from ..id_software import quake
 from ..id_software import quake3
 from . import call_of_duty1
@@ -65,9 +66,9 @@ class LUMP(enum.Enum):
 LumpHeader = call_of_duty1.LumpHeader
 
 
-# known lump changes from Call of Duty -> Call of Duty 2:
+# Known lump changes from Call of Duty -> Call of Duty 2:
 #   INDICES -> TRIANGLES
-# new:
+# New:
 #   COLLISION_AABBS
 #   COLLISION_BORDERS
 #   COLLISION_EDGES
@@ -81,7 +82,7 @@ class LUMP(enum.Enum):
 #   SHADOW_INDICES
 #   SHADOW_SOURCES
 #   SHADOW_VERTICES
-# deprecated:
+# Deprecated:
 #   COLLISION_INDICES
 #   LIGHT_INDICES
 
@@ -126,23 +127,23 @@ class Surface(enum.IntFlag):
 class CollisionEdge(base.Struct):  # LUMP 30
     unknown: int  # an index?
     position: List[float]
-    normal: List[List[float]]
+    normal: List[List[float]]  # x3?
     distance: float
     __slots__ = ["unknown", "position", "normal", "distance"]
     _format = "I13f"
-    _arrays = {"position": [*"xyz"],
-               "normal": {"A": [*"xyz"], "B": [*"xyz"], "C": [*"xyz"]}}
-    # NOTE: {"normal": {3: [*"xyz"]}} doesn't work /; (yet)
+    _arrays = {"position": [*"xyz"], "normal": {i: [*"xyz"] for i in "ABC"}}
+    _classes = {"position": vector.vec3, **{f"normal.{i}": vector.vec3 for i in "ABC"}}
 
 
 class CollisionTriangle(base.Struct):  # LUMP 31
     normal: List[float]
     distance: float
     unknown_1: List[float]
-    unknown_2: List[int]
+    unknown_2: List[int]  # ids?
     __slots__ = ["normal", "distance", "unknown_1", "unknown_2"]
-    _format = "12f6"
-    _arrays = {"normal": [*"xyz"], "unknown": 8, "id": 6}
+    _format = "12f6i"
+    _arrays = {"normal": [*"xyz"], "unknown_1": 8, "unknown_2": 6}
+    _classes = {"normal": vector.vec3}
 
 
 class Model(base.Struct):  # LUMP 35
@@ -158,6 +159,7 @@ class Model(base.Struct):  # LUMP 35
                  "first_mesh", "num_meshes", "first_brush", "num_brushes"]
     _format = "6f6i"
     _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]}
+    _classes = {"mins": vector.vec3, "maxs": vector.vec3}
 
 
 class Triangle(list):  # LUMP 9
@@ -190,6 +192,8 @@ class Vertex(base.Struct):  # LUMP 8
     _format = "6f4B10f"
     _arrays = {"position": [*"xyz"], "normal": [*"xyz"], "colour": [*"rgba"],
                "uv0": [*"uv"], "uv1": [*"uv"], "unknown": 6}
+    _classes = {"position": vector.vec3, "normal": vector.vec3}
+    # TODO: Colour32 & vec2_uv
 
 
 # {"LUMP_NAME": LumpClass}

bsp_tool/branches/respawn/titanfall_x360.py

@@ -35,7 +35,11 @@ class GameLump_SPRPv12_x360(titanfall.GameLump_SPRPv12):  # sprp GameLump (LUMP
 
 
 # {"LUMP_NAME": {version: LumpClass}}
-BASIC_LUMP_CLASSES, LumpClasses = x360.convert_versioned(titanfall.BASIC_LUMP_CLASSES)
+BASIC_LUMP_CLASSES = titanfall.BASIC_LUMP_CLASSES.copy()
+# big-endian BitField not yet supported
+BASIC_LUMP_CLASSES.pop("CM_PRIMITIVES")
+BASIC_LUMP_CLASSES.pop("TRICOLL_TRIANGLES")
+BASIC_LUMP_CLASSES, LumpClasses = x360.convert_versioned(BASIC_LUMP_CLASSES)
 # copy used LumpClasses to globals
 for LumpClass_name, LumpClass in LumpClasses.items():
     globals()[LumpClass_name] = LumpClass

bsp_tool/branches/ritual/fakk2.py

@@ -81,7 +81,7 @@ class Face(base.Struct):  # LUMP 3
     patch: List[float]  # control point dimentions?
     subdivisions: float  # patch subdivisions? dynamic lod?
     __slots__ = ["texture", "effect", "type", "first_vertex", "num_vertices",
-                 "first_index", "num_indices", "lightmap", "normal", "size", "subdivisions"]
+                 "first_index", "num_indices", "lightmap", "normal", "patch", "subdivisions"]
     _format = "12i12f2if"
     _arrays = {"lightmap": {"index": None, "top_left": [*"xy"], "size": ["width", "height"],
                             "origin": [*"xyz"], "vector": {"s": [*"xyz"], "t": [*"xyz"]}},

bsp_tool/branches/ritual/mohaa_demo.py

@@ -144,7 +144,7 @@ class Unknown14(base.Struct):  # LUMP 14
     __slots__ = ["mins", "maxs", "first_unknown_1", "num_unknown_1",
                  "first_unknown_2", "num_unknown_2"]
     _format = "6f4i"
-    _arrays = {"min": [*"xyz"], "max": [*"xyz"]}
+    _arrays = {"mins": [*"xyz"], "maxs": [*"xyz"]}
     _classes = {"mins": vector.vec3, "maxs": vector.vec3}
 
 

bsp_tool/branches/ritual/star_trek_elite_force2_demo.py

@@ -58,8 +58,8 @@ class LUMP(enum.Enum):
 LumpHeader = quake.LumpHeader
 
 
-# known lump changes from F.A.K.K. 2 -> Star Trek: Elite Force 2 Demo:
-# new:
+# Known lump changes from F.A.K.K. 2 -> Star Trek: Elite Force 2 Demo:
+# New:
 #   BASE_LIGHTMAPS
 #   CONT_LIGHTMAPS
 #   BASE_LIGHTING_VERTICES
@@ -112,17 +112,24 @@ class Face(base.Struct):  # LUMP 3
     base_lighting_face: int  # index into BaseLightingFace lump
     terrain: List[int]
     __slots__ = ["texture", "effect", "type", "first_vertex", "num_vertices",
-                 "first_index", "num_indices", "lightmap", "normal", "size",
+                 "first_index", "num_indices", "lightmap", "normal", "patch",
                  "subdivisions", "base_lighting_face", "terrain"]
     _format = "12i12f2if6i"
-    _arrays = {"lightmap": {"index": None, "top_left": [*"xy"],
-               "size": ["width", "height"], "origin": [*"xyz"],
-               "vector": {"s": [*"xyz"], "t": [*"xyz"]}},
-               "normal": [*"xyz"], "patch": ["width", "height"],
+    _arrays = {"lightmap": {"index": None,
+                            "top_left": [*"xy"],
+                            "size": ["width", "height"],
+                            "origin": [*"xyz"],
+                            "vector": {"s": [*"xyz"], "t": [*"xyz"]}},
+               "normal": [*"xyz"],
+               "patch": ["width", "height"],
                "terrain": {"inverted": None, "face_flags": 4}}
-    _classes = {"type": quake3.FaceType, "lightmap.top_left": vector.vec2,
-                "lightmap.size": vector.renamed_vec2("width", "height"), "lightmap.origin": vector.vec3,
-                "lightmap.vector.s": vector.vec3, "lightmap.vector.t": vector.vec3, "normal": vector.vec3,
+    _classes = {"type": quake3.FaceType,
+                "lightmap.top_left": vector.vec2,
+                "lightmap.size": vector.renamed_vec2("width", "height"),
+                "lightmap.origin": vector.vec3,
+                "lightmap.vector.s": vector.vec3,
+                "lightmap.vector.t": vector.vec3,
+                "normal": vector.vec3,
                 "patch": vector.renamed_vec2("width", "height")}
 
 

bsp_tool/branches/valve/source.py

@@ -583,7 +583,7 @@ class LeafAmbientIndex(base.MappedArray):  # LUMP 52
     _mapping = ["num_samples", "first_sample"]
 
 
-class LeafAmbientSample(base.MappedArray):  # LUMP 56
+class LeafAmbientSample(base.Struct):  # LUMP 56
     """cube of lighting samples"""
     cube: List[List[int]]  # unsure about orientation / face order
     origin: vector.vec3  # uint8_t; "fixed point fraction of Leaf bounds"

tests/branches/test_LumpClasses.py

@@ -7,27 +7,22 @@
 
 from bsp_tool import branches
 from bsp_tool.branches import base
+from bsp_tool.branches import vector
 
 
-# TODO: use tests/maplist.py to look at headers to ensure UNUSED_* / UNKNOWN_* lumps are correctly marked
-# -- looking for unmapped lump versions would be nice too
+# TODO: no skipped padding due to struct alignment; sizeof(_format) == sum(map(sizeof, _format))
+# -- gets complicated with Displacement Neighbours
 
-# TODO: assert LumpClasses capture all bytes (no gaps caused by alignment)
-# sizeof(_format) == sum(map(sizeof, _format))
-
-# TODO: verify LumpClass __annotations__ match _format ("hHiI": int, "fg": float, "s": str, "?": bool)
+# TODO: __annotations__ match _format ("hHiI": int, "fg": float, "s": str, "?": bool)
 # -- _classes will override base type, confirm the reverse conversions (enum & vec3 -> int)
 # -- List[type] is appropriate for MappedArrays defined in Struct._arrays
 # TODO: check for outdated annotations, allow hints for properties
 # TODO: commented out type hints for _arrays?
 # -- allow type hints / comments for derived members (e.g. titanfall.Brush.num_brush_sides)
 
-# TODO: verify __slots__, _format, _arrays & _mapping line up correctly
-# -- all LumpClasses must coherently translate to and from bytes
-# -- no skipped bytes! (single byte alignment gets wierd)
-# -- incomplete / mismatched / outdated type-hints should also be checked
-# -- this may require scanning comments for "deep" type-hints `# attr.sub: type  # desc`
-# TODO: check for attr.sub typos in _classes & _bitfields
+# TODO: ensure all mappings are internally consistent
+# -- no typos / regressions across type hints, _arrays, _classes etc.
+# TODO: include attr.sub in _classes,_bitfields & (commented out) type hints
 
 # NOTE: _classes might complain about EnumClass(0) (if not defined) when initialising LumpClasses
 # TODO: interrogate LumpClass instances, not just class definitions
@@ -66,6 +61,7 @@ def test_Struct(LumpClass):
     assert not hasattr(LumpClass, "_mapping"), "Struct doesn't use _mapping"  # MappedArray only
     assert not hasattr(LumpClass, "_fields"), "Struct doesn't use _fields"  # BitField only
     assert hasattr(LumpClass, "_arrays"), "use MappedArray for shallow Structs"
+    assert all([a in LumpClass.__slots__ for a in LumpClass._arrays]), "typo / regression in _arrays"
     # TODO: how does memory effeciency differ between Struct & MappedArray
     # -- should we restrict use of MappedArray to _arrays definitions?
     LumpClass()  # must initialise once to generate struct_attr_formats entry
@@ -80,7 +76,10 @@ def test_Struct(LumpClass):
 def test_MappedArray(LumpClass):
     assert hasattr(LumpClass, "_format")
     assert struct.calcsize(LumpClass._format) > 0, "invalid _format"
-    assert not hasattr(LumpClass, "__slots__"), "MappedArray doesn't use __slots__"  # Struct only
+    if not issubclass(LumpClass, vector.vec3):  # quake.Vertex special case
+        assert not hasattr(LumpClass, "__slots__"), "MappedArray doesn't use __slots__"  # Struct only
+    else:
+        assert LumpClass.__slots__ == LumpClass._mapping  # __slots__ inherited from vector.vec3
     assert not hasattr(LumpClass, "_arrays"), "MappedArray doesn't use _arrays"  # Struct only
     assert not hasattr(LumpClass, "_fields"), "MappedArray doesn't use _fields"  # BitField only
     assert len(LumpClass._mapping) != 0, "forgot to create _mapping"