Migrate to Construct 2.10

Move away from an internal copy of Construct to the latest version in order to support DWARF v5 required data types.

.github/workflows/ci.yml

@@ -23,6 +23,8 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: pip install -e .
     - name: Test
       run: |
         python test/all_tests.py

elftools/common/construct_utils.py

@@ -6,60 +6,61 @@
 # Eli Bendersky (eliben@gmail.com)
 # This code is in the public domain
 #-------------------------------------------------------------------------------
-from ..construct import (
-    Subconstruct, ConstructError, ArrayError, Adapter, Field, RepeatUntil,
-    Rename, SizeofError, Construct
-    )
+import itertools
+
+from construct import (
+    Subconstruct, Adapter, Bytes, RepeatUntil, SizeofError,
+    Construct, ListContainer, Container, StopFieldError,
+    singleton, GreedyBytes, NullTerminated, Struct
+)
 
 
 class RepeatUntilExcluding(Subconstruct):
     """ A version of construct's RepeatUntil that doesn't include the last
-        element (which casued the repeat to exit) in the return value.
+        element (which caused the repeat to exit) in the return value.
 
         Only parsing is currently implemented.
 
         P.S. removed some code duplication
     """
-    __slots__ = ["predicate"]
     def __init__(self, predicate, subcon):
-        Subconstruct.__init__(self, subcon)
+        super().__init__(subcon)
         self.predicate = predicate
-        self._clear_flag(self.FLAG_COPY_CONTEXT)
-        self._set_flag(self.FLAG_DYNAMIC)
-    def _parse(self, stream, context):
-        obj = []
-        try:
-            context_for_subcon = context
-            if self.subcon.conflags & self.FLAG_COPY_CONTEXT:
-                context_for_subcon = context.__copy__()
-
-            while True:
-                subobj = self.subcon._parse(stream, context_for_subcon)
-                if self.predicate(subobj, context):
-                    break
-                obj.append(subobj)
-        except ConstructError as ex:
-            raise ArrayError("missing terminator", ex)
-        return obj
-    def _build(self, obj, stream, context):
+
+    def _parse(self, stream, context, path):
+        predicate = self.predicate
+        if not callable(predicate):
+            predicate = lambda _1,_2,_3: predicate
+        obj = ListContainer()
+        for i in itertools.count():
+            context._index = i
+            e = self.subcon._parsereport(stream, context, path)
+            obj.append(e)
+            if predicate(e, obj, context):
+                del obj[-1]
+                return obj
+
+    def _build(self, obj, stream, context, path):
         raise NotImplementedError('no building')
-    def _sizeof(self, context):
-        raise SizeofError("can't calculate size")
+
+    def _sizeof(self, context, path):
+        raise SizeofError("cannot calculate size, amount depends on actual data", path=path)
 
 
 def _LEB128_reader():
     """ Read LEB128 variable-length data from the stream. The data is terminated
         by a byte with 0 in its highest bit.
     """
     return RepeatUntil(
-                lambda obj, ctx: ord(obj) < 0x80,
-                Field(None, 1))
+        lambda obj, list, ctx: ord(obj) < 0x80,
+        Bytes(1)
+    )
 
 
 class _ULEB128Adapter(Adapter):
     """ An adapter for ULEB128, given a sequence of bytes in a sub-construct.
     """
-    def _decode(self, obj, context):
+    def _decode(self, obj, context, path):
         value = 0
         for b in reversed(obj):
             value = (value << 7) + (ord(b) & 0x7F)
@@ -69,7 +70,7 @@ def _decode(self, obj, context):
 class _SLEB128Adapter(Adapter):
     """ An adapter for SLEB128, given a sequence of bytes in a sub-construct.
     """
-    def _decode(self, obj, context):
+    def _decode(self, obj, context, path):
         value = 0
         for b in reversed(obj):
             value = (value << 7) + (ord(b) & 0x7F)
@@ -79,16 +80,19 @@ def _decode(self, obj, context):
         return value
 
 
-def ULEB128(name):
+@singleton
+def ULEB128():
     """ A construct creator for ULEB128 encoding.
     """
-    return Rename(name, _ULEB128Adapter(_LEB128_reader()))
+    return _ULEB128Adapter(_LEB128_reader())
 
 
-def SLEB128(name):
+@singleton
+def SLEB128():
     """ A construct creator for SLEB128 encoding.
     """
-    return Rename(name, _SLEB128Adapter(_LEB128_reader()))
+    return _SLEB128Adapter(_LEB128_reader())
+
 
 class StreamOffset(Construct):
     """
@@ -101,12 +105,65 @@ class StreamOffset(Construct):
     StreamOffset("item_offset")
     """
     __slots__ = []
-    def __init__(self, name):
-        Construct.__init__(self, name)
-        self._set_flag(self.FLAG_DYNAMIC)
-    def _parse(self, stream, context):
+    def __init__(self):
+        Construct.__init__(self)
+    def _parse(self, stream, context, path):
         return stream.tell()
-    def _build(self, obj, stream, context):
+    def _build(self, obj, stream, context, path):
         context[self.name] = stream.tell()
-    def _sizeof(self, context):
+    def _sizeof(self, context, path):
         return 0
+
+
+class EmbeddableStruct(Struct):
+    r"""
+    A special Struct that allows embedding of fields with type Embed.
+    """
+
+    def __init__(self, *subcons, **subconskw):
+        super().__init__(*subcons, **subconskw)
+
+    def _parse(self, stream, context, path):
+        obj = Container()
+        obj._io = stream
+        context = Container(_ = context, _params = context._params, _root = None, _parsing = context._parsing, _building = context._building, _sizing = context._sizing, _subcons = self._subcons, _io = stream, _index = context.get("_index", None), _parent = obj)
+        context._root = context._.get("_root", context)
+        for sc in self.subcons:
+            try:
+                subobj = sc._parsereport(stream, context, path)
+                if sc.name:
+                    obj[sc.name] = subobj
+                    context[sc.name] = subobj
+                elif subobj and isinstance(sc, Embed):
+                    obj.update(subobj)
+
+            except StopFieldError:
+                break
+        return obj
+
+
+class Embed(Subconstruct):
+    r"""
+    Special wrapper that allows outer multiple-subcons construct to merge fields from another multiple-subcons construct.
+    Parsing building and sizeof are deferred to subcon.
+    :param subcon: Construct instance, its fields to embed inside a struct or sequence
+    Example::
+        >>> outer = EmbeddableStruct(
+        ...     Embed(Struct(
+        ...         "data" / Bytes(4),
+        ...     )),
+        ... )
+        >>> outer.parse(b"1234")
+        Container(data=b'1234')
+    """
+
+    def __init__(self, subcon):
+        super().__init__(subcon)
+
+
+@singleton
+def CStringBytes():
+    """
+    A stripped back version of CString that returns bytes instead of a unicode string.
+    """
+    return NullTerminated(GreedyBytes)

elftools/common/utils.py

@@ -8,7 +8,7 @@
 #-------------------------------------------------------------------------------
 from contextlib import contextmanager
 from .exceptions import ELFParseError, ELFError, DWARFError
-from ..construct import ConstructError, ULInt8
+from construct import ConstructError, Int8ul
 import os
 
 
@@ -108,7 +108,7 @@ def roundup(num, bits):
 def read_blob(stream, length):
     """Read length bytes from stream, return a list of ints
     """
-    return [struct_parse(ULInt8(''), stream) for i in range(length)]
+    return [struct_parse(Int8ul, stream) for i in range(length)]
 
 def save_dwarf_section(section, filename):
     """Debug helper: dump section contents into a file