Merge pull request #143 from Decompollaborate/develop

1.19.0

CHANGELOG.md

@@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.19.0] - 2023-12-04
+
+### Added
+
+- Try to detect when instructions are setting the `$gp` register with the
+  passed `GP_VALUE` and use the `_gp` symbol instead of symbolizing an
+  unrelated symbol.
+  - This check is only applied on non PIC mode.
+- Track if a symbol is accessed with a `%gp_rel`.
+- Add `--sequential-label-names` option to rename branch and jump table labels
+  after their containing function (#142).
+  - Thanks @cadmic
+
+### Changed
+
+- Emit which symbol produced the generation of an automatically generated and
+  unused pad.
+- Change how the variables csv file is parsed: If the size column is either
+  empty, a dash (`-`) or zero then that column is ignored.
+- Autogenerated symbol names are now padded up to 8 hex digits, contrary to the
+  old 6 hex digits.
+  - Since this may be a breaking change for some people the old functionallity
+    can be restored by enabling the `GlobalConfig.LEGACY_SYM_ADDR_ZERO_PADDING`
+    setting, or by passing `--legacy-sym-addr-zero-padding` when invoking a
+    command line front-end.
+
+### Deprecated
+
+- `GlobalConfig.LEGACY_SYM_ADDR_ZERO_PADDING` (and the CLI version
+  `--legacy-sym-addr-zero-padding`) are deprecated.
+  - There's no replacement for this option.
+
+### Fixed
+
+- Avoid using the wrong immediate for gp-relative references if the address
+  could not be symbolized.
+  - In example, if said address is outside of the known address range.
+- Fix some niche cases where spimdisasm may emit duplicated data symbol labels
+  but without their data.
+
 ## [1.18.0] - 2023-10-29
 
 ### Added
@@ -200,13 +240,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - The new option is used via the API
     `GlobalConfig.RODATA_STRING_GUESSER_LEVEL` or via the CLI
     `--rodata-string-guesser level`.
-  - The old `GlobalConfig.STRING_GUESSER` and
-    `GlobalConfig.AGGRESSIVE_STRING_GUESSER` options are now deprecated, same as
-    the CLI options `--string-guesser` and `--aggressive-string-guesser`.
-    - The old `GlobalConfig.STRING_GUESSER = True` is equivalent to the new
-      `GlobalConfig.RODATA_STRING_GUESSER_LEVEL = 1`
-    - The old `GlobalConfig.AGGRESSIVE_STRING_GUESSER = True` is equivalent to
-      the new `GlobalConfig.RODATA_STRING_GUESSER_LEVEL = 4`
   - Meaning of the new levels:
     - level 0: Completely disable the guessing feature.
     - level 1: The most conservative guessing level. Imposes the following restrictions:
@@ -225,6 +258,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Emit `jlabel` instead of `dlabel` for jumptable labels by default
 - Emit `dlabel` instead of `dlabel` for data, rodata and bss symbols by default
 
+### Deprecated
+
+- The old `GlobalConfig.STRING_GUESSER` and
+  `GlobalConfig.AGGRESSIVE_STRING_GUESSER` options are now deprecated, same as
+  the CLI options `--string-guesser` and `--aggressive-string-guesser`.
+  - The old `GlobalConfig.STRING_GUESSER = True` is equivalent to the new
+    `GlobalConfig.RODATA_STRING_GUESSER_LEVEL = 1`
+  - The old `GlobalConfig.AGGRESSIVE_STRING_GUESSER = True` is equivalent to
+    the new `GlobalConfig.RODATA_STRING_GUESSER_LEVEL = 4`
+
 ## [1.14.3] - 2023-06-19
 
 ### Added
@@ -1261,6 +1304,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Version 1.0.0
 
 [unreleased]: https://github.com/Decompollaborate/spimdisasm/compare/master...develop
+[1.19.0]: https://github.com/Decompollaborate/spimdisasm/compare/1.18.0...1.19.0
 [1.18.0]: https://github.com/Decompollaborate/spimdisasm/compare/1.17.4...1.18.0
 [1.17.4]: https://github.com/Decompollaborate/spimdisasm/compare/1.17.3...1.17.4
 [1.17.3]: https://github.com/Decompollaborate/spimdisasm/compare/1.17.2...1.17.3

pyproject.toml

@@ -4,7 +4,7 @@
 [project]
 name = "spimdisasm"
 # Version should be synced with spimdisasm/__init__.py
-version = "1.18.0"
+version = "1.19.0"
 description = "MIPS disassembler"
 # license = "MIT"
 readme = "README.md"

spimdisasm/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__: tuple[int, int, int] = (1, 18, 0)
+__version_info__: tuple[int, int, int] = (1, 19, 0)
 __version__ = ".".join(map(str, __version_info__))
 __author__ = "Decompollaborate"
 

spimdisasm/common/ContextSymbols.py

@@ -12,6 +12,7 @@
 
 from .GlobalConfig import GlobalConfig, Compiler
 from .FileSectionType import FileSectionType
+from .SortedDict import SortedDict
 
 
 class SymbolSpecialType(enum.Enum):
@@ -132,6 +133,13 @@ class ContextSymbol:
     referenceSymbols: set[ContextSymbol] = dataclasses.field(default_factory=set)
     "Which symbols reference this symbol"
 
+    parentFunction: ContextSymbol|None = None
+    "Parent function for branch labels, jump tables, and jump table labels"
+    branchLabels: SortedDict[ContextSymbol] = dataclasses.field(default_factory=SortedDict)
+    "For functions, the branch and jump table labels which are contained in this function"
+    jumpTables: SortedDict[ContextSymbol] = dataclasses.field(default_factory=SortedDict)
+    "For functions, the jump tables which are contained in this function"
+
     overlayCategory: str|None = None
 
     nameGetCallback: Callable[[ContextSymbol], str]|None = None
@@ -149,9 +157,12 @@ class ContextSymbol:
     isGotLocal: bool = False
     gotIndex: int|None = None
 
+    accessedAsGpRel: bool = False
+
     _isStatic: bool = False
 
     isAutoCreatedPad: bool = False
+    autoCreatedPadMainSymbol: ContextSymbol|None = None
 
     firstLoAccess: int|None = None
 
@@ -377,6 +388,21 @@ def isLateRodata(self) -> bool:
     def hasUserDeclaredSize(self) -> bool:
         return self.userDeclaredSize is not None
 
+    def getBranchLabelName(self, suffix: str) -> str:
+        if GlobalConfig.SEQUENTIAL_LABEL_NAMES and self.parentFunction is not None:
+            index = self.parentFunction.branchLabels.index(self.vram)
+            if index is not None:
+                return f".L{self.parentFunction.getName()}_{index + 1}"
+
+        return f".L{self.address:08X}{suffix}"
+
+    def getJumpTableName(self, suffix: str) -> str:
+        if GlobalConfig.SEQUENTIAL_LABEL_NAMES and self.parentFunction is not None:
+            index = self.parentFunction.jumpTables.index(self.vram)
+            if index is not None:
+                return f"jtbl_{self.parentFunction.getName()}_{index + 1}"
+
+        return f"jtbl_{self.address:08X}{suffix}"
 
     def getDefaultName(self) -> str:
         suffix = ""
@@ -394,17 +420,23 @@ def getDefaultName(self) -> str:
             if currentType == SymbolSpecialType.function:
                 return f"func_{self.address:08X}{suffix}"
             if currentType in {SymbolSpecialType.branchlabel, SymbolSpecialType.jumptablelabel}:
-                return f".L{self.address:08X}{suffix}"
+                return self.getBranchLabelName(suffix)
             if currentType == SymbolSpecialType.jumptable:
-                return f"jtbl_{self.address:08X}{suffix}"
+                return self.getJumpTableName(suffix)
 
         if GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE:
             if self.sectionType == FileSectionType.Rodata:
-                return f"RO_{self.address:06X}{suffix}"
+                if GlobalConfig.LEGACY_SYM_ADDR_ZERO_PADDING:
+                    return f"RO_{self.address:06X}{suffix}"
+                return f"RO_{self.address:08X}{suffix}"
             if self.sectionType == FileSectionType.Bss:
-                return f"B_{self.address:06X}{suffix}"
+                if GlobalConfig.LEGACY_SYM_ADDR_ZERO_PADDING:
+                    return f"B_{self.address:06X}{suffix}"
+                return f"B_{self.address:08X}{suffix}"
 
-        return f"D_{self.address:06X}{suffix}"
+        if GlobalConfig.LEGACY_SYM_ADDR_ZERO_PADDING:
+            return f"D_{self.address:06X}{suffix}"
+        return f"D_{self.address:08X}{suffix}"
 
     def getName(self) -> str:
         if self.nameGetCallback is not None:
@@ -526,7 +558,10 @@ def getLabelMacro(self, isInMiddleLabel: bool=False) -> str|None:
             if self.isStatic():
                 label += f"# static variable{GlobalConfig.LINE_ENDS}"
             if self.isAutogeneratedPad():
-                label += f"# Automatically generated and unreferenced pad{GlobalConfig.LINE_ENDS}"
+                mainSymbolInfo = ""
+                if self.autoCreatedPadMainSymbol is not None:
+                    mainSymbolInfo = f" (generated by the size of {self.autoCreatedPadMainSymbol.getName()})"
+                label += f"# Automatically generated and unreferenced pad{mainSymbolInfo}{GlobalConfig.LINE_ENDS}"
 
         currentType = self.getTypeSpecial()
         if currentType == SymbolSpecialType.jumptablelabel:
@@ -575,8 +610,8 @@ def getCsvHeader() -> str:
 
         output += "isDefined,isUserDeclared,isAutogenerated,isMaybeString,isMaybePascalString,"
         output += "referenceCounter,overlayCategory,unknownSegment,"
-        output += "isGot,isGotGlobal,isGotLocal,gotIndex,"
-        output += "firstLoAccess,isAutogeneratedPad,isElfNotype,"
+        output += "isGot,isGotGlobal,isGotLocal,gotIndex,accessedAsGpRel,"
+        output += "firstLoAccess,isAutogeneratedPad,autoCreatedPadMainSymbol,isElfNotype,"
         output += "isAutocreatedSymFromOtherSizedSym,isMips1Double"
         return output
 
@@ -601,8 +636,11 @@ def toCsv(self) -> str:
         output += f"0x{self.getSize():X},0x{self.getVrom():X},{self.sectionType.toStr()},"
         output += f"{self.isDefined},{self.isUserDeclared},{self.isAutogenerated},{self.isMaybeString},{self.isMaybePascalString},"
         output += f"{self.referenceCounter},{self.overlayCategory},{self.unknownSegment},"
-        output += f"{self.isGot},{self.isGotGlobal},{self.isGotLocal},{self.gotIndex},"
-        output += f"{self.firstLoAccess},{self.isAutogeneratedPad()},{self.isElfNotype},"
+        output += f"{self.isGot},{self.isGotGlobal},{self.isGotLocal},{self.gotIndex},{self.accessedAsGpRel},"
+        autoCreatedPadMainSymbolName = ""
+        if self.autoCreatedPadMainSymbol is not None:
+            autoCreatedPadMainSymbolName = self.autoCreatedPadMainSymbol.getName()
+        output += f"{self.firstLoAccess},{self.isAutogeneratedPad()},{autoCreatedPadMainSymbolName},{self.isElfNotype},"
         output += f"{self.isAutocreatedSymFromOtherSizedSym},{self.isMips1Double}"
         return output
 

spimdisasm/common/GlobalConfig.py

@@ -160,6 +160,17 @@ def AGGRESSIVE_STRING_GUESSER(self, value: bool) -> None:
 
     Use STR_ for strings, FLT_ for floats and DBL_ for doubles"""
 
+    SEQUENTIAL_LABEL_NAMES: bool = False
+    """Name branch and jump table labels after their containing function and a sequential number"""
+
+    #! @deprecated
+    LEGACY_SYM_ADDR_ZERO_PADDING: bool = False
+    """
+    Restore the legacy behavior of padding up to 6 digits with zeroes the autogenerated symbol names.
+    The current behavior is to pad up to 8 digits with zeroes.
+    This option is deprecated and may be removed in the future.
+    """
+
     CUSTOM_SUFFIX: str = ""
 
     COMPILER: Compiler = Compiler.IDO
@@ -309,6 +320,9 @@ def addParametersToArgParse(self, parser: argparse.ArgumentParser):
 
         backendConfig.add_argument("--name-vars-by-section", help=f"Toggles the naming-after-section feature for autogenerated names. This means autogenerated symbols get a RO_ or B_ prefix if the symbol is from a rodata or bss section. Defaults to {self.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE}", action=Utils.BooleanOptionalAction)
         backendConfig.add_argument("--name-vars-by-type", help=f"Toggles the naming-after-type feature for autogenerated names. This means autogenerated symbols can get a STR_, FLT_ or DBL_ prefix if the symbol is a string, float or double. Defaults to {self.AUTOGENERATED_NAMES_BASED_ON_DATA_TYPE}", action=Utils.BooleanOptionalAction)
+        backendConfig.add_argument("--sequential-label-names", help=f"Toggles naming branch and jump table labels after their containing function and a sequential number. Defaults to {self.SEQUENTIAL_LABEL_NAMES}", action=Utils.BooleanOptionalAction)
+
+        backendConfig.add_argument("--legacy-sym-addr-zero-padding", help=f"Restore the legacy behavior of padding up to 6 digits with zeroes the autogenerated symbol names. The current behavior is to pad up to 8 digits with zeroes. This option is deprecated and may be removed in the future. Defaults to {self.LEGACY_SYM_ADDR_ZERO_PADDING}", action=Utils.BooleanOptionalAction)
 
         backendConfig.add_argument("--custom-suffix", help="Set a custom suffix for automatically generated symbols")
 
@@ -445,6 +459,11 @@ def parseArgs(self, args: argparse.Namespace):
             self.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE = args.name_vars_by_section
         if args.name_vars_by_type is not None:
             self.AUTOGENERATED_NAMES_BASED_ON_DATA_TYPE = args.name_vars_by_type
+        if args.sequential_label_names is not None:
+            self.SEQUENTIAL_LABEL_NAMES = args.sequential_label_names
+
+        if args.legacy_sym_addr_zero_padding is not None:
+            self.LEGACY_SYM_ADDR_ZERO_PADDING = args.legacy_sym_addr_zero_padding
 
         if args.custom_suffix:
             self.CUSTOM_SUFFIX = args.custom_suffix

spimdisasm/common/Relocation.py

@@ -37,6 +37,12 @@ class RelocType(enum.Enum):
     CUSTOM_CONSTANT_HI  = -1
     CUSTOM_CONSTANT_LO  = -2
 
+    def getPercentRel(self) -> str|None:
+        return _percentRel.get(self)
+
+    def getWordRel(self) -> str|None:
+        return _wordRel.get(self)
+
     @staticmethod
     def fromValue(value: int) -> RelocType|None:
         try:
@@ -181,11 +187,11 @@ def getName(self, isSplittedSymbol: bool=False) -> str:
     def getNameWithReloc(self, isSplittedSymbol: bool=False) -> str:
         name = self.getName(isSplittedSymbol=isSplittedSymbol)
 
-        percentRel = _percentRel.get(self.relocType)
+        percentRel = self.relocType.getPercentRel()
         if percentRel is not None:
             return f"{percentRel}({name})"
 
-        wordRel = _wordRel.get(self.relocType)
+        wordRel = self.relocType.getWordRel()
         if wordRel is not None:
             return f"{wordRel} {name}"
 

spimdisasm/common/SortedDict.py

@@ -111,6 +111,12 @@ def getRangeAndPop(self, startKey: int, endKey: int, startInclusive: bool=True,
             self.remove(key)
             yield (key, value)
 
+    def index(self, key: int) -> int|None:
+        """Returns the index of the passed `key` in the sorted dictionary, or None if the key is not present."""
+        if key not in self.map:
+            return None
+        return bisect.bisect_left(self.sortedKeys, key)
+
     def __getitem__(self, key: int) -> ValueType:
         return self.map[key]
 

spimdisasm/common/SymbolsSegment.py

@@ -21,7 +21,7 @@ class SymbolsSegment:
     def __init__(self, context: "Context", vromStart: int|None, vromEnd: int|None, vramStart: int, vramEnd: int, overlayCategory: str|None=None):
         assert vramStart < vramEnd
         if vromStart is not None and vromEnd is not None:
-            assert vromStart <= vromEnd, f"0x{vromStart:X} <= 0x{vromEnd:X}"
+            assert vromStart <= vromEnd, f"0x{vromStart:06X} <= 0x{vromEnd:06X}"
 
         self.vromStart: int|None = vromStart
         self.vromEnd: int|None = vromEnd
@@ -67,8 +67,8 @@ def isVramInRange(self, vram: int) -> bool:
         return self.vramStart <= vram < self.vramEnd
 
     def changeRanges(self, vromStart: int, vromEnd: int, vramStart: int, vramEnd: int) -> None:
-        assert vromStart <= vromEnd, f"0x{vromStart:X} <= 0x{vromEnd:X}"
-        assert vramStart <= vramEnd, f"0x{vramStart:X} <= 0x{vramEnd:X}"
+        assert vromStart <= vromEnd, f"0x{vromStart:06X} <= 0x{vromEnd:06X}"
+        assert vramStart <= vramEnd, f"0x{vramStart:08X} <= 0x{vramEnd:08X}"
 
         self.vromStart = vromStart
         self.vromEnd = vromEnd
@@ -430,8 +430,12 @@ def readVariablesCsv(self, filepath: Path):
                 continue
 
             vram = int(vramStr, 16)
-            varSize = int(varSizeStr, 16)
-            if varType == "":
+            varSize = None
+            if varSizeStr not in {"", "-"}:
+                varSize = int(varSizeStr, 16)
+                if varSize == 0:
+                    varSize = None
+            if varType in {"", "-"}:
                 varType = None
 
             specialType = SymbolSpecialType.fromStr(varType)

spimdisasm/elf32/Elf32File.py

@@ -218,6 +218,12 @@ def _processSection_PROGBITS(self, array_of_bytes: bytes, entry: Elf32SectionHea
         elif Elf32SectionHeaderFlag.STRINGS in flags:
             # Why not?
             self.progbitsNoWrite[sectionEntryName] = entry
+        elif sectionEntryName == ".comment":
+            # Usually strings specifying the compiler used to build
+            pass
+        elif sectionEntryName == ".pdr":
+            # Debugging section, dunno what it actually has
+            pass
         elif not common.GlobalConfig.QUIET:
             common.Utils.eprint(f"Unhandled PROGBITS found: '{sectionEntryName}', flags: {flags}, unknownFlags: {unknownFlags}\n")
 

spimdisasm/mips/sections/MipsSectionBss.py

@@ -62,7 +62,7 @@ def analyze(self):
                 newSymbolVram = symbolVram + contextSym.getSize()
                 if newSymbolVram != self.bssVramEnd:
                     assert newSymbolVram >= self.bssVramStart
-                    assert newSymbolVram < self.bssVramEnd, f"{self.name}, symbolVram={symbolVram:X}, newSymbolVram={newSymbolVram:X}, self.bssVramEnd={self.bssVramEnd:X}"
+                    assert newSymbolVram < self.bssVramEnd, f"{self.name}, symbolVram={symbolVram:08X}, newSymbolVram={newSymbolVram:08X}, self.bssVramEnd={self.bssVramEnd:08X}"
                     symOffset = symbolVram + contextSym.getSize() - self.bssVramStart
                     bssSymbolOffsets.add(symOffset)
                     autoCreatedPads.add(symOffset)