Merge pull request #137 from Decompollaborate/develop

1.17.1

pyproject.toml

@@ -4,7 +4,7 @@
 [project]
 name = "spimdisasm"
 # Version should be synced with spimdisasm/__init__.py
-version = "1.17.0"
+version = "1.17.1"
 description = "MIPS disassembler"
 # license = "MIT"
 readme = "README.md"

spimdisasm/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__: tuple[int, int, int] = (1, 17, 0)
+__version_info__: tuple[int, int, int] = (1, 17, 1)
 __version__ = ".".join(map(str, __version_info__))
 __author__ = "Decompollaborate"
 

spimdisasm/common/Context.py

@@ -17,7 +17,7 @@
 
 
 @dataclasses.dataclass
-class SymbolRange:
+class AddressRange:
     start: int
     end: int
 
@@ -34,6 +34,34 @@ def increaseEnd(self, address: int) -> None:
             self.end = address
         return None
 
+class SymbolsRanges:
+    def __init__(self, start: int, end: int):
+        self.mainAddressRange = AddressRange(start, end)
+        self.specialRanges: list[AddressRange] = list()
+
+    def isInRange(self, address: int) -> bool:
+        if self.mainAddressRange.isInRange(address):
+            return True
+
+        for spRange in self.specialRanges:
+            if spRange.isInRange(address):
+                return True
+
+        return False
+
+    def decreaseStart(self, address: int) -> None:
+        self.mainAddressRange.decreaseStart(address)
+
+    def increaseEnd(self, address: int) -> None:
+        self.mainAddressRange.increaseEnd(address)
+
+    def addSpecialRange(self, start: int, end: int) -> AddressRange|None:
+        # print(f"addSpecialRange: {start:X} {end:X}")
+        if end <= start:
+            return None
+        addrRange = AddressRange(start, end)
+        self.specialRanges.append(addrRange)
+        return addrRange
 
 class Context:
     N64DefaultBanned = {
@@ -54,12 +82,12 @@ def __init__(self):
         self.overlaySegments: dict[str, dict[int, SymbolsSegment]] = dict()
         "Outer key is overlay type, inner key is the vrom of the overlay's segment"
 
-        self.totalVramRange: SymbolRange = SymbolRange(self.globalSegment.vramStart, self.globalSegment.vramEnd)
+        self.totalVramRange: SymbolsRanges = SymbolsRanges(self.globalSegment.vramStart, self.globalSegment.vramEnd)
         self._defaultVramRanges: bool = True
 
         # Stuff that looks like pointers, but the disassembler shouldn't count it as a pointer
         self.bannedSymbols: set[int] = set()
-        self.bannedRangedSymbols: list[SymbolRange] = list()
+        self.bannedRangedSymbols: list[AddressRange] = list()
 
         self.globalRelocationOverrides: dict[int, RelocationInfo] = dict()
         "key: vrom address"
@@ -74,8 +102,8 @@ def changeGlobalSegmentRanges(self, vromStart: int, vromEnd: int, vramStart: int
             Utils.eprint(f"Warning: globalSegment's will has its vramStart equal to the vramEnd (0x{vramStart:X})")
         self.globalSegment.changeRanges(vromStart, vromEnd, vramStart, vramEnd)
         if self._defaultVramRanges:
-            self.totalVramRange.start = vramStart
-            self.totalVramRange.end = vramEnd
+            self.totalVramRange.mainAddressRange.start = vramStart
+            self.totalVramRange.mainAddressRange.end = vramEnd
             self._defaultVramRanges = False
         self.totalVramRange.decreaseStart(vramStart)
         self.totalVramRange.increaseEnd(vramEnd)
@@ -87,14 +115,20 @@ def addOverlaySegment(self, overlayCategory: str, segmentVromStart: int, segment
         self.overlaySegments[overlayCategory][segmentVromStart] = segment
 
         if self._defaultVramRanges:
-            self.totalVramRange.start = segmentVramStart
-            self.totalVramRange.end = segmentVramEnd
+            self.totalVramRange.mainAddressRange.start = segmentVramStart
+            self.totalVramRange.mainAddressRange.end = segmentVramEnd
             self._defaultVramRanges = False
         self.totalVramRange.decreaseStart(segmentVramStart)
         self.totalVramRange.increaseEnd(segmentVramEnd)
 
         return segment
 
+    def isInTotalVramRange(self, address: int) -> bool:
+        return self.totalVramRange.isInRange(address)
+
+    def addSpecialVramRange(self, start: int, end: int) -> AddressRange|None:
+        return self.totalVramRange.addSpecialRange(start, end)
+
 
     def initGotTable(self, pltGot: int, localsTable: list[int], globalsTable: list[int]):
         self.gpAccesses.initGotTable(pltGot, localsTable, globalsTable)
@@ -120,10 +154,10 @@ def addBannedSymbol(self, address: int):
         self.bannedSymbols.add(address)
 
     def addBannedSymbolRange(self, rangeStart: int, rangeEnd: int):
-        self.bannedRangedSymbols.append(SymbolRange(rangeStart, rangeEnd))
+        self.bannedRangedSymbols.append(AddressRange(rangeStart, rangeEnd))
 
     def addBannedSymbolRangeBySize(self, rangeStart: int, size: int):
-        self.bannedRangedSymbols.append(SymbolRange(rangeStart, rangeStart + size))
+        self.bannedRangedSymbols.append(AddressRange(rangeStart, rangeStart + size))
 
     def isAddressBanned(self, address: int) -> bool:
         if address in self.bannedSymbols:

spimdisasm/common/ContextSymbols.py

@@ -180,6 +180,8 @@ class ContextSymbol:
 
     isAutocreatedSymFromOtherSizedSym: bool = False
 
+    isMips1Double: bool = False
+
 
     @property
     def vram(self) -> int:
@@ -575,7 +577,7 @@ def getCsvHeader() -> str:
         output += "referenceCounter,overlayCategory,unknownSegment,"
         output += "isGot,isGotGlobal,isGotLocal,gotIndex,"
         output += "firstLoAccess,isAutogeneratedPad,isElfNotype,"
-        output += "isAutocreatedSymFromOtherSizedSym"
+        output += "isAutocreatedSymFromOtherSizedSym,isMips1Double"
         return output
 
     def toCsv(self) -> str:
@@ -601,7 +603,7 @@ def toCsv(self) -> str:
         output += f"{self.referenceCounter},{self.overlayCategory},{self.unknownSegment},"
         output += f"{self.isGot},{self.isGotGlobal},{self.isGotLocal},{self.gotIndex},"
         output += f"{self.firstLoAccess},{self.isAutogeneratedPad()},{self.isElfNotype},"
-        output += f"{self.isAutocreatedSymFromOtherSizedSym}"
+        output += f"{self.isAutocreatedSymFromOtherSizedSym},{self.isMips1Double}"
         return output
 
 

spimdisasm/common/GlobalConfig.py

@@ -198,9 +198,11 @@ def AGGRESSIVE_STRING_GUESSER(self, value: bool) -> None:
     EMIT_INLINE_RELOC: bool = False
 
     SYMBOL_FINDER_FILTER_LOW_ADDRESSES: bool = True
-    """Toggle pointer detection for lower addresses (lower than 0x40000000)"""
+    """Toggle pointer detection for lower addresses (lower than SYMBOL_FINDER_FILTER_ADDRESSES_ADDR_LOW)"""
+    SYMBOL_FINDER_FILTER_ADDRESSES_ADDR_LOW: int = 0x80000000
     SYMBOL_FINDER_FILTER_HIGH_ADDRESSES: bool = True
-    """Toggle pointer detection for higher addresses (higher than 0xC0000000)"""
+    """Toggle pointer detection for higher addresses (higher than SYMBOL_FINDER_FILTER_ADDRESSES_ADDR_HIGH)"""
+    SYMBOL_FINDER_FILTER_ADDRESSES_ADDR_HIGH: int = 0xC0000000
     SYMBOL_FINDER_FILTERED_ADDRESSES_AS_CONSTANTS: bool = True
     """Treat filtered out addresses as constants pairs"""
     SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO: bool = False

spimdisasm/common/SymbolsSegment.py

@@ -357,24 +357,40 @@ def saveContextToFile(self, f: TextIO):
 
 
     def fillLibultraSymbols(self):
+        lowestVram = 0xFFFFFFFF
+        highestVram = 0x00000000
         for vram, (name, type, size) in self.N64LibultraSyms.items():
             contextSym = self.addSymbol(vram)
             contextSym.name = name
             contextSym.setTypeSpecial(type, isAutogenerated=False)
             contextSym.userDeclaredSize = size
             contextSym.isDefined = True
             contextSym.isUserDeclared = True
+            if vram > highestVram:
+                highestVram = vram
+            if vram < lowestVram:
+                lowestVram = vram
+        self.context.totalVramRange.addSpecialRange(lowestVram, highestVram)
 
     def fillIQueSymbols(self):
+        lowestVram = 0xFFFFFFFF
+        highestVram = 0x00000000
         for vram, (name, type, size) in self.iQueLibultraSyms.items():
             contextSym = self.addSymbol(vram)
             contextSym.name = name
             contextSym.setTypeSpecial(type, isAutogenerated=False)
             contextSym.userDeclaredSize = size
             contextSym.isDefined = True
             contextSym.isUserDeclared = True
+            if vram > highestVram:
+                highestVram = vram
+            if vram < lowestVram:
+                lowestVram = vram
+        self.context.totalVramRange.addSpecialRange(lowestVram, highestVram)
 
     def fillHardwareRegs(self, useRealNames: bool=False):
+        lowestVram = 0xFFFFFFFF
+        highestVram = 0x00000000
         for vram, name in self.N64HardwareRegs.items():
             nameToUse = None
             if useRealNames:
@@ -392,6 +408,11 @@ def fillHardwareRegs(self, useRealNames: bool=False):
                 contextSym.userDeclaredSize = 4
                 contextSym.isDefined = True
                 contextSym.isUserDeclared = True
+            if vram > highestVram:
+                highestVram = vram
+            if vram < lowestVram:
+                lowestVram = vram
+        self.context.totalVramRange.addSpecialRange(lowestVram, highestVram)
 
 
     def readVariablesCsv(self, filepath: Path):

spimdisasm/elf32/Elf32File.py

@@ -218,8 +218,8 @@ def _processSection_PROGBITS(self, array_of_bytes: bytes, entry: Elf32SectionHea
         elif Elf32SectionHeaderFlag.STRINGS in flags:
             # Why not?
             self.progbitsNoWrite[sectionEntryName] = entry
-        else:
-            common.Utils.eprint(f"Unhandled PROGBITS found: '{sectionEntryName}', flags: {flags}\n")
+        elif not common.GlobalConfig.QUIET:
+            common.Utils.eprint(f"Unhandled PROGBITS found: '{sectionEntryName}', flags: {flags}, unknownFlags: {unknownFlags}\n")
 
     def _processSection_SYMTAB(self, array_of_bytes: bytes, entry: Elf32SectionHeaderEntry, sectionEntryName: str) -> None:
         if sectionEntryName == ".symtab":

spimdisasm/mips/sections/MipsSectionBase.py

@@ -16,8 +16,16 @@ def checkWordIsASymbolReference(self, word: int) -> bool:
         if self.context.isAddressBanned(word):
             return False
 
-        if self.getSymbol(word, tryPlusOffset=True, checkUpperLimit=True) is not None:
-            return False
+        contextSym = self.getSymbol(word, tryPlusOffset=True, checkUpperLimit=False)
+        if contextSym is not None:
+            symType = contextSym.getTypeSpecial()
+            if symType in {common.SymbolSpecialType.function, common.SymbolSpecialType.branchlabel, common.SymbolSpecialType.jumptablelabel}:
+                # Avoid generating extra symbols in the middle of functions
+                return False
+
+            if word < contextSym.vram + contextSym.getSize():
+                # Avoid generating symbols in the middle of other symbols with known sizes
+                return False
 
         self.addPointerInDataReference(word)
         return True

spimdisasm/mips/sections/MipsSectionData.py

@@ -65,9 +65,11 @@ def analyze(self):
                 currentVram = self.getVramOffset(localOffset)
                 currentVrom = self.getVromOffset(localOffset)
 
-                contextSym = self.getSymbol(currentVram, vromAddress=currentVrom, tryPlusOffset=True, checkUpperLimit=True)
-                if contextSym is None and self.popPointerInDataReference(currentVram) is not None:
-                    contextSym = self.addSymbol(currentVram, sectionType=self.sectionType, isAutogenerated=True)
+                if self.popPointerInDataReference(currentVram) is not None:
+                    contextSym = self.getSymbol(currentVram, vromAddress=currentVrom, tryPlusOffset=True, checkUpperLimit=True)
+                    if contextSym is None:
+                        contextSym = self.addSymbol(currentVram, sectionType=self.sectionType, isAutogenerated=True)
+                    contextSym.sectionType = self.sectionType
                     contextSym.isMaybeString = self._stringGuesser(contextSym, localOffset)
                     contextSym.isMaybePascalString = self._pascalStringGuesser(contextSym, localOffset)
                     symbolList.append((localOffset, contextSym))

spimdisasm/mips/sections/MipsSectionText.py

@@ -144,10 +144,13 @@ def _findFunctions_checkFunctionEnded(self, instructionOffset: int, instr: rabbi
                         # If this instruction is a jump and it is jumping to a function then
                         # we can consider this as a function end. This can happen as a
                         # tail-optimization in modern compilers
-                        targetVram = instr.getInstrIndexAsVram()
-                        auxSym = self.getSymbol(targetVram, tryPlusOffset=False, checkGlobalSegment=False)
-                        if auxSym is not None and auxSym.isTrustableFunction(self.instrCat == rabbitizer.InstrCategory.RSP):
+                        if not rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch:
                             functionEnded = True
+                        else:
+                            targetVram = instr.getInstrIndexAsVram()
+                            auxSym = self.getSymbol(targetVram, tryPlusOffset=False, checkGlobalSegment=False)
+                            if auxSym is not None and auxSym.isTrustableFunction(self.instrCat == rabbitizer.InstrCategory.RSP):
+                                functionEnded = True
 
         return functionEnded, prevFuncHadUserDeclaredSize
 

spimdisasm/mips/symbols/MipsSymbolFunction.py

@@ -460,6 +460,7 @@ def analyze(self):
                                     contextSym.isGotGlobal = gotGlobal
                                 contextSym.accessType = rabbitizer.AccessType.DOUBLEFLOAT
                                 contextSym.unsignedAccessType = False
+                                contextSym.isMips1Double = True
 
         self._generateRelocsFromInstructionAnalyzer()
 
@@ -647,7 +648,7 @@ def getLabelForOffset(self, instructionOffset: int, migrate: bool=False) -> str:
         labelSym.isDefined = True
         labelSym.sectionType = self.sectionType
         labelSymType = labelSym.getTypeSpecial()
-        if labelSymType == common.SymbolSpecialType.function or (labelSymType == common.SymbolSpecialType.jumptablelabel and not migrate):
+        if labelSymType is None or labelSymType == common.SymbolSpecialType.function or (labelSymType == common.SymbolSpecialType.jumptablelabel and not migrate):
             label = labelSym.getReferenceeSymbols()
             labelMacro = labelSym.getLabelMacro(isInMiddleLabel=True)
             if labelMacro is not None:
@@ -740,10 +741,10 @@ def disassemble(self, migrate: bool=False, useGlobalLabel: bool=True, isSplitted
             instructionOffset += 4
 
             if instructionOffset == symSize:
-                output += self.getSizeDirective(symName)
+                if common.GlobalConfig.ASM_TEXT_END_LABEL:
+                    output += f"{common.GlobalConfig.ASM_TEXT_END_LABEL} {self.getName()}" + common.GlobalConfig.LINE_ENDS
 
-        if common.GlobalConfig.ASM_TEXT_END_LABEL:
-            output += f"{common.GlobalConfig.ASM_TEXT_END_LABEL} {self.getName()}" + common.GlobalConfig.LINE_ENDS
+                output += self.getSizeDirective(symName)
 
         nameEnd = self.getNameEnd()
         if nameEnd is not None:

spimdisasm/mips/symbols/MipsSymbolRodata.py

@@ -66,6 +66,9 @@ def shouldMigrate(self) -> bool:
         if self.contextSym.forceNotMigration:
             return False
 
+        if self.contextSym.isMips1Double:
+            return True
+
         if self.isRdata():
             if common.GlobalConfig.COMPILER not in {common.Compiler.SN64, common.Compiler.PSYQ}:
                 return False

spimdisasm/mips/symbols/analysis/InstrAnalyzer.py

@@ -236,9 +236,15 @@ def processSymbol(self, address: int, luiOffset: int|None, lowerInstr: rabbitize
                 return None
 
         # filter out stuff that may not be a real symbol
-        filterOut = common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES and address < 0x80000000
-        filterOut |= common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES and address >= 0xC0000000
-        if filterOut and lowerInstr.uniqueId != rabbitizer.InstrId.cpu_addiu:
+        filterOut = False
+        if not self.context.totalVramRange.isInRange(address):
+            if common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES or common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES:
+                filterOut |= common.GlobalConfig.SYMBOL_FINDER_FILTER_LOW_ADDRESSES and address < common.GlobalConfig.SYMBOL_FINDER_FILTER_ADDRESSES_ADDR_LOW
+                filterOut |= common.GlobalConfig.SYMBOL_FINDER_FILTER_HIGH_ADDRESSES and address >= common.GlobalConfig.SYMBOL_FINDER_FILTER_ADDRESSES_ADDR_HIGH
+            else:
+                filterOut |= True
+
+        if address > 0 and filterOut and lowerInstr.uniqueId != rabbitizer.InstrId.cpu_addiu:
             if common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_CONSTANTS:
                 # Let's pretend this value is a constant
                 constant = address
@@ -356,7 +362,7 @@ def symbolFinder(self, regsTracker: rabbitizer.RegistersTracker, instr: rabbitiz
             luiInstr = self.luiInstrs.get(luiOffset)
             if luiInstr is not None:
                 if luiInstr.rt in {rabbitizer.RegGprO32.gp, rabbitizer.RegGprN32.gp}:
-                    if instr.rs in {rabbitizer.RegGprO32.gp, rabbitizer.RegGprN32.gp} and instr.rt in {rabbitizer.RegGprO32.gp, rabbitizer.RegGprN32.gp}:
+                    if instr.readsRs() and instr.rs in {rabbitizer.RegGprO32.gp, rabbitizer.RegGprN32.gp} and instr.modifiesRt() and instr.rt in {rabbitizer.RegGprO32.gp, rabbitizer.RegGprN32.gp}:
                         # cpload
                         self.unpairedCploads.append(CploadInfo(luiOffset, instrOffset))
                         # early return to avoid counting this pairing as a symbol