diff --git a/LICENSE b/LICENSE index 10cf9e2d..b5d6dbf2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017-2021 Markus Gaasedelen +Copyright (c) 2017-2024 Markus Gaasedelen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index deac1bf2..b7fad83b 100644 --- a/README.md +++ b/README.md @@ -39,22 +39,13 @@ Use the instructions below for your respective disassembler. ## Binary Ninja Installation -Lighthouse can be installed through the plugin manager on newer versions of Binary Ninja (>2.4.2918). The plugin will have to be installed manually on older versions. - -### Auto Install +Lighthouse can be installed through the plugin manager on Binary Ninja, supporting v3.5 and newer. 1. Open Binary Ninja's plugin manager by navigating the following submenus: - `Edit` -> `Preferences` -> `Manage Plugins` 2. Search for Lighthouse in the plugin manager, and click the `Enable` button in the bottom right. 3. Restart your disassembler. -### Manual Install - -1. Open Binary Ninja's plugin folder by navigating the following submenus: - - `Tools` -> `Open Plugins Folder...` -2. Copy the contents of this repository's `/plugins/` folder to the listed directory. -3. Restart your disassembler. - # Usage Once properly installed, there will be a few new menu entries available in the disassembler. These are the entry points for a user to load coverage data and start using Lighthouse. @@ -203,6 +194,7 @@ Lighthouse will remember your theme preference for future loads and uses. Time and motivation permitting, future work may include: +* Nag Vector35 to fix HLIL highlighting ([bug](https://github.com/Vector35/binaryninja-api/issues/2584)) in Binary Ninja * ~~Asynchronous composition, painting, metadata collection~~ * ~~Multifile/coverage support~~ * Profiling based heatmaps/painting diff --git a/binjastub/plugin.json b/binjastub/plugin.json index 696b50ba..bff03c14 100644 --- a/binjastub/plugin.json +++ b/binjastub/plugin.json @@ -6,10 +6,10 @@ "description": "A Coverage Explorer for Reverse Engineers", "license": { "name": "MIT", - "text": "Copyright (c) 2021> Markus Gaasedelen\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." + "text": "Copyright (c) 2024> Markus Gaasedelen\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." }, "longdescription": "", - "minimumbinaryninjaversion": 2918, + "minimumbinaryninjaversion": 4526, "name": "Lighthouse", "platforms": [ "Darwin", @@ -20,5 +20,5 @@ "type": [ "helper" ], - "version": "0.9.2" + "version": "0.9.3" } \ No newline at end of file diff --git a/coverage/frida/frida-drcov.py b/coverage/frida/frida-drcov.py index bb060559..88be051d 100755 --- a/coverage/frida/frida-drcov.py +++ b/coverage/frida/frida-drcov.py @@ -65,7 +65,7 @@ var filtered_maps = new ModuleMap(function (m) { if (whitelist.indexOf('all') >= 0) { return true; } - return whitelist.indexOf(m.name) >= 0; + return whitelist.some(item => m.name.toLowerCase().includes(item.toLowerCase())); }); // This function takes a list of GumCompileEvents and converts it into a DRcov diff --git a/coverage/pin/README.md b/coverage/pin/README.md index dd8f2c07..e7da50d1 100644 --- a/coverage/pin/README.md +++ b/coverage/pin/README.md @@ -1,6 +1,6 @@ # CodeCoverage Pintool -The `CodeCoverage` pintool runs ontop of the [Intel Pin](https://software.intel.com/en-us/articles/pin-a-dynamic-binary-instrumentation-tool) DBI framework and collects code coverage data in a log format compatible with [Lighthouse](https://github.com/gaasedelen/lighthouse). The log produced by this pintool emulates that of [drcov](http://dynamorio.org/docs/page_drcov.html) as shipped with [DynamoRIO](http://www.dynamorio.org). +The `CodeCoverage` pintool runs ontop of the [Intel Pin](https://software.intel.com/en-us/articles/pin-a-dynamic-binary-instrumentation-tool) DBI framework and collects code coverage data in a log format compatible with [Lighthouse](https://github.com/gaasedelen/lighthouse). The log produced by this pintool emulates that of [drcov](http://dynamorio.org/docs/page_drcov.html) as shipped with [DynamoRIO](http://www.dynamorio.org). This pintool is labeled only as a prototype. @@ -12,7 +12,7 @@ Follow the build instructions below for your respective platform. ## Building for MacOS or Linux -On MacOS or Liunux, one can compile the pintool using the following commands. +On MacOS or Linux, one can compile the pintool using the following commands. ``` # Location of this repo / pintool source @@ -39,7 +39,11 @@ Launch a command prompt and build the pintool with the following commands. ### 32bit Pintool ``` -"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 +REM If you are on VS 2022 or so you can run this line: +"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x86 + +REM VS 2015 or so you can run this line instead: +REM "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 REM Location of this repo / pintool source cd C:\Users\user\lighthouse\coverage\pin @@ -53,7 +57,11 @@ build-x86.bat ### 64bit Pintool ``` -"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64 +REM If you are on VS 2022 or so you can run this line: +"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 + +REM VS 2015 or so you can run this line instead: +REM "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64 REM Location of this repo / pintool source cd C:\Users\user\lighthouse\coverage\pin @@ -64,7 +72,7 @@ set PATH=%PATH%;%PIN_ROOT% build-x64.bat ``` -The resulting binaries will be labaled based on their architecture (eg, 64 is the 64bit pintool). +The resulting binaries will be labeled based on their architecture (eg, 64 is the 64bit pintool). * CodeCoverage.dll * CodeCoverage64.dll diff --git a/coverage/pin/build-x64.bat b/coverage/pin/build-x64.bat index dd6476cd..0090054d 100644 --- a/coverage/pin/build-x64.bat +++ b/coverage/pin/build-x64.bat @@ -27,7 +27,7 @@ link ^ /LIBPATH:%PIN_ROOT%\intel64\lib ^ /LIBPATH:"%PIN_ROOT%\intel64\lib-ext" ^ /LIBPATH:"%PIN_ROOT%\extras\xed-intel64\lib" ^ - /LIBPATH:%PIN_ROOT%\intel64\runtime\pincrt pin.lib xed.lib pinvm.lib pincrt.lib ntdll-64.lib kernel32.lib crtbeginS.obj ^ + /LIBPATH:%PIN_ROOT%\intel64\runtime\pincrt pin.lib xed.lib pinipc.lib pincrt.lib kernel32.lib crtbeginS.obj ^ /NODEFAULTLIB ^ /MANIFEST:NO ^ /OPT:NOREF ^ diff --git a/coverage/pin/build-x86.bat b/coverage/pin/build-x86.bat index bc80c268..156886b7 100644 --- a/coverage/pin/build-x86.bat +++ b/coverage/pin/build-x86.bat @@ -28,7 +28,7 @@ link ^ /LIBPATH:%PIN_ROOT%\ia32\lib ^ /LIBPATH:"%PIN_ROOT%\ia32\lib-ext" ^ /LIBPATH:"%PIN_ROOT%\extras\xed-ia32\lib" ^ - /LIBPATH:%PIN_ROOT%\ia32\runtime\pincrt pin.lib xed.lib pinvm.lib pincrt.lib ntdll-32.lib kernel32.lib crtbeginS.obj ^ + /LIBPATH:%PIN_ROOT%\ia32\runtime\pincrt pin.lib xed.lib pinipc.lib pincrt.lib kernel32.lib crtbeginS.obj ^ /NODEFAULTLIB ^ /MANIFEST:NO ^ /OPT:NOREF ^ diff --git a/plugins/lighthouse/composer/shell.py b/plugins/lighthouse/composer/shell.py index 2696c61f..82a732fd 100644 --- a/plugins/lighthouse/composer/shell.py +++ b/plugins/lighthouse/composer/shell.py @@ -1045,7 +1045,7 @@ def _ui_init(self): # set the height of the textbox based on some arbitrary math :D LINE_PADDING = self.document().documentMargin()*2 line_height = self._font_metrics.height() + LINE_PADDING + 2 - self.setFixedHeight(line_height) + self.setFixedHeight(int(line_height)) #-------------------------------------------------------------------------- # QPlainTextEdit Overloads diff --git a/plugins/lighthouse/coverage.py b/plugins/lighthouse/coverage.py index f3ac1411..39224cb0 100644 --- a/plugins/lighthouse/coverage.py +++ b/plugins/lighthouse/coverage.py @@ -2,12 +2,11 @@ import time import logging import weakref -import datetime import itertools import collections from lighthouse.util import * -from lighthouse.util.qt import compute_color_on_gradiant +from lighthouse.util.qt import compute_color_on_gradient from lighthouse.metadata import DatabaseMetadata logger = logging.getLogger("Lighthouse.Coverage") @@ -97,7 +96,7 @@ def __init__(self, palette, name="", filepath=None, data=None): # the addresses executed in the coverage log # - self._hitmap = build_hitmap(data) + self._hitmap = collections.Counter(data) self._imagebase = BADADDR # @@ -140,9 +139,7 @@ def __init__(self, palette, name="", filepath=None, data=None): # initially, all loaded coverage data is marked as unmapped # - self._unmapped_data = set(self._hitmap.keys()) - self._unmapped_data.add(BADADDR) - self._misaligned_data = set() + self.unmapped_addresses = set(self._hitmap.keys()) # # at runtime, the map_coverage() member function of this class is @@ -166,9 +163,14 @@ def __init__(self, palette, name="", filepath=None, data=None): self.nodes = {} self.functions = {} self.instruction_percent = 0.0 + + # blocks that have not been fully executed (eg, crash / exception) self.partial_nodes = set() self.partial_instructions = set() + # addresses that have been executed, but are not in a defined node + self.orphan_addresses = set() + # # we instantiate a single weakref of ourself (the DatbaseCoverage # object) such that we can distribute it to the children we create @@ -191,7 +193,7 @@ def data(self): @property def coverage(self): """ - Return the instruction-level coverage bitmap/mask. + Return the coverage (address) bitmap/mask. """ return viewkeys(self._hitmap) @@ -266,6 +268,7 @@ def update_metadata(self, metadata, delta=None): if self._imagebase == BADADDR: self._imagebase = self._metadata.imagebase + self._normalize_coverage() # # if the imagebase for this coverage exists, then it is susceptible to @@ -301,9 +304,6 @@ def refresh(self): # update the coverage hash incase the hitmap changed self._update_coverage_hash() - # dump the unmappable coverage data - #self.dump_unmapped() - def refresh_theme(self): """ Refresh UI facing elements to reflect the current theme. @@ -311,7 +311,7 @@ def refresh_theme(self): Does not require @disassembler.execute_ui decorator as no Qt is touched. """ for function in self.functions.values(): - function.coverage_color = compute_color_on_gradiant( + function.coverage_color = compute_color_on_gradient( function.instruction_percent, self.palette.table_coverage_bad, self.palette.table_coverage_good @@ -390,7 +390,7 @@ def add_data(self, data, update=True): self._update_coverage_hash() # mark these touched addresses as dirty - self._unmapped_data |= viewkeys(data) + self.unmapped_addresses |= viewkeys(data) def add_addresses(self, addresses, update=True): """ @@ -409,7 +409,7 @@ def add_addresses(self, addresses, update=True): self._update_coverage_hash() # mark these touched addresses as dirty - self._unmapped_data |= set(addresses) + self.unmapped_addresses |= set(addresses) def subtract_data(self, data): """ @@ -468,6 +468,60 @@ def _update_coverage_hash(self): # Coverage Mapping #-------------------------------------------------------------------------- + def _normalize_coverage(self): + """ + Normalize basic block coverage into instruction coverage. + + TODO: It would be interesting if we could do away with this entirely, + working off the original instruction/bb coverage data (hitmap) instead. + """ + coverage_addresses = viewkeys(self._hitmap) + if not coverage_addresses: + return + + # bucketize the exploded coverage addresses + instructions = coverage_addresses & self._metadata.instructions + basic_blocks = instructions & viewkeys(self._metadata.nodes) + + # + # here we attempt to compute the ratio between basic block addresses, + # and instruction addresses in the incoming coverage data. + # + # this will help us determine if the existing instruction data is + # sufficient, or whether we need to explode/flatten the basic block + # addresses into their respective child instructions + # + + block_ratio = len(basic_blocks) / float(len(instructions)) + block_trace_confidence = 0.80 + logger.debug("Block confidence %f" % block_ratio) + + # + # a low basic block to instruction ratio implies the data is probably + # from an instruction trace, or a drcov trace that was exploded from + # (bb_address, size) into its respective addresses + # + + if block_ratio < block_trace_confidence: + return + + # + # take each basic block address, and explode it into a list of all the + # instruction addresses contained within the basic block as determined + # by the database metadata cache + # + # it is *possible* that this may introduce 'inaccurate' paint should + # the user provide a basic block trace that crashes mid-block. but + # that is not something we can account for in a block trace... + # + + for bb_address in basic_blocks: + bb_hits = self._hitmap[bb_address] + for inst_address in self._metadata.nodes[bb_address].instructions: + self._hitmap[inst_address] = bb_hits + + logger.debug("Converted basic block trace to instruction trace...") + def _map_coverage(self): """ Map loaded coverage data to the underlying database metadata. @@ -480,10 +534,11 @@ def _map_nodes(self): """ Map loaded coverage data to database defined nodes (basic blocks). """ + db_metadata = self._metadata dirty_nodes = {} # the coverage data we will attempt to process in this function - coverage_addresses = collections.deque(sorted(self._unmapped_data)) + coverage_addresses = sorted(self.unmapped_addresses) # # the loop below is the core of our coverage mapping process. @@ -501,23 +556,27 @@ def _map_nodes(self): # speed. please be careful if you wish to modify it... # - while coverage_addresses: + i, num_addresses = 0, len(coverage_addresses) + + while i < num_addresses: # get the next coverage address to map - address = coverage_addresses.popleft() + address = coverage_addresses[i] # get the node (basic block) metadata that this address falls in - node_metadata = self._metadata.get_node(address) + node_metadata = db_metadata.get_node(address) # # should we fail to locate node metadata for the coverage address # that we are trying to map, then the address must not fall inside - # of a defined function. - # - # in this case, the coverage address will remain unmapped... + # of a defined function # if not node_metadata: + self.orphan_addresses.add(address) + if address in db_metadata.instructions: + self.unmapped_addresses.discard(address) + i += 1 continue # @@ -540,6 +599,10 @@ def _map_nodes(self): node_coverage = NodeCoverage(node_metadata.address, self._weak_self) self.nodes[node_metadata.address] = node_coverage + # alias for speed, prior to looping + node_start = node_metadata.address + node_end = node_start + node_metadata.size + # # the loop below is as an inlined fast-path that assumes the next # several coverage addresses will likely belong to the same node @@ -552,29 +615,32 @@ def _map_nodes(self): while 1: # - # map the hitmap data for the current address (an instruction) - # to this NodeCoverage and mark the instruction as mapped by - # discarding its address from the unmapped data list + # map the hitmap data for the current address if it falls on + # an actual instruction start within the node + # + # if the address falls within an instruction, it will just be + # 'ignored', remaining in the 'unmapped' / invisible data # - node_coverage.executed_instructions[address] = self._hitmap[address] - self._unmapped_data.discard(address) + if address in node_metadata.instructions: + node_coverage.executed_instructions[address] = self._hitmap[address] + self.unmapped_addresses.discard(address) # get the next address to attempt mapping on try: - address = coverage_addresses.popleft() + i += 1 + address = coverage_addresses[i] # an IndexError implies there is nothing left to map... except IndexError: - break; + break # # if the next address is not in this node, it's time break out # of this loop and send it through the full node lookup path # - if not (address in node_metadata.instructions): - coverage_addresses.appendleft(address) + if not (node_start <= address < node_end): break # the node was updated, so save its coverage as dirty @@ -642,27 +708,10 @@ def unmap_all(self): self.functions = {} self.partial_nodes = set() self.partial_instructions = set() - self._misaligned_data = set() + self.orphan_addresses = set() # dump the source coverage data back into an 'unmapped' state - self._unmapped_data = set(self._hitmap.keys()) - self._unmapped_data.add(BADADDR) - - #-------------------------------------------------------------------------- - # Debug - #-------------------------------------------------------------------------- - - def dump_unmapped(self): - """ - Dump the unmapped coverage data. - """ - lmsg("Unmapped coverage data for %s:" % self.name) - if len(self._unmapped_data) == 1: # 1 is going to be BADADDR - lmsg(" * (there is no unmapped data!)") - return - - for address in self._unmapped_data: - lmsg(" * 0x%X" % address) + self.unmapped_addresses = set(self._hitmap.keys()) #------------------------------------------------------------------------------ # Function Coverage @@ -749,7 +798,7 @@ def finalize(self): self.executions = float(node_sum) / function_metadata.node_count # bake colors - self.coverage_color = compute_color_on_gradiant( + self.coverage_color = compute_color_on_gradient( self.instruction_percent, self.database.palette.table_coverage_bad, self.database.palette.table_coverage_good diff --git a/plugins/lighthouse/director.py b/plugins/lighthouse/director.py index 15309106..b7aeacfa 100644 --- a/plugins/lighthouse/director.py +++ b/plugins/lighthouse/director.py @@ -3,7 +3,6 @@ import string import logging import threading -import traceback import collections from lighthouse.util.misc import * @@ -417,9 +416,8 @@ def load_coverage_batch(self, filepaths, batch_name, progress_callback=logger.de if not aggregate_addresses: return (None, errors) - # optimize the aggregated data (once) and save it to the director - coverage_data = self._optimize_coverage_data(aggregate_addresses) - coverage = self.create_coverage(batch_name, coverage_data) + # save the batched coverage data to the director + coverage = self.create_coverage(batch_name, aggregate_addresses) # evaluate coverage if not coverage.nodes: @@ -472,7 +470,6 @@ def load_coverage_files(self, filepaths, progress_callback=logger.debug): try: coverage_file = self.reader.open(filepath) coverage_addresses = self._extract_coverage_data(coverage_file) - coverage_data = self._optimize_coverage_data(coverage_addresses) # save and suppress warnings generated from loading coverage files except CoverageParsingError as e: @@ -484,18 +481,18 @@ def load_coverage_files(self, filepaths, progress_callback=logger.debug): errors[CoverageMissingError].append(CoverageMissingError(filepath)) continue - # save the attribution data for this coverage data - for address in coverage_data: - if address in self.metadata.nodes: - self.owners[address].add(filepath) - # # request a name for the new coverage mapping that the director will # generate from the loaded coverage data # coverage_name = self._suggest_coverage_name(filepath) - coverage = self.create_coverage(coverage_name, coverage_data, filepath) + coverage = self.create_coverage(coverage_name, coverage_addresses, filepath) + + # save the attribution data for this coverage data + for address in coverage.data: + if address in self.metadata.nodes: # TODO/UNMAPPED: support right click unmapped addrs + self.owners[address].add(filepath) # evaluate coverage if not coverage.nodes: @@ -621,81 +618,6 @@ def _extract_coverage_data(self, coverage_file): # well, this one is probably the fault of the CoverageFile author... raise NotImplementedError("Incomplete CoverageFile implementation") - def _optimize_coverage_data(self, coverage_addresses): - """ - Optimize exploded coverage data to the current metadata cache. - """ - logger.debug("Optimizing coverage data...") - addresses = set(coverage_addresses) - - # bucketize the exploded coverage addresses - instructions = addresses & set(self.metadata.instructions) - basic_blocks = instructions & viewkeys(self.metadata.nodes) - - if not instructions: - logger.debug("No mappable instruction addresses in coverage data") - return [] - - """ - # - # TODO/LOADING: display undefined/misaligned data somehow? - # - - unknown = addresses - instructions - - # bucketize the uncategorized exploded addresses - undefined, misaligned = [], [] - for address in unknown: - - # size == -1 (undefined inst) - if self.metadata.get_instruction_size(address): - undefined.append(address) - - # size == 0 (misaligned inst) - else: - misaligned.append(address) - """ - - # - # here we attempt to compute the ratio between basic block addresses, - # and instruction addresses in the incoming coverage data. - # - # this will help us determine if the existing instruction data is - # sufficient, or whether we need to explode/flatten the basic block - # addresses into their respective child instructions - # - - block_ratio = len(basic_blocks) / float(len(instructions)) - block_trace_confidence = 0.80 - logger.debug("Block confidence %f" % block_ratio) - - # - # a low basic block to instruction ratio implies the data is probably - # from an instruction trace, or a basic block trace has been flattened - # exploded already (eg, a drcov log) - # - - if block_ratio < block_trace_confidence: - logger.debug("Optimized as instruction trace...") - return list(instructions) - - # - # take each basic block address, and explode it into a list of all the - # instruction addresses contained within the basic block as determined - # by the database metadata cache - # - # it is *possible* that this may introduce 'inaccurate' paint should - # the user provide a basic block trace that crashes mid-block. but - # that is not something we can account for in a block trace... - # - - block_instructions = set([]) - for address in basic_blocks: - block_instructions |= set(self.metadata.nodes[address].instructions) - - logger.debug("Optimized as basic block trace...") - return list(block_instructions | instructions) - def _suggest_coverage_name(self, filepath): """ Return a suggested coverage name for the given filepath. @@ -831,7 +753,7 @@ def _find_fuzzy_name(self, coverage_file, target_name): def get_address_coverage(self, address): """ - Return a list of coverage object containing the given address. + Return a list of database coverage objects containing the given address. """ found = [] @@ -1052,12 +974,6 @@ def get_coverage_string(self, coverage_name): return "%s - %s%% - %s" % (symbol, percent_str, coverage_name) - def dump_unmapped(self): - """ - Dump the unmapped coverage data for the active set. - """ - self.coverage.dump_unmapped() - #---------------------------------------------------------------------- # Aliases #---------------------------------------------------------------------- diff --git a/plugins/lighthouse/integration/binja_integration.py b/plugins/lighthouse/integration/binja_integration.py index 3b9ea6c0..cb5df93a 100644 --- a/plugins/lighthouse/integration/binja_integration.py +++ b/plugins/lighthouse/integration/binja_integration.py @@ -56,7 +56,7 @@ def get_context(self, dctx, startup=True): # starts trying to use lighthouse for their session. # # so we initialize the lighthouse context (with start()) on the - # second context request which will go throught the else block + # second context request which will go through the else block # below... any subsequent call to start() is effectively a nop! # @@ -96,16 +96,23 @@ def binja_close_context(self, dctx): #-------------------------------------------------------------------------- # - # TODO / HACK / XXX / V35: Some of Binja's UI elements (such as the + # TODO / HACK / XXX / V35 / 2021: Some of Binja's UI elements (such as the # terminal) do not get assigned a BV, even if there is only one open. # - # this is problematic, because if the user 'clicks' onto the termial, and + # this is problematic, because if the user 'clicks' onto the terminal, and # then tries to execute our UIActions (like 'Load Coverage File'), the # given 'context.binaryView' will be None # # in the meantime, we have to use this workaround that will try to grab # the 'current' bv from the dock. this is not ideal, but it will suffice. # + # ----------------- + # + # XXX: It's now 2024, Binja's UI / API stack has grown a lot. it's more + # powerful and a bunch of the oddities / hacks lighthouse employed for + # binja may no longer apply. this whole file should probably be revisited + # and re-factored at some point point.. sorry if it's hard to follow + # def _interactive_load_file(self, context): dctx = disassembler.binja_get_bv_from_dock() @@ -121,8 +128,58 @@ def _interactive_load_batch(self, context): return super(LighthouseBinja, self).interactive_load_batch(dctx) - def _open_coverage_xref(self, dctx, addr): - super(LighthouseBinja, self).open_coverage_xref(addr, dctx) + def _open_coverage_xref(self, context): + super(LighthouseBinja, self).open_coverage_xref(context.address, context.binaryView) + + def _interactive_coverage_xref(self, context): + + if context is None: + return + + # + # this is a special case where we check if the ctx exists rather than + # blindly creating a new one. again, this is because binja may call + # this function at random times to decide whether it should display the + # XREF menu option. + # + # but asking whether or not the xref menu option should be shown is not + # a good indication of 'is the user actually using lighthouse' so we + # do not want this to be one that creates lighthouse contexts + # + + dctx = context.binaryView + if not dctx: + return + + dctx_id = ctypes.addressof(dctx.handle.contents) + lctx = self.lighthouse_contexts.get(dctx_id, None) + if not lctx: + return + + # + # is there even any coverage loaded into lighthouse? if not, the user + # probably isn't even using it. so don't bother showing the xref action + # + + if not lctx.director.coverage_names: + return + + if context.view is None: + return + + view = context.view + context_menu = view.contextMenu() + + # + # Create a new, temporary Coverage Xref action to inject into the + # right click context menu that is being shown... + # + + action = "Coverage Xref" + UIAction.registerAction(action) + action_handler = view.actionHandler() + action_handler.bindAction(action, UIAction(self._open_coverage_xref)) + context_menu.addAction(action, "Plugins") def _is_xref_valid(self, dctx, addr): @@ -152,6 +209,12 @@ def _open_coverage_overview(self, context): return super(LighthouseBinja, self).open_coverage_overview(dctx) + def _stub(self, context): + # XXX: This was added as a last minute bodge prior to releasing v0.9.3, + # it fixes a crash-on-close that was manifesting on binja macOS, when + # using a lambda instead of a concrete function/stub like this. + return None + #-------------------------------------------------------------------------- # Binja Actions #-------------------------------------------------------------------------- @@ -165,31 +228,28 @@ def _install_load_file(self): action = self.ACTION_LOAD_FILE UIAction.registerAction(action) UIActionHandler.globalActions().bindAction(action, UIAction(self._interactive_load_file)) - Menu.mainMenu("Tools").addAction(action, "Loading", 0) + Menu.mainMenu("Plugins").addAction(action, "Loading", 0) logger.info("Installed the 'Code coverage file' menu entry") def _install_load_batch(self): action = self.ACTION_LOAD_BATCH UIAction.registerAction(action) UIActionHandler.globalActions().bindAction(action, UIAction(self._interactive_load_batch)) - Menu.mainMenu("Tools").addAction(action, "Loading", 1) + Menu.mainMenu("Plugins").addAction(action, "Loading", 1) logger.info("Installed the 'Code coverage batch' menu entry") - # TODO/V35: convert to a UI action once we can disable/disable them on the fly def _install_open_coverage_xref(self): - PluginCommand.register_for_address( - self.ACTION_COVERAGE_XREF, - "Open the coverage xref window", - self._open_coverage_xref, - self._is_xref_valid - ) + action = self.ACTION_COVERAGE_XREF + UIAction.registerAction(action) + UIActionHandler.globalActions().bindAction(action, UIAction(self._stub, self._interactive_coverage_xref)) + Menu.mainMenu("Plugins").addAction(action, "Loading", 2) # NOTE/V35: Binja automatically creates View --> Show Coverage Overview def _install_open_coverage_overview(self): action = self.ACTION_COVERAGE_OVERVIEW UIAction.registerAction(action) UIActionHandler.globalActions().bindAction(action, UIAction(self._open_coverage_overview)) - Menu.mainMenu("Tools").addAction(action, "Windows", 0) + Menu.mainMenu("Plugins").addAction(action, "Windows", 0) logger.info("Installed the 'Open Coverage Overview' menu entry") # NOTE/V35: Binja doesn't really 'unload' plugins, so whatever... diff --git a/plugins/lighthouse/integration/core.py b/plugins/lighthouse/integration/core.py index fc2cb551..e1c05f11 100644 --- a/plugins/lighthouse/integration/core.py +++ b/plugins/lighthouse/integration/core.py @@ -1,4 +1,3 @@ -import os import abc import logging @@ -7,10 +6,10 @@ from lighthouse.util import lmsg from lighthouse.util.qt import * from lighthouse.util.update import check_for_update -from lighthouse.util.disassembler import disassembler, DisassemblerContextAPI +from lighthouse.util.disassembler import disassembler from lighthouse.ui import * -from lighthouse.metadata import DatabaseMetadata, metadata_progress +from lighthouse.metadata import metadata_progress from lighthouse.exceptions import * logger = logging.getLogger("Lighthouse.Core") @@ -26,9 +25,9 @@ class LighthouseCore(object): # Plugin Metadata #-------------------------------------------------------------------------- - PLUGIN_VERSION = "0.9.2" + PLUGIN_VERSION = "0.9.3" AUTHORS = "Markus Gaasedelen" - DATE = "2021" + DATE = "2024" #-------------------------------------------------------------------------- # Initialization diff --git a/plugins/lighthouse/metadata.py b/plugins/lighthouse/metadata.py index 23f974d6..95f086f3 100644 --- a/plugins/lighthouse/metadata.py +++ b/plugins/lighthouse/metadata.py @@ -78,7 +78,7 @@ def __init__(self, lctx=None): # the cache of key database structures self.nodes = {} self.functions = {} - self.instructions = [] + self.instructions = set() # internal members to help index & navigate the cached metadata self._name2func = {} @@ -152,14 +152,6 @@ def terminate(self): # Providers #-------------------------------------------------------------------------- - def get_instructions_slice(self, start_address, end_address): - """ - Get the instructions addresses that fall within a given range. - """ - index_start = bisect.bisect_left(self.instructions, start_address) - index_end = bisect.bisect_left(self.instructions, end_address) - return self.instructions[index_start:index_end] - def get_instruction_size(self, address): """ Get the size of an instruction at a given address. @@ -403,19 +395,6 @@ def abort_refresh(self, join=False): if join: worker.join() - def _refresh_instructions(self): - """ - Refresh the list of database instructions (from function metadata). - """ - instructions = [] - for function_metadata in itervalues(self.functions): - instructions.append(function_metadata.instructions) - instructions = list(set(itertools.chain.from_iterable(instructions))) - instructions.sort() - - # commit the updated instruction list - self.instructions = instructions - def _refresh_lookup(self): """ Refresh the internal fast lookup address lists. @@ -470,11 +449,11 @@ def _refresh_async(self, result_queue, progress_callback=None): def _clear_cache(self): """ - Cleare the metadata cache of all collected info. + Clear the metadata cache of all collected info. """ self.nodes = {} self.functions = {} - self.instructions = [] + self.instructions = set() self._node2func = collections.defaultdict(list) self._refresh_lookup() self.cached = False @@ -520,9 +499,6 @@ def _refresh(self, progress_callback=None, is_async=False): end = time.time() logger.debug("Metadata collection took %s seconds" % (end - start)) - # regenerate the instruction list from collected metadata - self._refresh_instructions() - # refresh the internal function/node fast lookup lists self._refresh_lookup() @@ -574,6 +550,8 @@ def _sync_collect_metadata(self, function_addresses, progress_callback, progress completed += CHUNK_SIZE if function_addresses else len(addresses_chunk) progress_callback(completed, total) + self._cache_instructions() + @not_mainthread def _async_collect_metadata(self, function_addresses, progress_callback): """ @@ -658,6 +636,59 @@ def _cache_functions(self, addresses_chunk): self.nodes.update(function_metadata.nodes) self.functions[address] = function_metadata + def _cache_instructions(self): + """ + This will be replaced with a disassembler-specific function at runtime. + + NOTE: Read the 'MONKEY PATCHING' section at the end of this file. + """ + raise RuntimeError("This function should have been monkey patched...") + + def _binja_cache_instructions(self): + """ + Cache the list of instructions by doing a full scrape of the Binary Ninja database. + """ + instructions = [] + + # + # since 'code' does not exist outside of functions in binary ninja, + # just scrape instructions from our existing cached nodes + # + + for function_metadata in itervalues(self.functions): + instructions.append(function_metadata.instructions) + + # commit the updated instruction list + self.instructions = set(itertools.chain.from_iterable(instructions)) + + def _ida_cache_instructions(self): + """ + Cache the list of instructions by doing a full scrape of the IDA database. + """ + instructions = set() + + # alias for speed + ida_is_code = idaapi.is_code + ida_get_flags = idaapi.get_flags + ida_next_head = idaapi.next_head + add_instruction = instructions.add + + # scrape instruction addresses from the database + for seg_address in idautils.Segments(): + seg = idaapi.getseg(seg_address) + + current_address = seg_address + end_address = seg.end_ea + + # save the address of each defined instruction in the segment + while current_address < end_address: + if ida_is_code(ida_get_flags(current_address)): + add_instruction(current_address) + current_address = ida_next_head(current_address, end_address) + + # commit the updated instruction set + self.instructions = instructions + #-------------------------------------------------------------------------- # Signal Handlers #-------------------------------------------------------------------------- @@ -905,7 +936,7 @@ def _binja_refresh_nodes(self, disassembler_ctx): for i in range(0, count.value): if edges[i].target: - function_metadata.edges[edge_src].append(node._create_instance(BNNewBasicBlockReference(edges[i].target), bv).start) + function_metadata.edges[edge_src].append(node._create_instance(BNNewBasicBlockReference(edges[i].target)).start) core.BNFreeBasicBlockEdgeList(edges, count.value) # NOTE/PERF ~28% of metadata collection time alone... @@ -1160,6 +1191,7 @@ def metadata_progress(completed, total): if disassembler.NAME == "IDA": import idaapi import idautils + DatabaseMetadata._cache_instructions = DatabaseMetadata._ida_cache_instructions FunctionMetadata._refresh_nodes = FunctionMetadata._ida_refresh_nodes NodeMetadata._cache_node = NodeMetadata._ida_cache_node @@ -1170,6 +1202,7 @@ def metadata_progress(completed, total): import ctypes import binaryninja from binaryninja import core + DatabaseMetadata._cache_instructions = DatabaseMetadata._binja_cache_instructions FunctionMetadata._refresh_nodes = FunctionMetadata._binja_refresh_nodes NodeMetadata._cache_node = NodeMetadata._binja_cache_node diff --git a/plugins/lighthouse/painting/binja_painter.py b/plugins/lighthouse/painting/binja_painter.py index 996153e1..f3e8f7d2 100644 --- a/plugins/lighthouse/painting/binja_painter.py +++ b/plugins/lighthouse/painting/binja_painter.py @@ -38,6 +38,8 @@ def _paint_instructions(self, instructions): def _clear_instructions(self, instructions): bv = disassembler[self.lctx].bv + state = bv.begin_undo_actions() + for address in instructions: for func in bv.get_functions_containing(address): func.set_auto_instr_highlight(address, HighlightStandardColor.NoHighlightColor) @@ -45,6 +47,11 @@ def _clear_instructions(self, instructions): self._painted_instructions -= set(instructions) self._action_complete.set() + if hasattr(bv, "forget_undo_actions"): + bv.forget_undo_actions(state) + else: + bv.commit_undo_actions(state) + def _partial_paint(self, bv, instructions, color): for address in instructions: for func in bv.get_functions_containing(address): @@ -57,6 +64,8 @@ def _paint_nodes(self, node_addresses): db_coverage = self.director.coverage db_metadata = self.director.metadata + state = bv.begin_undo_actions() + r, g, b, _ = self.palette.coverage_paint.getRgb() color = HighlightColor(red=r, green=g, blue=b) @@ -83,10 +92,17 @@ def _paint_nodes(self, node_addresses): self._painted_nodes |= (set(node_addresses) - partial_nodes) self._action_complete.set() + if hasattr(bv, "forget_undo_actions"): + bv.forget_undo_actions(state) + else: + bv.commit_undo_actions(state) + def _clear_nodes(self, node_addresses): bv = disassembler[self.lctx].bv db_metadata = self.director.metadata + state = bv.begin_undo_actions() + for node_address in node_addresses: node_metadata = db_metadata.nodes.get(node_address, None) @@ -102,6 +118,11 @@ def _clear_nodes(self, node_addresses): self._painted_nodes -= set(node_addresses) self._action_complete.set() + if hasattr(bv, "forget_undo_actions"): + bv.forget_undo_actions(state) + else: + bv.commit_undo_actions(state) + def _refresh_ui(self): pass diff --git a/plugins/lighthouse/painting/ida_painter.py b/plugins/lighthouse/painting/ida_painter.py index e066b8b8..00f66baf 100644 --- a/plugins/lighthouse/painting/ida_painter.py +++ b/plugins/lighthouse/painting/ida_painter.py @@ -247,7 +247,7 @@ def _paint_nodes(self, node_addresses): # # if we did not get *everything* that we needed, then it is - # possible the database changesd, or the coverage set changed... + # possible the database changed, or the coverage set changed... # # this is kind of what we get for not using locks :D but that's # okay, just stop painting here and let the painter sort it out diff --git a/plugins/lighthouse/painting/painter.py b/plugins/lighthouse/painting/painter.py index 63b60034..066f27aa 100644 --- a/plugins/lighthouse/painting/painter.py +++ b/plugins/lighthouse/painting/painter.py @@ -537,8 +537,6 @@ def _rebase_database(self): a rebase occurs while the painter is running. """ db_metadata = self.director.metadata - instructions = db_metadata.instructions - nodes = viewvalues(db_metadata.nodes) # a rebase has not occurred if not db_metadata.cached or (db_metadata.imagebase == self._imagebase): diff --git a/plugins/lighthouse/reader/parsers/drcov.py b/plugins/lighthouse/reader/parsers/drcov.py index 64cf6238..6d8932c2 100644 --- a/plugins/lighthouse/reader/parsers/drcov.py +++ b/plugins/lighthouse/reader/parsers/drcov.py @@ -100,7 +100,22 @@ def get_offset_blocks(self, module_name): mod_ids = [module.id for module in modules] # loop through the coverage data and filter out data for the target ids - coverage_blocks = [(bb.start, bb.size) for bb in self.bbs if bb.mod_id in mod_ids] + if self.version < 3: + coverage_blocks = [(bb.start, bb.size) for bb in self.bbs if bb.mod_id in mod_ids] + + # + # drcov version 3 does not include the 'preferred' / sub-module base + # in the bb offset, so we must add that base offset before returning + # the block offsets to correctly normalize things + # + # it's unclear if the preferred_base for given sub-module segments + # will always be correct, so we opt to simply use the first segment + # in a given module as the base to compute the known runtime offset + # + + else: + mod_bases = dict([(module.id, module.start - modules[0].start) for module in modules]) + coverage_blocks = [(mod_bases[bb.mod_id] + bb.start, bb.size) for bb in self.bbs if bb.mod_id in mod_ids] # return the filtered coverage blocks return coverage_blocks @@ -137,7 +152,7 @@ def _parse_drcov_header(self, f): flavor_line = f.readline().decode('utf-8').strip() self.flavor = flavor_line.split(":")[1] - assert self.version == 2, "Only drcov version 2 log files supported" + assert self.version == 2 or self.version == 3, "Only drcov versions 2 and 3 log files supported" def _parse_module_table(self, f): """ @@ -227,6 +242,12 @@ def _parse_module_table_columns(self, f): Mac/Linux: 'Columns: id, containing_id, start, end, entry, offset, path' + DynamoRIO v10.0.19734, table version 5: + Windows: + 'Columns: id, containing_id, start, end, entry, offset, preferred_base, checksum, timestamp, path' + Mac/Linux: + 'Columns: id, containing_id, start, end, entry, offset, preferred_base, path' + """ # NOTE/COMPAT: there is no 'Columns' line for the v1 table... @@ -450,19 +471,19 @@ def _parse_module_v5(self, data): """ Parse a module table v5 entry. """ - self.id = int(data[0]) - self.containing_id = int(data[1]) - self.base = int(data[2], 16) - self.end = int(data[3], 16) - self.entry = int(data[4], 16) - self.offset = int(data[5], 16) - self.preferred_base= int(data[6], 16) + self.id = int(data[0]) + self.containing_id = int(data[1]) + self.base = int(data[2], 16) + self.end = int(data[3], 16) + self.entry = int(data[4], 16) + self.offset = int(data[5], 16) + self.preferred_base = int(data[6], 16) if len(data) > 8: # Windows Only - self.checksum = int(data[7], 16) - self.timestamp = int(data[8], 16) - self.path = str(data[-1]) - self.size = self.end-self.base - self.filename = os.path.basename(self.path.replace('\\', os.sep)) + self.checksum = int(data[7], 16) + self.timestamp = int(data[8], 16) + self.path = str(data[-1]) + self.size = self.end-self.base + self.filename = os.path.basename(self.path.replace('\\', os.sep)) #------------------------------------------------------------------------------ diff --git a/plugins/lighthouse/ui/coverage_combobox.py b/plugins/lighthouse/ui/coverage_combobox.py index 0d4bafa9..d6dd5c39 100644 --- a/plugins/lighthouse/ui/coverage_combobox.py +++ b/plugins/lighthouse/ui/coverage_combobox.py @@ -118,7 +118,7 @@ def _ui_init(self): self.setSizeAdjustPolicy(QtWidgets.QComboBox.AdjustToContentsOnFirstShow) self.setSizePolicy(QtWidgets.QSizePolicy.Ignored, QtWidgets.QSizePolicy.Ignored) - self.setMaximumHeight(self._font_metrics.height()*1.75) + self.setMaximumHeight(int(self._font_metrics.height()*1.75)) # draw the QComboBox with a 'Windows'-esque style self.setStyle(QtWidgets.QStyleFactory.create("Windows")) @@ -437,10 +437,7 @@ def _ui_init(self): # hh.setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch) - hh.setSectionResizeMode(1, QtWidgets.QHeaderView.Fixed) vh.setSectionResizeMode(QtWidgets.QHeaderView.ResizeToContents) - - hh.setMinimumSectionSize(0) vh.setMinimumSectionSize(0) # get the column width hint from the model for the 'X' delete column @@ -451,7 +448,9 @@ def _ui_init(self): ) # set the 'X' delete icon column width to a fixed size based on the hint + hh.setMinimumSectionSize(icon_column_width) hh.resizeSection(COLUMN_DELETE, icon_column_width) + hh.setSectionResizeMode(1, QtWidgets.QHeaderView.ResizeToContents) # install a delegate to do some custom painting against the combobox self.setItemDelegate(ComboBoxDelegate(self)) @@ -533,7 +532,7 @@ def __init__(self, director, parent=None): delete_icon = QtGui.QPixmap(plugin_resource("icons/delete_coverage.png")) # compute the appropriate size for the deletion icon - icon_height = self._font_metrics.height()*0.75 + icon_height = int(self._font_metrics.height()*0.75) icon_width = icon_height # scale the icon as appropriate (very likely scaling it down) diff --git a/plugins/lighthouse/ui/coverage_overview.py b/plugins/lighthouse/ui/coverage_overview.py index 75a37fe6..9028012e 100644 --- a/plugins/lighthouse/ui/coverage_overview.py +++ b/plugins/lighthouse/ui/coverage_overview.py @@ -194,7 +194,7 @@ def _ui_layout(self): # layout the major elements of our widget layout = QtWidgets.QGridLayout() - layout.setSpacing(get_dpi_scale()*5.0) + layout.setSpacing(int(get_dpi_scale()*5)) layout.addWidget(self._table_view) layout.addWidget(self._toolbar) @@ -214,8 +214,8 @@ def _ui_show_settings(self): -1*self._settings_menu.sizeHint().height() ) center = QtCore.QPoint( - self._settings_button.sizeHint().width()/2, - self._settings_button.sizeHint().height()/2 + int(self._settings_button.sizeHint().width()/2), + int(self._settings_button.sizeHint().height()/2) ) where = self._settings_button.mapToGlobal(center+delta) self._settings_menu.popup(where) diff --git a/plugins/lighthouse/ui/coverage_settings.py b/plugins/lighthouse/ui/coverage_settings.py index 6fa39789..44c08b25 100644 --- a/plugins/lighthouse/ui/coverage_settings.py +++ b/plugins/lighthouse/ui/coverage_settings.py @@ -68,10 +68,6 @@ def _ui_init_actions(self): self._action_refresh_metadata.setToolTip("Refresh the database metadata and coverage mapping") self.addAction(self._action_refresh_metadata) - self._action_dump_unmapped = QtWidgets.QAction("Dump unmapped coverage", None) - self._action_dump_unmapped.setToolTip("Print all coverage data not mapped to a function") - self.addAction(self._action_dump_unmapped) - self._action_export_html = QtWidgets.QAction("Generate HTML report", None) self._action_export_html.setToolTip("Export the coverage table to HTML") self.addAction(self._action_export_html) @@ -91,7 +87,6 @@ def connect_signals(self, controller, lctx): self._action_disable_paint.triggered[bool].connect(lambda x: lctx.painter.set_enabled(not x)) self._action_force_clear.triggered.connect(lctx.painter.force_clear) self._action_export_html.triggered.connect(controller.export_to_html) - self._action_dump_unmapped.triggered.connect(lctx.director.dump_unmapped) lctx.painter.status_changed(self._ui_painter_changed_status) #-------------------------------------------------------------------------- diff --git a/plugins/lighthouse/ui/coverage_table.py b/plugins/lighthouse/ui/coverage_table.py index ddae1761..15565376 100644 --- a/plugins/lighthouse/ui/coverage_table.py +++ b/plugins/lighthouse/ui/coverage_table.py @@ -145,7 +145,7 @@ def _ui_init_table(self): entry_rect = entry_fm.boundingRect(entry_text) # select the larger of the two potential column widths - column_width = max(title_rect.width(), entry_rect.width()*1.2) + column_width = int(max(title_rect.width(), entry_rect.width()*1.2)) # save the final column width self.setColumnWidth(i, column_width) @@ -191,13 +191,17 @@ def _ui_init_table(self): # NOTE: don't ask too many questions about this voodoo math :D spacing = entry_fm.height() - entry_fm.xHeight() tweak = (17*get_dpi_scale() - spacing)/get_dpi_scale() - vh.setDefaultSectionSize(entry_fm.height()+tweak) + vh.setDefaultSectionSize(int(entry_fm.height()+tweak)) def _ui_init_table_ctx_menu_actions(self): """ Initialize the right click context menu actions for the table view. """ + # misc actions + self._action_dump_orphan = QtWidgets.QAction("Dump orphan addresses", None) + self._action_dump_internal = QtWidgets.QAction("Dump internal addresses (Debug)", None) + # function actions self._action_rename = QtWidgets.QAction("Rename", None) self._action_copy_name = QtWidgets.QAction("Copy name", None) @@ -307,8 +311,8 @@ def _populate_table_ctx_menu(self): """ # get the list rows currently selected in the coverage table - selected_rows = self.selectionModel().selectedRows() - if len(selected_rows) == 0: + selected_row_indexes = self.selectionModel().selectedRows() + if len(selected_row_indexes) == 0: return None # the context menu we will dynamically populate @@ -320,13 +324,25 @@ def _populate_table_ctx_menu(self): # copy function name, address, or renaming the function. # - if len(selected_rows) == 1: - ctx_menu.addAction(self._action_rename) - ctx_menu.addSeparator() - ctx_menu.addAction(self._action_copy_name) - ctx_menu.addAction(self._action_copy_address) - ctx_menu.addAction(self._action_copy_name_and_address) - ctx_menu.addSeparator() + if len(selected_row_indexes) == 1: + + row = selected_row_indexes[0].row() + function_address = self._model.row2func[row] + + # special handling for right click of orphan coverage row + if function_address == BADADDR: + ctx_menu.addAction(self._action_dump_orphan) + ctx_menu.addAction(self._action_dump_internal) + return ctx_menu + + # normal right click of a function row + else: + ctx_menu.addAction(self._action_rename) + ctx_menu.addSeparator() + ctx_menu.addAction(self._action_copy_name) + ctx_menu.addAction(self._action_copy_address) + ctx_menu.addAction(self._action_copy_name_and_address) + ctx_menu.addSeparator() # # if multiple functions are selected then show actions available @@ -385,6 +401,14 @@ def _process_table_ctx_menu_action(self, action): elif action == self._action_clear_prefix: self._controller.clear_function_prefixes(rows) + # handle the 'Dump orphan addresses' action + elif action == self._action_dump_orphan: + self._controller.dump_orphan() + + # handle the 'Dump internal addresses' action + elif action == self._action_dump_internal: + self._controller.dump_internal() + #-------------------------------------------------------------------------- # Context Menu (Table Header) #-------------------------------------------------------------------------- @@ -438,6 +462,9 @@ def rename_table_function(self, row): # retrieve details about the function targeted for rename function_address = self._model.row2func[row] + if function_address == BADADDR: + return + original_name = disassembler[self.lctx].get_function_raw_name_at(function_address) # prompt the user for a new function name @@ -536,6 +563,38 @@ def copy_name_and_address(self, rows): copy_to_clipboard(function_name_and_address.rstrip()) return function_name_and_address + #-------------------------------------------------------------------------- + # Dumping + #-------------------------------------------------------------------------- + + def dump_orphan(self): + """ + Dump the orphan coverage data. + """ + coverage = self.lctx.director.coverage + lmsg("Orphan coverage addresses for %s:" % coverage.name) + self._dump_addresses(coverage.orphan_addresses) + + def dump_internal(self): + """ + Dump the internal coverage data. + """ + coverage = self.lctx.director.coverage + lmsg("Internal coverage addresses for %s:" % coverage.name) + self._dump_addresses(coverage.unmapped_addresses) + + def _dump_addresses(self, coverage_addresses): + """ + Dump the given list of addresses to the terminal. + """ + coverage_addresses = sorted(coverage_addresses) + if not coverage_addresses: + lmsg(" * (there is no addresses to dump)") + return + + for address in coverage_addresses: + lmsg(" * 0x%X" % address) + #--------------------------------------------------------------------------- # Misc #--------------------------------------------------------------------------- @@ -547,6 +606,8 @@ def navigate_to_function(self, row): # get the clicked function address function_address = self._model.row2func[row] + if function_address == BADADDR: + return # # if there is actually coverage in the function, attempt to locate the @@ -608,9 +669,13 @@ def export_to_html(self): { "filter": "HTML Files (*.html)", "caption": "Save HTML Report", - "directory": suggested_filepath } + if USING_PYQT5: + kwargs["directory"] = suggested_filepath + else: + kwargs["dir"] = suggested_filepath + # prompt the user with the file dialog, and await their chosen filename(s) filename, _ = file_dialog.getSaveFileName(**kwargs) if not filename: @@ -636,7 +701,8 @@ def _get_function_addresses(self, rows): function_addresses = [] for row_number in rows: address = self._model.row2func[row_number] - function_addresses.append(address) + if address != BADADDR: + function_addresses.append(address) return function_addresses #------------------------------------------------------------------------------ @@ -835,88 +901,120 @@ def headerData(self, column, orientation, role=QtCore.Qt.DisplayRole): # unhandeled header request return None - def data(self, index, role=QtCore.Qt.DisplayRole): + def _data_function_display(self, function_address, column): """ - Define how Qt should access the underlying model data. + Return a string to diplay in the requested column of the given function. """ - # a request has been made for what text to show in a table cell - if role == QtCore.Qt.DisplayRole: + # lookup the function info for the given function + try: + function_metadata = self.lctx.metadata.functions[function_address] - # alias the requested column number once, for readability & perf - column = index.column() + # + # if we hit a KeyError, it is probably because the database metadata + # is being refreshed and the model (this object) has yet to be + # updated. + # + # this should only ever happen as a result of the user using the + # right click 'Refresh metadata' action. And even then, only when + # a function they undefined in the IDB is visible in the coverage + # overview table view. + # + # In theory, the table should get refreshed *after* the metadata + # refresh completes. So for now, we simply return return the filler + # string '?' + # - # lookup the function info for this row - try: - function_address = self.row2func[index.row()] - function_metadata = self.lctx.metadata.functions[function_address] + except KeyError: + return "?" - # - # if we hit a KeyError, it is probably because the database metadata - # is being refreshed and the model (this object) has yet to be - # updated. - # - # this should only ever happen as a result of the user using the - # right click 'Refresh metadata' action. And even then, only when - # a function they undefined in the IDB is visible in the coverage - # overview table view. - # - # In theory, the table should get refreshed *after* the metadata - # refresh completes. So for now, we simply return return the filler - # string '?' - # + # + # remember, if a function does *not* have coverage data, it will + # not have an entry in the coverage map. that means we should + # yield a default, 'blank', coverage item in these instances + # - except KeyError: - return "?" + function_coverage = self._director.coverage.functions.get( + function_address, + self._blank_coverage + ) - # - # remember, if a function does *not* have coverage data, it will - # not have an entry in the coverage map. that means we should - # yield a default, 'blank', coverage item in these instances - # + # Coverage % - (by instruction execution) + if column == self.COV_PERCENT: + return "%5.2f" % (function_coverage.instruction_percent*100) - function_coverage = self._director.coverage.functions.get( - function_address, - self._blank_coverage - ) + # Function Name + elif column == self.FUNC_NAME: + return function_metadata.name - # Coverage % - (by instruction execution) - if column == self.COV_PERCENT: - return "%5.2f" % (function_coverage.instruction_percent*100) + # Function Address + elif column == self.FUNC_ADDR: + return "0x%X" % function_metadata.address - # Function Name - elif column == self.FUNC_NAME: - return function_metadata.name + # Basic Blocks + elif column == self.BLOCKS_HIT: + return "%3u / %-3u" % (function_coverage.nodes_executed, + function_metadata.node_count) - # Function Address - elif column == self.FUNC_ADDR: - return "0x%X" % function_metadata.address + # Instructions Hit + elif column == self.INST_HIT: + return "%4u / %-4u" % (function_coverage.instructions_executed, + function_metadata.instruction_count) - # Basic Blocks - elif column == self.BLOCKS_HIT: - return "%3u / %-3u" % (function_coverage.nodes_executed, - function_metadata.node_count) + # Function Size + elif column == self.FUNC_SIZE: + return "%u" % function_metadata.size - # Instructions Hit - elif column == self.INST_HIT: - return "%4u / %-4u" % (function_coverage.instructions_executed, - function_metadata.instruction_count) + # Cyclomatic Complexity + elif column == self.COMPLEXITY: + return "%u" % function_metadata.cyclomatic_complexity - # Function Size - elif column == self.FUNC_SIZE: - return "%u" % function_metadata.size + # unhandeled? maybe make this an assert? + return None - # Cyclomatic Complexity - elif column == self.COMPLEXITY: - return "%u" % function_metadata.cyclomatic_complexity + def _data_orphan_display(self, column): + """ + Return a string to be displayed by the table + """ + if column == self.FUNC_NAME: + return "Orphan Coverage" + elif column == self.INST_HIT: + return "%u" % len(self._director.coverage.orphan_addresses) + return "N/A" + + def data(self, index, role=QtCore.Qt.DisplayRole): + """ + Define how Qt should access the underlying model data. + """ + + # a request has been made for what text to show in a table cell + if role == QtCore.Qt.DisplayRole: + + column = index.column() + function_address = self.row2func[index.row()] + + if function_address == BADADDR: + return self._data_orphan_display(column) + else: + return self._data_function_display(function_address, column) # cell background color request elif role == QtCore.Qt.BackgroundRole: function_address = self.row2func[index.row()] + + # special handling for 'orphan' coverage + if function_address == BADADDR: + + # if there was *ANY* coverage, color the 'orphan' line red + if self._director.coverage.orphan_addresses: + return self.lctx.palette.table_coverage_bad + + # normal handling function_coverage = self._director.coverage.functions.get( function_address, self._blank_coverage ) + return function_coverage.coverage_color # cell font style format request @@ -953,6 +1051,13 @@ def sort(self, column, sort_order): self.layoutChanged.emit() return + # + # In PySide6 (eg. Binary Ninja) the Qt.SortOrder type does not convert + # to a simple integer (unlike PyQt5) so we reduce the type for comapt + # + + direction = (sort_order == QtCore.Qt.SortOrder.DescendingOrder) + # # NOTE: attrgetter appears to profile ~8-12% faster than lambdas # accessing the member on the member, hence the strange paradigm @@ -963,7 +1068,7 @@ def sort(self, column, sort_order): sorted_functions = sorted( itervalues(self._visible_metadata), key=attrgetter(sort_field), - reverse=sort_order + reverse=direction ) # sort the table entries by a function coverage attribute @@ -971,7 +1076,7 @@ def sort(self, column, sort_order): sorted_functions = sorted( itervalues(self._visible_coverage), key=attrgetter(sort_field), - reverse=sort_order + reverse=direction ) # @@ -989,7 +1094,7 @@ def sort(self, column, sort_order): # items (0%) should be appended to the *end* # - if sort_order: + if direction: sorted_functions += self._no_coverage # @@ -1005,6 +1110,7 @@ def sort(self, column, sort_order): # finally, rebuild the row2func mapping and notify views of this change self.row2func = dict(zip(xrange(len(sorted_functions)), sorted_addresses)) + self.row2func[len(self.row2func)] = BADADDR self.func2row = {v: k for k, v in iteritems(self.row2func)} self.layoutChanged.emit() @@ -1316,6 +1422,9 @@ def _refresh_data(self): self.row2func[row] = function_address row += 1 + # add a special entry for 'orphan coverage' + self.row2func[len(self.row2func)] = BADADDR + # build the inverse func --> row mapping self.func2row = {v: k for k, v in iteritems(self.row2func)} diff --git a/plugins/lighthouse/ui/coverage_xref.py b/plugins/lighthouse/ui/coverage_xref.py index d7bd4b86..e23e37f6 100644 --- a/plugins/lighthouse/ui/coverage_xref.py +++ b/plugins/lighthouse/ui/coverage_xref.py @@ -116,7 +116,7 @@ def _populate_table(self): name_entry.setToolTip(coverage.filepath) self._table.setItem(i, 2, name_entry) date_entry = QtWidgets.QTableWidgetItem() - date_entry.setData(QtCore.Qt.DisplayRole, QtCore.QDateTime.fromMSecsSinceEpoch(coverage.timestamp*1000)) + date_entry.setData(QtCore.Qt.DisplayRole, QtCore.QDateTime.fromMSecsSinceEpoch(int(coverage.timestamp*1000))) self._table.setItem(i, 3, QtWidgets.QTableWidgetItem(date_entry)) # filepaths @@ -135,7 +135,7 @@ def _populate_table(self): name_entry.setToolTip(filepath) self._table.setItem(i, 2, name_entry) date_entry = QtWidgets.QTableWidgetItem() - date_entry.setData(QtCore.Qt.DisplayRole, QtCore.QDateTime.fromMSecsSinceEpoch(timestamp*1000)) + date_entry.setData(QtCore.Qt.DisplayRole, QtCore.QDateTime.fromMSecsSinceEpoch(int(timestamp*1000))) self._table.setItem(i, 3, date_entry) self._table.resizeColumnsToContents() @@ -153,8 +153,8 @@ def _ui_layout(self): layout.addWidget(self._table) # scale widget dimensions based on DPI - height = get_dpi_scale() * 250 - width = get_dpi_scale() * 600 + height = int(get_dpi_scale() * 250) + width = int(get_dpi_scale() * 600) self.setMinimumHeight(height) self.setMinimumWidth(width) diff --git a/plugins/lighthouse/ui/module_selector.py b/plugins/lighthouse/ui/module_selector.py index 32b652a0..a339fa4b 100644 --- a/plugins/lighthouse/ui/module_selector.py +++ b/plugins/lighthouse/ui/module_selector.py @@ -148,8 +148,8 @@ def _ui_layout(self): layout.addWidget(self._checkbox_ignore_missing) # scale widget dimensions based on DPI - height = get_dpi_scale() * 250 - width = get_dpi_scale() * 400 + height = int(get_dpi_scale() * 250) + width = int(get_dpi_scale() * 400) self.setMinimumHeight(height) self.setMinimumWidth(width) diff --git a/plugins/lighthouse/ui/resources/themes/long_night.json b/plugins/lighthouse/ui/resources/themes/long_night.json new file mode 100644 index 00000000..ae8c6eb0 --- /dev/null +++ b/plugins/lighthouse/ui/resources/themes/long_night.json @@ -0,0 +1,69 @@ +{ + "name": "Long Night", + "author": "https://github.com/ioncodes", + + "colors": + { + "black": [33, 33, 33], + "white": [241, 239, 236], + + "darkGray": [20, 20, 20], + "darkGray2": [30, 30, 30], + "darkGray3": [54, 54, 54], + + "gray": [100, 100, 100], + "lightGray": [55, 55, 55], + + "red": [188, 101, 141], + "green": [64, 255, 64], + "blue": [104, 134, 197], + "lightBlue": [128, 200, 255], + "darkBlue": [44, 44, 44], + "purple": [121, 104, 197], + + "focusRed": [255, 83, 112], + "selection": [67, 67, 67] + }, + + "fields": + { + "coverage_paint": ["darkBlue", "lightBlue"], + + "table_text": "white", + "table_grid": "black", + "table_coverage_none": "black", + "table_coverage_bad": "red", + "table_coverage_good": "blue", + "table_background": "black", + "table_selection": "purple", + + "html_summary_text": "white", + "html_table_header": "white", + "html_page_background": "black", + + "shell_text": "white", + "shell_text_valid": "lightBlue", + "shell_text_invalid": "red", + "shell_highlight_invalid": "red", + + "shell_border": "lightGray", + "shell_border_focus": "focusRed", + "shell_background": "black", + + "shell_hint_text": "white", + "shell_hint_background": "black", + + "logic_token": "red", + "comma_token": "green", + "paren_token": "green", + "coverage_token": "lightBlue", + + "combobox_text": "white", + "combobox_selection_text": "white", + "combobox_selection_background": "selection", + + "combobox_border": "lightGray", + "combobox_border_focus": "focusRed", + "combobox_background": "black" + } +} diff --git a/plugins/lighthouse/util/disassembler/binja_api.py b/plugins/lighthouse/util/disassembler/binja_api.py index f0998092..243602da 100644 --- a/plugins/lighthouse/util/disassembler/binja_api.py +++ b/plugins/lighthouse/util/disassembler/binja_api.py @@ -289,14 +289,18 @@ def navigate_to_function(self, function_address, address): return vi.navigateToFunction(func, address) - @BinjaCoreAPI.execute_write def set_function_name_at(self, function_address, new_name): func = self.bv.get_function_at(function_address) + if not func: return + if new_name == "": new_name = None + + state = self.bv.begin_undo_actions() func.name = new_name + self.bv.commit_undo_actions(state) #-------------------------------------------------------------------------- # Hooks API @@ -322,9 +326,6 @@ class RenameHooks(binaryview.BinaryDataNotification): def __init__(self, bv): self._bv = bv - self.symbol_added = self.__symbol_handler - self.symbol_updated = self.__symbol_handler - self.symbol_removed = self.__symbol_handler def hook(self): self._bv.register_notification(self) @@ -332,11 +333,25 @@ def hook(self): def unhook(self): self._bv.unregister_notification(self) - def __symbol_handler(self, view, symbol): + def symbol_added(self, *args): + self.__symbol_handler(*args) + + def symbol_updated(self, *args): + self.__symbol_handler(*args) + + def symbol_removed(self, *args): + self.__symbol_handler(*args, True) + + def __symbol_handler(self, view, symbol, removed=False): + func = self._bv.get_function_at(symbol.address) - if not func.start == symbol.address: + if not func or not func.start == symbol.address: return - self.name_changed(symbol.address, symbol.name) + + if removed: + self.name_changed(symbol.address, "sub_%x" % symbol.address) + else: + self.name_changed(symbol.address, symbol.name) def name_changed(self, address, name): """ diff --git a/plugins/lighthouse/util/misc.py b/plugins/lighthouse/util/misc.py index e7ce56a3..6d071e80 100644 --- a/plugins/lighthouse/util/misc.py +++ b/plugins/lighthouse/util/misc.py @@ -207,33 +207,4 @@ def notify_callback(callback_list, *args): # remove the deleted callbacks for callback_ref in cleanup: callback_list.remove(callback_ref) - -#------------------------------------------------------------------------------ -# Coverage Util -#------------------------------------------------------------------------------ - -def build_hitmap(data): - """ - Build a hitmap from the given list of address. - - A hitmap is a map of address --> number of executions. - - The list of input addresses can be any sort of runtime trace, coverage, - or profiling data that one would like to build a hitmap for. - """ - output = collections.defaultdict(int) - - # if there is no input data, simply return an empty hitmap - if not data: - return output - - # - # walk through the given list of given addresses and build a - # corresponding hitmap for them - # - - for address in data: - output[address] += 1 - - # return the hitmap - return output + \ No newline at end of file diff --git a/plugins/lighthouse/util/qt/util.py b/plugins/lighthouse/util/qt/util.py index 5b9d01c4..baac7770 100644 --- a/plugins/lighthouse/util/qt/util.py +++ b/plugins/lighthouse/util/qt/util.py @@ -37,8 +37,8 @@ def copy_to_clipboard(data): Copy the given data (a string) to the system clipboard. """ cb = QtWidgets.QApplication.clipboard() - cb.clear(mode=cb.Clipboard) - cb.setText(data, mode=cb.Clipboard) + cb.clear(mode=QtGui.QClipboard.Mode.Clipboard) + cb.setText(data, mode=QtGui.QClipboard.Mode.Clipboard) def flush_qt_events(): """ @@ -71,17 +71,17 @@ def get_dpi_scale(): # xHeight is expected to be 40.0 at normal DPI return fm.height() / 173.0 -def compute_color_on_gradiant(percent, color1, color2): +def compute_color_on_gradient(percent, color1, color2): """ Compute the color specified by a percent between two colors. """ r1, g1, b1, _ = color1.getRgb() r2, g2, b2, _ = color2.getRgb() - # compute the new color across the gradiant of color1 -> color 2 - r = r1 + percent * (r2 - r1) - g = g1 + percent * (g2 - g1) - b = b1 + percent * (b2 - b1) + # compute the new color across the gradient of color1 -> color 2 + r = r1 + int(percent * (r2 - r1)) + g = g1 + int(percent * (g2 - g1)) + b = b1 + int(percent * (b2 - b1)) # return the new color return QtGui.QColor(r,g,b) @@ -121,8 +121,8 @@ def prompt_string(label, title, default=""): dlg.setWindowTitle(title) dlg.setTextValue(default) dlg.resize( - dpi_scale*400, - dpi_scale*50 + int(dpi_scale*400), + int(dpi_scale*50) ) dlg.setModal(True) dlg.show() diff --git a/plugins/lighthouse/util/qt/waitbox.py b/plugins/lighthouse/util/qt/waitbox.py index 8fb33005..f3f47c1a 100644 --- a/plugins/lighthouse/util/qt/waitbox.py +++ b/plugins/lighthouse/util/qt/waitbox.py @@ -62,7 +62,7 @@ def _ui_init(self): # configure the main widget / form self.setSizeGripEnabled(False) self.setModal(True) - self._dpi_scale = get_dpi_scale()*5.0 + self._dpi_scale = get_dpi_scale()*5 # initialize abort button self._abort_button = QtWidgets.QPushButton("Cancel") @@ -83,19 +83,19 @@ def _ui_layout(self): v_layout.setAlignment(QtCore.Qt.AlignCenter) v_layout.addWidget(self._text_label) if self._abort: - self._abort_button.clicked.connect(abort) + self._abort_button.clicked.connect(self._abort) v_layout.addWidget(self._abort_button) - v_layout.setSpacing(self._dpi_scale*3) + v_layout.setSpacing(int(self._dpi_scale*3)) v_layout.setContentsMargins( - self._dpi_scale*5, - self._dpi_scale, - self._dpi_scale*5, - self._dpi_scale + int(self._dpi_scale*5), + int(self._dpi_scale), + int(self._dpi_scale*5), + int(self._dpi_scale) ) # scale widget dimensions based on DPI - height = self._dpi_scale * 15 + height = int(self._dpi_scale * 15) self.setMinimumHeight(height) # compute the dialog layout diff --git a/plugins/lighthouse/util/update.py b/plugins/lighthouse/util/update.py index c99c6a33..b197809b 100644 --- a/plugins/lighthouse/util/update.py +++ b/plugins/lighthouse/util/update.py @@ -23,7 +23,7 @@ def check_for_update(current_version, callback): update_thread = threading.Thread( target=async_update_check, args=(current_version, callback,), - name="UpdateChecker" + name="Lighthouse UpdateChecker" ) update_thread.start() @@ -42,7 +42,7 @@ def async_update_check(current_version, callback): logger.debug(" - Failed to reach GitHub for update check...") return - # convert vesrion #'s to integer for easy compare... + # convert version #'s to integer for easy compare... version_remote = int(''.join(re.findall('\d+', remote_version))) version_local = int(''.join(re.findall('\d+', current_version)))