diff --git a/.gitignore b/.gitignore index 4df859f..3199320 100644 --- a/.gitignore +++ b/.gitignore @@ -1,28 +1,10 @@ -src/*.vpj -src/*.vpw -src/*.vpwhist -src/*.vtg -src/*.pyc -src/libs/*.pyc -src/config/*.pyc -src/core/*.pyc -src/disassembler/*.pyc -src/disassembler/IDA/*.pyc +*.pyc +__pycache__ -/build/* -/dist/* -/Karta.egg-info/* +/build +/dist +Karta.egg-info docs/_build/* -docs/_build/*/* -docs/_build/*/*/* -docs/_build/*/*/*/* docs/_static/* docs/_templates/* -/*.pyc -/*/*.pyc -/*/*/*.pyc -/*/*/*/*.pyc -/src/thumbs_up/analyzers/__pycache__/*.pyc -/src/*/*/__pycache__/*.pyc -/src/*/*/__pycache__/*.pyc diff --git a/README.md b/README.md index dfc776d..10b5949 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ We believe that there are 3 main use cases for this IDA plugin: https://karta.readthedocs.io/ ## Installation (Python 3 & IDA >= 7.4) -For the latest versions, using Python 3, simply git clone the repository and run the ```setup.py install``` script. +For the latest versions, using Python 3, simply git clone the repository and run ```pip3 install .```. +To install the plugin for use in ida run +```python -m karta.installers.ida_installer``` Python 3 is supported since versions v2.0.0 and above. ## Installation (Python 2 & IDA < 7.4) diff --git a/docs/Installation.md b/docs/Installation.md index 292a793..68ac5ef 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -7,7 +7,9 @@ Prerequisites * [sark](https://github.com/tmr232/Sark) Using the ```setup.py``` script, one can install all of these prerequisites, and be ready to go: -```./setup.py install``` +```pip3 install .``` +To install the plugin for use in ida run +```python -m karta.installers.ida_installer``` Installing the Plugin ------------------------ diff --git a/setup.py b/setup.py index 1801de7..d466585 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 from setuptools import setup, find_packages -from codecs import open with open("README.md", "r") as fh: long_description = fh.read() @@ -15,11 +14,19 @@ long_description_content_type="text/markdown", url='https://github.com/CheckPointSW/Karta', license='MIT', - packages=find_packages(), + packages=find_packages(where="src"), + package_dir={"": "src"}, install_requires=['elementals', 'sark', 'pydocstyle', 'flake8', 'click', 'scikit-learn'], + python_requires='>=3', classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License (MIT License)", "Operating System :: OS Independent", ], - zip_safe=False) + entry_points={ + 'console_scripts': [ + 'karta_analyze_src = karta.karta_analyze_src:main' + ] + }, + zip_safe=False + ) diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index eeee0da..0000000 --- a/src/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import karta_analyze_src -import karta_identifier -import karta_matcher -import karta_manual_anchor -import karta_manual_identifier -import thumbs_up.thumbs_up_ELF -import thumbs_up.thumbs_up_firmware diff --git a/src/disassembler/IDA/ida_cmd_api.py b/src/disassembler/IDA/ida_cmd_api.py deleted file mode 100644 index e2083c5..0000000 --- a/src/disassembler/IDA/ida_cmd_api.py +++ /dev/null @@ -1,62 +0,0 @@ -from disassembler.disas_api import DisasCMD -from disassembler.factory import registerDisassemblerCMD -import os - -class IdaCMD(DisasCMD): - """DisasCMD implementation for the IDA disassembler.""" - - # Overridden base function - @staticmethod - def identify(path): - """Check if the given command-line path refers to this disassembler. - - Args: - path (str): command-line path to some disassembler (maybe for us) - - Return Value: - True iff the command-line path refers to our program - """ - return os.path.split(path)[-1].split(".")[0].lower().startswith("ida") - - # Overridden base function - @staticmethod - def name(): - """Return the program's name (used mainly for bug fixes in our code...). - - Return Value: - String name of the disassembler program - """ - return "IDA" - - # Overridden base function - def createDatabase(self, binary_file, is_windows): - """Create a database file for the given binary file, compiled to windows or linux as specified. - - Args: - binary_file (path): path to the input binary (*.o / *.obj) file - is_windows (bool): True if this is a binary that was compiled for windows (*.obj), False otherwise (*.o) - - Return Value: - path to the created database file - """ - type = "elf" if not is_windows else "coff" - suffix = ".i64" if self._path.endswith("64") else ".idb" - database_file = binary_file + suffix - # execute the program - os.system(f"{self._path} -A -B -T{type} -o{database_file} {binary_file}") - # return back the (should be) created database file path - return database_file - - # Overridden base function - def executeScript(self, database, script): - """Execute the given script over the given database file that was created earlier. - - Args: - database (path): path to a database file created by the same program - script (path): python script to be executed once the database is loaded - """ - os.system(f"{self._path} -A -S{script} {database}") - - -# Don't forget to register at the factory -registerDisassemblerCMD(IdaCMD.identify, IdaCMD) diff --git a/src/karta/__init__.py b/src/karta/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/analyze_src_file.py b/src/karta/analyze_src_file.py similarity index 87% rename from src/analyze_src_file.py rename to src/karta/analyze_src_file.py index ecf265b..7443753 100644 --- a/src/analyze_src_file.py +++ b/src/karta/analyze_src_file.py @@ -1,9 +1,11 @@ -from config.utils import * -from disassembler.factory import createDisassemblerHandler -from function_context import SourceContext, BinaryContext, IslandContext -from elementals import Logger import logging import traceback +from elementals import Logger + +# cannot have realtive imports for a script running in ida +from karta.config.utils import * +from karta.disassembler.factory import createDisassemblerHandler +from karta.function_context import SourceContext, BinaryContext, IslandContext def analyzeFile(): """Analyzes all of the (source) functions for a single compiled file.""" diff --git a/src/ar_parser.py b/src/karta/ar_parser.py similarity index 100% rename from src/ar_parser.py rename to src/karta/ar_parser.py diff --git a/src/config/__init__.py b/src/karta/config/__init__.py similarity index 100% rename from src/config/__init__.py rename to src/karta/config/__init__.py diff --git a/src/config/anchor.py b/src/karta/config/anchor.py similarity index 99% rename from src/config/anchor.py rename to src/karta/config/anchor.py index 627e162..d032289 100644 --- a/src/config/anchor.py +++ b/src/karta/config/anchor.py @@ -1,5 +1,5 @@ from .anchor_config import * -from config.utils import * +from .utils import * def isAnchor(context, seen_strings, seen_consts, functions_list, logger): """Check if the given context represents an Anchor function. diff --git a/src/config/anchor_config.py b/src/karta/config/anchor_config.py similarity index 100% rename from src/config/anchor_config.py rename to src/karta/config/anchor_config.py diff --git a/src/config/libc_config.py b/src/karta/config/libc_config.py similarity index 100% rename from src/config/libc_config.py rename to src/karta/config/libc_config.py diff --git a/src/config/score_config.py b/src/karta/config/score_config.py similarity index 100% rename from src/config/score_config.py rename to src/karta/config/score_config.py diff --git a/src/config/utils.py b/src/karta/config/utils.py similarity index 88% rename from src/config/utils.py rename to src/karta/config/utils.py index dde5c89..674ebb9 100644 --- a/src/config/utils.py +++ b/src/karta/config/utils.py @@ -1,13 +1,16 @@ -from .score_config import * import json import os +from .score_config import * + ################################# ## Basic Global Configurations ## ################################# -DISASSEMBLER_PATH = "/opt/ida-7.4/ida" -SCRIPT_PATH = os.path.abspath("analyze_src_file.py") +DISASSEMBLER_PATH = None +CONFIG_DIR = os.path.dirname(os.path.realpath(__file__)) +DEFAULT_DISASSEMBLER = os.path.join(CONFIG_DIR, "default_disassembler_path") +SCRIPT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "analyze_src_file.py") LIBRARY_NAME = "Karta" STATE_FILE_SUFFIX = "_file_state.json" @@ -455,24 +458,67 @@ def isMatching(): """ return matching_mode -def setDisassemblerPath(prompter): - """Update the disassembler path according to input from the user. +def addDisassembler(name, path): + """Add an installation of a dissasembler. Args: - prompter (prompter): prompter elementals instance + name (str): name of the installation file + path (str): directory of the disassembler installtion """ - global DISASSEMBLER_PATH + with open(os.path.join(CONFIG_DIR, name), "w") as f: + f.write(path) + +def disassemblerInstallationExists(name): + """Check whether there is an existing installation with the filename. + + Args: + name (str): filename of the installtion to check + + Return Value: + return true if such installtion exists + """ + return os.path.exists(os.path.join(CONFIG_DIR, name)) + +def getDisassembler(name): + """Get directory of disassembler from configuration by it's configuration file name. + + Args: + name (str): name of the disassembler to search for + """ + if disassemblerInstallationExists(name): + with open(os.path.join(CONFIG_DIR, name), "r") as f: + return f.read() - new_path = prompter.input(f"Please insert the command (path) needed in order to execute your disassembler (IDA for instance) ({DISASSEMBLER_PATH}): ") - if len(new_path.strip()) != 0: - DISASSEMBLER_PATH = new_path +def setDefaultDisassembler(name): + """Set the default disassembler in the configuration file. -def getDisasPath(): + Args: + name (str): name of the file that contains the default disassembler to use. + """ + with open(os.path.join(CONFIG_DIR, DEFAULT_DISASSEMBLER), "w") as f: + if os.path.isfile(os.path.join(CONFIG_DIR, name)): + f.write(name) + +def getDisasPath(prompter): """Return the updated path to the disassembler. Return Value: The (updated) path to the disassembler program """ + global DISASSEMBLER_PATH + if DISASSEMBLER_PATH is not None: + pass + elif os.path.isfile(DEFAULT_DISASSEMBLER): + actual_disas_path = None + with open(DEFAULT_DISASSEMBLER, 'r') as f: + actual_disas_path = f.read() + full_disas_name_path = os.path.join(CONFIG_DIR, actual_disas_path) + if os.path.isfile(full_disas_name_path): + with open(full_disas_name_path, 'r') as f: + DISASSEMBLER_PATH = f.read() + else: + DISASSEMBLER_PATH = prompter.input("Please enter a path to your disassembler: ") + prompter.info("I may suggest using one of the installers to set a default disassembler") return DISASSEMBLER_PATH def libraryName(): diff --git a/src/core/__init__.py b/src/karta/core/__init__.py similarity index 100% rename from src/core/__init__.py rename to src/karta/core/__init__.py diff --git a/src/core/file_layer.py b/src/karta/core/file_layer.py similarity index 100% rename from src/core/file_layer.py rename to src/karta/core/file_layer.py diff --git a/src/core/function_context.py b/src/karta/core/function_context.py similarity index 100% rename from src/core/function_context.py rename to src/karta/core/function_context.py diff --git a/src/core/matching_engine.py b/src/karta/core/matching_engine.py similarity index 99% rename from src/core/matching_engine.py rename to src/karta/core/matching_engine.py index d7c7fe4..1af302e 100644 --- a/src/core/matching_engine.py +++ b/src/karta/core/matching_engine.py @@ -1,7 +1,9 @@ -from config.utils import * -from collections import defaultdict -import config.anchor as anchor import time +from collections import defaultdict + +from ..config.utils import * +from ..config import anchor + class MatchEngine(object): """A class that handles the book-keeping for the matching process. diff --git a/src/disassembler/IDA/__init__.py b/src/karta/disassembler/IDA/__init__.py similarity index 100% rename from src/disassembler/IDA/__init__.py rename to src/karta/disassembler/IDA/__init__.py diff --git a/src/disassembler/IDA/ida_analysis_api.py b/src/karta/disassembler/IDA/ida_analysis_api.py similarity index 99% rename from src/disassembler/IDA/ida_analysis_api.py rename to src/karta/disassembler/IDA/ida_analysis_api.py index b8ecefc..071b368 100644 --- a/src/disassembler/IDA/ida_analysis_api.py +++ b/src/karta/disassembler/IDA/ida_analysis_api.py @@ -1,8 +1,9 @@ import idaapi import sark -from config.utils import * from hashlib import md5 +from ...config.utils import * + class AnalyzerIDA(object): """Logic instance for the IDA disassembler API. Contains the heart of Karta's canonical representation. @@ -22,6 +23,7 @@ def __init__(self, disas): disas (disassembler): disassembler layer instance """ self.disas = disas + idaapi.auto_wait() def funcNameInner(self, raw_func_name): """Return the name of the function (including windows name fixes). diff --git a/src/disassembler/IDA/ida_api.py b/src/karta/disassembler/IDA/ida_api.py similarity index 99% rename from src/disassembler/IDA/ida_api.py rename to src/karta/disassembler/IDA/ida_api.py index 581acc5..89f417d 100644 --- a/src/disassembler/IDA/ida_api.py +++ b/src/karta/disassembler/IDA/ida_api.py @@ -1,3 +1,5 @@ +import logging + # Dependencies that only exist inside IDA import idautils import idaapi @@ -9,9 +11,9 @@ import sark from .ida_analysis_api import AnalyzerIDA # Basic dependencies (only basic python packages) -from config.utils import * -from disassembler.disas_api import DisasAPI -import logging +from ...config.utils import * +from ..disas_api import DisasAPI + class IdaLogHandler(logging.Handler): """Integrate the log messages with IDA's output window.""" diff --git a/src/karta/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py new file mode 100644 index 0000000..1fc62ea --- /dev/null +++ b/src/karta/disassembler/IDA/ida_cmd_api.py @@ -0,0 +1,133 @@ +import os +import asyncio + +from ..disas_api import DisasCMD +from ..factory import registerDisassemblerCMD + + +class IdaCMD(DisasCMD): + """DisasCMD implementation for the IDA disassembler.""" + + # Overridden base function + @staticmethod + def identify(path): + """Check if the given command-line path refers to this disassembler. + + Args: + path (str): command-line path to some disassembler (maybe for us) + + Return Value: + True iff the command-line path refers to our program + """ + return os.path.split(path)[-1].split(".")[0].lower().startswith("ida") + + # Overridden base function + @staticmethod + def name(): + """Return the program's name (used mainly for bug fixes in our code...). + + Return Value: + String name of the disassembler program + """ + return "IDA" + + # Overridden base function + async def createDatabase(self, binary_file, is_windows): + """Create a database file asynchonously for the given binary file , compiled to windows or linux as specified. + + Args: + binary_file (path): path to the input binary (*.o / *.obj) file + is_windows (bool): True if this is a binary that was compiled for windows (*.obj), False otherwise (*.o) + + Return Value: + path to the created database file + """ + type = "elf" if not is_windows else "coff" + + if not hasattr(self, "is64"): + self.decideArchitecureChoices(binary_file, is_windows) + + database_file = binary_file + self.suffix + # execute the program + process = await asyncio.create_subprocess_exec(self._path, "-A", "-B", f"-T{type}", f"-o{database_file}", binary_file) + await process.wait() + # return back the (should be) created database file path + return database_file + + # Overridden base function + async def executeScript(self, database, script): + """Execute the given script asynchonously over the given database file that was created earlier. + + Args: + database (path): path to a database file created by the same program + script (path): python script to be executed once the database is loaded + """ + process = await asyncio.create_subprocess_exec(self._path, "-A", f"-S{script}", database) + await process.wait() + + def isSupported(self, feature_name): + """Check if feature is enabled. + + Args: + feature_name (str): name of the feature to check if enabled + + Return Value: + returns whether the current class has a member name "feature_name" + """ + return hasattr(self, feature_name) + + async def createAndExecute(self, binary_file, is_windows, script): + """Execute the given script over the given database asynchonously without closing it first. + + Args: + binary_file (str): filename of the binary to analyze + is_windows (bool): whether the file is a windows compiled file or linux compiled file + script (str): filename of the script to use on the binary in ida + """ + type = "elf" if not is_windows else "coff" + + if not hasattr(self, "is64"): + self.decideArchitecureChoices(binary_file, is_windows) + + # execute the program + process = await asyncio.create_subprocess_exec(self._path, "-A", "-c", f"-S{script}", f"-T{type}", binary_file) + await process.wait() + + def decideArchitecureChoices(self, binary_file, is_windows): + """Automate deciding whether to send the files to ida64 or ida. + + Args: + binary_file (str): filename of the file to be a test case for analysis + is_windows (bool): whether the test file is a linux or windows binary + """ + # machine type header of pe + # specified in that order, amd64, arm64, ia64, loongarch64, riscv64 + ARCH64PE = [b"\x64\x86", b"\x64\xaa", b"\x00\x02", b"\x64\x62", b"\x64\x50"] + + # machine type header of elf + ARCH64ELF64 = b"\x02" + + # because we get the path to the folder of the dissasembler + # we need to define whether it is a 64 or 32 bit one + # so we check the respective fields in coff files and elf files + # and define the file ending and the ida to use based on tehm + self._path = os.path.join(self._path, "ida") + self.is64 = False + with open(binary_file, 'rb') as f: + if is_windows: + # read machine type header and check whether it is in a list + # of 64 bit architectures + machine_type = f.read(2) + if machine_type in ARCH64PE: + self.is64 = True + else: + # read the 32bit / 64bit field + machine = f.read(5)[4:] + if machine == ARCH64ELF64: + self.is64 = True + self.suffix = ".i64" if self.is64 else ".idb" + self._path += "64" if self.is64 else "" + + +# Don't forget to register at the factory +registerDisassemblerCMD(IdaCMD.identify, IdaCMD) diff --git a/src/disassembler/IDA/ida_verifier_api.py b/src/karta/disassembler/IDA/ida_verifier_api.py similarity index 91% rename from src/disassembler/IDA/ida_verifier_api.py rename to src/karta/disassembler/IDA/ida_verifier_api.py index a844b95..2dedf4e 100644 --- a/src/disassembler/IDA/ida_verifier_api.py +++ b/src/karta/disassembler/IDA/ida_verifier_api.py @@ -1,5 +1,5 @@ -from disassembler.disas_api import DisasVerifier -from disassembler.factory import registerDisassembler +from ..disas_api import DisasVerifier +from ..factory import registerDisassembler from .ida_cmd_api import IdaCMD class IdaVerifier(DisasVerifier): diff --git a/src/disassembler/__init__.py b/src/karta/disassembler/__init__.py similarity index 100% rename from src/disassembler/__init__.py rename to src/karta/disassembler/__init__.py diff --git a/src/disassembler/disas_api.py b/src/karta/disassembler/disas_api.py similarity index 97% rename from src/disassembler/disas_api.py rename to src/karta/disassembler/disas_api.py index 1ea9752..18261dd 100644 --- a/src/disassembler/disas_api.py +++ b/src/karta/disassembler/disas_api.py @@ -2,6 +2,7 @@ ## Note: This API is an indirection point, so we could (maybe) add support for more disassemblers in the future ## ################################################################################################################## +import os from collections import defaultdict class DisasAPI(object): @@ -527,7 +528,7 @@ def __init__(self, path): Args: path (path): command line path for the program """ - self._path = path + self._path = os.path.normpath(path) @staticmethod def identify(path): @@ -550,8 +551,8 @@ def name(): """ raise NotImplementedError("Subclasses should implement this!") - def createDatabase(self, binary_file, is_windows): - """Create a database file for the given binary file, compiled to windows or linux as specified. + async def createDatabase(self, binary_file, is_windows): + """Create a database file asynchonously for the given binary file, compiled to windows or linux as specified. Args: binary_file (path): path to the input binary (*.o / *.obj) file @@ -562,8 +563,8 @@ def createDatabase(self, binary_file, is_windows): """ raise NotImplementedError("Subclasses should implement this!") - def executeScript(self, database, script): - """Execute the given script over the given database file that was created earlier. + async def executeScript(self, database, script): + """Execute the given script asynchonously over the given database file that was created earlier. Args: database (path): path to a database file created by the same program @@ -571,6 +572,14 @@ def executeScript(self, database, script): """ raise NotImplementedError("Subclasses should implement this!") + def isSupported(self, feature_name): + """Allow script to query whether a feature is enabled in it's disas_api. + + Return Value: + False, no featue is supported on this Abstract class + """ + return False + class DisasVerifier(object): """Abstract class that verifies that we are running inside our matching disassembler. diff --git a/src/disassembler/factory.py b/src/karta/disassembler/factory.py similarity index 100% rename from src/disassembler/factory.py rename to src/karta/disassembler/factory.py diff --git a/src/file_layer.py b/src/karta/file_layer.py similarity index 99% rename from src/file_layer.py rename to src/karta/file_layer.py index de29965..9df1d86 100644 --- a/src/file_layer.py +++ b/src/karta/file_layer.py @@ -1,6 +1,6 @@ -from core.file_layer import * -from config.utils import * -import config.anchor as anchor +from .core.file_layer import * +from .config.utils import * +from .config import anchor class FileMatcher(FileMatch): """A wrapper for Matched Files with an additional matching logic layer. diff --git a/src/function_context.py b/src/karta/function_context.py similarity index 99% rename from src/function_context.py rename to src/karta/function_context.py index 7aa31aa..ac66f6c 100644 --- a/src/function_context.py +++ b/src/karta/function_context.py @@ -1,7 +1,8 @@ -from core.function_context import * -from config.utils import * from collections import defaultdict -import config.libc_config as libc + +from .core.function_context import * +from .config.utils import * +from .config import libc_config as libc class ExternalFunction(CodeContext): """A context that describes a source-code point of view of an external function, using references from/to it. diff --git a/src/karta/installers/__init__.py b/src/karta/installers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/karta/installers/common_installer.py b/src/karta/installers/common_installer.py new file mode 100644 index 0000000..692ab8d --- /dev/null +++ b/src/karta/installers/common_installer.py @@ -0,0 +1,65 @@ +import os +import platform + +from ..config.utils import addDisassembler, getDisassembler, setDefaultDisassembler + + +def commonLocations(): + """Get a list of common places where a disassembler may be installed on any os.""" + location_list = list() + # based on the operating system installation location may vary + system_lower = platform.system().lower() + if system_lower == "windows".lower(): + # add drives + from ctypes import windll + kernel32 = windll.kernel32 + logical_drives = kernel32.GetLogicalDrives() + current_letter = 'A' + while logical_drives > 0: + if logical_drives & 1 == 1: + location_list.append(current_letter + ':\\') + logical_drives >>= 1 + current_letter = chr(ord(current_letter) + 1) + # add program files + location_list.append(os.environ['ProgramFiles']) + elif system_lower == "linux".lower(): + location_list.append("~") + location_list.append("/opt") + return location_list + + +def detectInstallation(pattern, installation_file): + """Detect a disassembler's installation by regex, if not found get it with input. + + Args: + pattern (re.Pattern): pattern to detect an installation + installation_file (str): name of the file to save the disassembler name in + + Return Value: + Directory of the disassembler file + """ + disassembler = getDisassembler(installation_file) + if disassembler is not None: + return disassembler + + for location in commonLocations(): + for directory in next(os.walk(location))[1]: + if pattern.match(directory): + install_directory = os.path.join(location, directory) + addDisassembler(installation_file, install_directory) + return install_directory + try: + disass_path = input("Please enter the path to the folder of the disassembler: ") + if os.path.isdir(disass_path): + path = os.path.abspath(disass_path) + addDisassembler(installation_file, path) + return path + else: + print("The path you entered is not a directory") + except KeyboardInterrupt: + exit(0) + + +def commonSetDefaultDisassembler(disassembler_name): + """Call config utils setDefaultDisassembler function.""" + setDefaultDisassembler(disassembler_name) diff --git a/src/karta/installers/ida_installer.py b/src/karta/installers/ida_installer.py new file mode 100644 index 0000000..b4882d4 --- /dev/null +++ b/src/karta/installers/ida_installer.py @@ -0,0 +1,36 @@ +import os +import re +from shutil import copyfile + +from .common_installer import detectInstallation, commonSetDefaultDisassembler + + +disassembler_name = "ida_path" + +def detectIda(save_file): + """Find where ida is installed by this regex, should work for both windows and linux. + + Args: + save_file (str): name of the file which will save the path to the disassembler directory + + Return Value: + Installation folder for the ida pro disassembler + """ + pattern = re.compile("ida( pro )?-?\\d\\.\\d", re.IGNORECASE) + return detectInstallation(pattern, save_file) + +def main(): + """Detect ida copy plugin file into its plugins directory.""" + path = detectIda(disassembler_name) + commonSetDefaultDisassembler(disassembler_name) + src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + ida_plugin = os.path.join(src_dir, "plugins", "ida_plugin.py") + plugin_dst = os.path.join(path, "plugins", "ida_karta.py") + copyfile( + ida_plugin, + plugin_dst + ) + + +if __name__ == "__main__": + main() diff --git a/src/karta_analyze_src.py b/src/karta/karta_analyze_src.py old mode 100755 new mode 100644 similarity index 63% rename from src/karta_analyze_src.py rename to src/karta/karta_analyze_src.py index 6ed3ca2..e395939 --- a/src/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -1,17 +1,19 @@ -#!/usr/bin/python +#!/usr/bin/python3 -from ar_parser import getArchiveFiles -from config.utils import * -from elementals import Prompter, ProgressBar -from disassembler.factory import identifyDisassemblerHandler -from disassembler.IDA import ida_cmd_api -from function_context import SourceContext, BinaryContext, IslandContext -import config.anchor as anchor - -import os import sys import argparse import logging +import asyncio +from os import path, walk +from elementals import Prompter, ProgressBar + +from .ar_parser import getArchiveFiles +from .config.utils import * +from .disassembler.factory import identifyDisassemblerHandler +from .disassembler.IDA import ida_cmd_api +from .function_context import SourceContext, BinaryContext, IslandContext +from .config import anchor + #################### ## Global Configs ## @@ -36,24 +38,67 @@ def locateFiles(bin_dir, file_list, suffix): Return Value: Generator for a tuples of the form: (abs_path, compiled_file file name) """ - for root, dirs, files in os.walk(bin_dir): + for root, dirs, files in walk(bin_dir): if file_list is not None: for compiled_file in set(files) & set(file_list): - yield os.path.abspath(os.path.join(root, compiled_file)), compiled_file + yield path.abspath(path.join(root, compiled_file)), compiled_file file_list.remove(compiled_file) else: for file in filter(lambda x: x.endswith("." + suffix), files): - yield os.path.abspath(os.path.join(root, file)), file + yield path.abspath(path.join(root, file)), file -def analyzeFile(full_file_path, is_windows): - """Analyze a single file using analyzer script. +async def analyzeFile(full_file_path, is_windows): + """Analyze a single file asynchonously using analyzer script. Args: full_file_path (str): full path to the specific (*.obj / *.o) file is_windows (bool): True iff a windows compilation (*.obj or *.o) """ - database_path = disas_cmd.createDatabase(full_file_path, is_windows) - disas_cmd.executeScript(database_path, SCRIPT_PATH) + if disas_cmd.isSupported("createAndExecute"): + await disas_cmd.createAndExecute(full_file_path, is_windows, SCRIPT_PATH) + else: + database_path = await disas_cmd.createDatabase(full_file_path, is_windows) + await disas_cmd.executeScript(database_path, SCRIPT_PATH) + +async def processFile(full_file_path, is_windows, compiled_file, progress_bar, prompter, semaphore): + """Analyze a file asynchonously using a disassembler and parse the file stats. + + Args: + full_file_path (str): full path to the specific (*.obj / *.o) file + is_windows (bool): True iff a windows compilation (*.obj or *.o) + compiled_file (str): file name of the compiled_file + progress_bar (progressBar): progress bar to update once file processing is finished + prompter (prompter): notify the user when an error occured + semaphore (semaphore): release it when the disassembler finished processing our file + """ + # run the analsys on the files in an asynchrnous way + prompter.debug(f"{full_file_path} - {compiled_file}") + if progress_bar is None: + prompter.info(f"{compiled_file} - {full_file_path}") + # no need to analyze the file again if the process was canceled and the file state was saved + if not os.path.exists(full_file_path + STATE_FILE_SUFFIX): + await analyzeFile(full_file_path, is_windows) + # load the JSON data from it + try: + fd = open(full_file_path + STATE_FILE_SUFFIX, "r") + except IOError: + prompter.error(f"Failed to create the .JSON file for file: {compiled_file}") + prompter.error("Read the log file for more information:") + prompter.addIndent() + prompter.error(constructInitLogPath(full_file_path)) + prompter.error(constructLogPath(full_file_path)) + prompter.removeIndent() + prompter.removeIndent() + prompter.removeIndent() + prompter.error("Encountered an error, exiting") + exit(1) + # all was OK, can continue + parseFileStats(full_file_path, json.load(fd)) + fd.close() + if progress_bar is not None: + progress_bar.advance(1) + # release semaphore and allow an instance to run on another file + semaphore.release() def resolveUnknowns(): """Resolve "unknown" references between the different compiled files.""" @@ -67,7 +112,7 @@ def resolveUnknowns(): src_func_ctx.recordCall(resolved_call) src_func_ctx.unknown_fptrs.clear() -def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): +async def analyzeLibrary(config_name, bin_dirs, compiled_ars, concurrency, prompter): """Analyze the open source library, file-by-file and merge the results. Args: @@ -113,37 +158,20 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): progress_bar = None # it makes more sense to have a sorted list archive_files.sort() - # start the work itself + # analyze many files at the same time, actions are parallel + semaphore = asyncio.Semaphore(concurrency) for full_file_path, compiled_file in archive_files: # ida has severe bugs, make sure to warn the user in advance if disas_cmd.name() == "IDA" and ' ' in full_file_path: prompter.error("IDA does not support spaces (' ') in the file's path (in script mode). Please move the binary directory accordingly (I feel your pain)") prompter.removeIndent() return - prompter.debug(f"{full_file_path} - {compiled_file}") - if progress_bar is None: - prompter.info(f"{compiled_file} - {full_file_path}") - # analyze the file - analyzeFile(full_file_path, is_windows) - # load the JSON data from it - try: - fd = open(full_file_path + STATE_FILE_SUFFIX, "r") - except IOError: - prompter.error(f"Failed to create the .JSON file for file: {compiled_file}") - prompter.error("Read the log file for more information:") - prompter.addIndent() - prompter.error(constructInitLogPath(full_file_path)) - prompter.error(constructLogPath(full_file_path)) - prompter.removeIndent() - prompter.removeIndent() - prompter.removeIndent() - prompter.error("Encountered an error, exiting") - exit(1) - # all was OK, can continue - parseFileStats(full_file_path, json.load(fd)) - fd.close() - if progress_bar is not None: - progress_bar.advance(1) + await semaphore.acquire() + # start the work itself + asyncio.create_task(processFile(full_file_path, is_windows, compiled_file, progress_bar, prompter, semaphore)) + # we want to wait for all the disassemblers to finish + for _ in range(concurrency): + await semaphore.acquire() # wrap it up if progress_bar is not None: progress_bar.finish() @@ -220,12 +248,12 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): file_dict = {} # find a common file prefix, and remove it form the file path if len(src_file_mappings) > 1: - base_value = list(src_file_mappings.keys())[0].split(os.path.sep) - comparison_value = list(src_file_mappings.keys())[-1].split(os.path.sep) + base_value = list(src_file_mappings.keys())[0].split(path.sep) + comparison_value = list(src_file_mappings.keys())[-1].split(path.sep) for index in range(min(len(comparison_value), len(base_value))): if base_value[index] != comparison_value[index]: break - common_path_len = len(os.path.sep.join(base_value[:index])) + 1 + common_path_len = len(path.sep.join(base_value[:index])) + 1 else: common_path_len = len(bin_dirs[0]) + 1 @@ -243,7 +271,49 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): prompter.info(f"Anchor to function ratio is: {len(anchors_list)}/{len(src_functions_list)}") prompter.removeIndent() -def main(args): +def verifyArchivesAndObjects(using_archives, couples, prompter): + """Verify that the archive and object files karta processes are valid. + + Args: + using_archives (bool): is karta analyzing object files only or library files too + couples (list): couples of dir and lib files or only bin dirs according to the using_archives parameter + prompter (prompter): prompter to notify the user if any error occures + """ + bin_dirs = [] + archive_paths = [] + error_occured = False + if using_archives: + for i in range(0, len(couples), 2): + if not path.exists(couples[i]): + prompter.error(f"Error the path {couples[i]} does not exist!") + error_occured = True + elif not path.isdir(couples[i]): + prompter.error(f"Error the path {couples[i]} exists but is not a directory!") + error_occured = True + else: + bin_dirs.append(couples[i]) + + if not path.exists(couples[i + 1]): + prompter.error(f"Error the path {couples[i + 1]} does not exist!") + error_occured = True + elif not path.isfile(couples[i + 1]): + prompter.error(f"Error the path {couples[i + 1]} exists but is not a file!") + error_occured = True + else: + archive_paths.append(couples[i + 1]) + else: + for bin_dir in couples: + if not path.exists(bin_dir): + prompter.error(f"Error the path {bin_dir} does not exist!") + error_occured = True + elif not path.isdir(bin_dir): + prompter.error(f"Error the path {bin_dir} exists but is not a directory!") + error_occured = True + bin_dirs = couples + + return (bin_dirs, archive_paths) if not error_occured else None + +def main(args=None): """Create a .json configuration for the open source library version. Args: @@ -259,6 +329,7 @@ def main(args): help="version string (case sensitive) as used by the identifier") parser.add_argument("couples", metavar="dir archive", type=str, nargs="+", help="directory with the compiled *.o / *.obj files + path to the matching *.a / *.lib file (if didn't use \"--no-archive\")") + parser.add_argument("-C", "--concurrency", default=5, type=int, help="number of disassemblers to run in parallel") parser.add_argument("-D", "--debug", action="store_true", help="set logging level to logging.DEBUG") parser.add_argument("-N", "--no-archive", action="store_false", help="extract data from all *.o / *.obj files in the directory") parser.add_argument("-W", "--windows", action="store_true", help="signals that the binary was compiled for Windows") @@ -271,25 +342,25 @@ def main(args): is_windows = args.windows using_archives = args.no_archive couples = args.couples + concurrency = args.concurrency - bin_dirs = [] - archive_paths = [] + # open the log + prompter = Prompter(min_log_level=logging.INFO if not is_debug else logging.DEBUG) if using_archives: if len(couples) % 2 != 0: parser.error("Odd length in list of dir,archive couples, should be: [(directory, archive name), ...]") - for i in range(0, len(couples), 2): - bin_dirs.append(couples[i]) - archive_paths.append(couples[i + 1]) - else: - bin_dirs = couples - # open the log - prompter = Prompter(min_log_level=logging.INFO if not is_debug else logging.DEBUG) + paths = verifyArchivesAndObjects(using_archives, couples, prompter) + + if paths is None: + return + + bin_dirs, archive_paths = paths + prompter.info("Starting the Script") # requesting the path to the chosen disassembler - setDisassemblerPath(prompter) - disas_cmd = identifyDisassemblerHandler(getDisasPath(), prompter) + disas_cmd = identifyDisassemblerHandler(getDisasPath(prompter), prompter) if disas_cmd is None: return @@ -300,14 +371,8 @@ def main(args): if is_windows: setWindowsMode() - # Check if launched from the src directory - if not os.path.exists(SCRIPT_PATH): - prompter.error("The script should be executed from Karta's src directory!") - prompter.error("Exiting") - return - # analyze the open source library - analyzeLibrary(constructConfigPath(library_name, library_version), bin_dirs, archive_paths, prompter) + asyncio.run(analyzeLibrary(constructConfigPath(library_name, library_version), bin_dirs, archive_paths, concurrency, prompter)) # finished prompter.info("Finished Successfully") diff --git a/src/karta_identifier.py b/src/karta/karta_identifier.py similarity index 97% rename from src/karta_identifier.py rename to src/karta/karta_identifier.py index 38bf731..aadc55d 100644 --- a/src/karta_identifier.py +++ b/src/karta/karta_identifier.py @@ -1,9 +1,9 @@ -from config.utils import * -from disassembler.factory import createDisassemblerHandler -from libs import lib_factory - -from elementals import Logger import logging +from elementals import Logger + +from karta.config.utils import * +from karta.disassembler.factory import createDisassemblerHandler +from karta.libs import lib_factory #################### ## Global Configs ## diff --git a/src/karta_manual_anchor.py b/src/karta/karta_manual_anchor.py old mode 100755 new mode 100644 similarity index 98% rename from src/karta_manual_anchor.py rename to src/karta/karta_manual_anchor.py index 9916fb8..ba1065d --- a/src/karta_manual_anchor.py +++ b/src/karta/karta_manual_anchor.py @@ -1,14 +1,14 @@ -#!/usr/bin/python - -from config.utils import * -from elementals import Prompter -from function_context import SourceContext, BinaryContext, IslandContext +#!/usr/bin/python3 import os import sys import argparse import logging from collections import defaultdict +from elementals import Prompter + +from karta.config.utils import * +from karta.function_context import SourceContext, BinaryContext, IslandContext def recordManualAnchors(library_config, knowledge_config, lib_name, prompter): """Record the list of user defined manual anchor matches. diff --git a/src/karta_manual_identifier.py b/src/karta/karta_manual_identifier.py similarity index 97% rename from src/karta_manual_identifier.py rename to src/karta/karta_manual_identifier.py index bbc6c0d..c4bcacc 100644 --- a/src/karta_manual_identifier.py +++ b/src/karta/karta_manual_identifier.py @@ -1,13 +1,14 @@ -#!/usr/bin/python - -from config.utils import * -from elementals import Prompter +#!/usr/bin/python3 import sys import argparse import logging from collections import defaultdict -from libs import lib_factory +from elementals import Prompter + +from karta.config.utils import * +from karta.libs import lib_factory + def recordManualVersions(knowledge_config, prompter): """Record the list of user defined manual library versions. diff --git a/src/karta_matcher.py b/src/karta/karta_matcher.py similarity index 97% rename from src/karta_matcher.py rename to src/karta/karta_matcher.py index 430cbe7..c142c65 100644 --- a/src/karta_matcher.py +++ b/src/karta/karta_matcher.py @@ -1,10 +1,10 @@ -from config.utils import * -from disassembler.factory import createDisassemblerHandler -from matching_engine import KartaMatcher -from libs import lib_factory - -from elementals import Logger import logging +from elementals import Logger + +from karta.config.utils import * +from karta.disassembler.factory import createDisassemblerHandler +from karta.matching_engine import KartaMatcher +from karta.libs import lib_factory ###################### ## Global Variables ## diff --git a/src/libs/__init__.py b/src/karta/libs/__init__.py similarity index 100% rename from src/libs/__init__.py rename to src/karta/libs/__init__.py diff --git a/src/libs/gsoap.py b/src/karta/libs/gsoap.py similarity index 100% rename from src/libs/gsoap.py rename to src/karta/libs/gsoap.py diff --git a/src/libs/icu.py b/src/karta/libs/icu.py similarity index 100% rename from src/libs/icu.py rename to src/karta/libs/icu.py diff --git a/src/libs/lib_factory.py b/src/karta/libs/lib_factory.py similarity index 100% rename from src/libs/lib_factory.py rename to src/karta/libs/lib_factory.py diff --git a/src/libs/lib_template.py b/src/karta/libs/lib_template.py similarity index 99% rename from src/libs/lib_template.py rename to src/karta/libs/lib_template.py index f6ba387..626558d 100644 --- a/src/libs/lib_template.py +++ b/src/karta/libs/lib_template.py @@ -1,6 +1,7 @@ -from .lib_factory import registerLibrary import string +from .lib_factory import registerLibrary + class Seeker(object): """Abstract class that represents a basic library seeker. diff --git a/src/libs/libjpeg.py b/src/karta/libs/libjpeg.py similarity index 99% rename from src/libs/libjpeg.py rename to src/karta/libs/libjpeg.py index b7f2167..afd4c66 100644 --- a/src/libs/libjpeg.py +++ b/src/karta/libs/libjpeg.py @@ -1,6 +1,7 @@ -from .lib_template import * import string +from .lib_template import * + class LibJPEGSeeker(Seeker): """Seeker (Identifier) for the libjpeg (ITU) open source library.""" diff --git a/src/libs/libjpeg_turbo.py b/src/karta/libs/libjpeg_turbo.py similarity index 100% rename from src/libs/libjpeg_turbo.py rename to src/karta/libs/libjpeg_turbo.py diff --git a/src/libs/libpng.py b/src/karta/libs/libpng.py similarity index 99% rename from src/libs/libpng.py rename to src/karta/libs/libpng.py index a91f232..9a0518b 100644 --- a/src/libs/libpng.py +++ b/src/karta/libs/libpng.py @@ -1,5 +1,5 @@ from .lib_template import * -from config.utils import getDisas +from ..config.utils import getDisas class LibpngSeeker(Seeker): """Seeker (Identifier) for the libpng open source library.""" diff --git a/src/libs/libtiff.py b/src/karta/libs/libtiff.py similarity index 100% rename from src/libs/libtiff.py rename to src/karta/libs/libtiff.py diff --git a/src/libs/libvpx.py b/src/karta/libs/libvpx.py similarity index 100% rename from src/libs/libvpx.py rename to src/karta/libs/libvpx.py diff --git a/src/libs/libxml2.py b/src/karta/libs/libxml2.py similarity index 100% rename from src/libs/libxml2.py rename to src/karta/libs/libxml2.py diff --git a/src/libs/mactelnet.py b/src/karta/libs/mactelnet.py similarity index 100% rename from src/libs/mactelnet.py rename to src/karta/libs/mactelnet.py diff --git a/src/libs/mdnsresponder.py b/src/karta/libs/mdnsresponder.py similarity index 100% rename from src/libs/mdnsresponder.py rename to src/karta/libs/mdnsresponder.py diff --git a/src/libs/netsnmp.py b/src/karta/libs/netsnmp.py similarity index 100% rename from src/libs/netsnmp.py rename to src/karta/libs/netsnmp.py diff --git a/src/libs/openssh.py b/src/karta/libs/openssh.py similarity index 100% rename from src/libs/openssh.py rename to src/karta/libs/openssh.py diff --git a/src/libs/openssl.py b/src/karta/libs/openssl.py similarity index 99% rename from src/libs/openssl.py rename to src/karta/libs/openssl.py index abf76cf..82179d9 100644 --- a/src/libs/openssl.py +++ b/src/karta/libs/openssl.py @@ -1,6 +1,7 @@ -from .lib_template import * import string +from .lib_template import * + class OpenSSLSeeker(Seeker): """Seeker (Identifier) for the OpenSSL open source library.""" diff --git a/src/libs/treck.py b/src/karta/libs/treck.py similarity index 100% rename from src/libs/treck.py rename to src/karta/libs/treck.py diff --git a/src/libs/zlib.py b/src/karta/libs/zlib.py similarity index 99% rename from src/libs/zlib.py rename to src/karta/libs/zlib.py index 7f255f0..20ba91e 100644 --- a/src/libs/zlib.py +++ b/src/karta/libs/zlib.py @@ -1,6 +1,7 @@ -from .lib_template import * from collections import defaultdict +from .lib_template import * + class ZlibSeeker(Seeker): """Seeker (Identifier) for the zlib open source library.""" diff --git a/src/matching_engine.py b/src/karta/matching_engine.py similarity index 99% rename from src/matching_engine.py rename to src/karta/matching_engine.py index ef59340..c83476c 100644 --- a/src/matching_engine.py +++ b/src/karta/matching_engine.py @@ -1,11 +1,13 @@ -from core.matching_engine import MatchEngine -from core.file_layer import AssumptionException -from config.utils import * -from file_layer import FileMatcher -from function_context import ExternalFunction, SourceContext, BinaryContext, IslandContext -from collections import defaultdict -import config.libc_config as libc import sys +from collections import defaultdict + +from .core.matching_engine import MatchEngine +from .core.file_layer import AssumptionException +from .config.utils import * +from .file_layer import FileMatcher +from .function_context import ExternalFunction, SourceContext, BinaryContext, IslandContext +from .config import libc_config as libc + class KartaMatcher(MatchEngine): """Complete matching engine logic for Karta, based on the file matching logic of the base MatchEngine. diff --git a/src/karta/plugins/__init__.py b/src/karta/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/karta/plugins/ida_plugin.py b/src/karta/plugins/ida_plugin.py new file mode 100644 index 0000000..46117e2 --- /dev/null +++ b/src/karta/plugins/ida_plugin.py @@ -0,0 +1 @@ +pass diff --git a/src/thumbs_up/README.md b/src/karta/thumbs_up/README.md similarity index 100% rename from src/thumbs_up/README.md rename to src/karta/thumbs_up/README.md diff --git a/src/karta/thumbs_up/__init__.py b/src/karta/thumbs_up/__init__.py new file mode 100644 index 0000000..f5c8111 --- /dev/null +++ b/src/karta/thumbs_up/__init__.py @@ -0,0 +1,2 @@ +from . import thumbs_up_ELF +from . import thumbs_up_firmware diff --git a/src/thumbs_up/analyzer_utils.py b/src/karta/thumbs_up/analyzer_utils.py similarity index 99% rename from src/thumbs_up/analyzer_utils.py rename to src/karta/thumbs_up/analyzer_utils.py index c0f1ed6..e94707a 100644 --- a/src/thumbs_up/analyzer_utils.py +++ b/src/karta/thumbs_up/analyzer_utils.py @@ -4,8 +4,9 @@ import ida_funcs import idaapi from collections import defaultdict -from utils.code_metric import CodeMetric -from utils.code_regions import CodeRegion, CodeRegions + +from .utils.code_metric import CodeMetric +from .utils.code_regions import CodeRegion, CodeRegions def gatherIntel(analyzer, scs, sds): """Gather all of the intelligence about the program's different features. diff --git a/src/thumbs_up/analyzers/__init__.py b/src/karta/thumbs_up/analyzers/__init__.py similarity index 100% rename from src/thumbs_up/analyzers/__init__.py rename to src/karta/thumbs_up/analyzers/__init__.py diff --git a/src/thumbs_up/analyzers/analyzer.py b/src/karta/thumbs_up/analyzers/analyzer.py similarity index 100% rename from src/thumbs_up/analyzers/analyzer.py rename to src/karta/thumbs_up/analyzers/analyzer.py diff --git a/src/thumbs_up/analyzers/analyzer_factory.py b/src/karta/thumbs_up/analyzers/analyzer_factory.py similarity index 100% rename from src/thumbs_up/analyzers/analyzer_factory.py rename to src/karta/thumbs_up/analyzers/analyzer_factory.py diff --git a/src/thumbs_up/analyzers/arm.py b/src/karta/thumbs_up/analyzers/arm.py similarity index 96% rename from src/thumbs_up/analyzers/arm.py rename to src/karta/thumbs_up/analyzers/arm.py index 531dcf6..3583489 100644 --- a/src/thumbs_up/analyzers/arm.py +++ b/src/karta/thumbs_up/analyzers/arm.py @@ -1,11 +1,12 @@ import idc + from .analyzer import Analyzer from .analyzer_factory import registerAnalyzer -from utils.function import FunctionClassifier -from utils.strings import StringIdentifier -from utils.local_constants import LocalsIdentifier -from utils.fptr import FptrIdentifier -from utils.switch_table import SwitchIdentifier +from ..utils.function import FunctionClassifier +from ..utils.strings import StringIdentifier +from ..utils.local_constants import LocalsIdentifier +from ..utils.fptr import FptrIdentifier +from ..utils.switch_table import SwitchIdentifier ############################### # Architecture Configurations # diff --git a/src/thumbs_up/analyzers/intel.py b/src/karta/thumbs_up/analyzers/intel.py similarity index 95% rename from src/thumbs_up/analyzers/intel.py rename to src/karta/thumbs_up/analyzers/intel.py index de2ec5e..57188e7 100644 --- a/src/thumbs_up/analyzers/intel.py +++ b/src/karta/thumbs_up/analyzers/intel.py @@ -1,10 +1,10 @@ from .analyzer import Analyzer from .analyzer_factory import registerAnalyzer -from utils.function import FunctionClassifier -from utils.strings import StringIdentifier -from utils.local_constants import LocalsIdentifier -from utils.fptr import FptrIdentifier -from utils.switch_table import SwitchIdentifier +from ..utils.function import FunctionClassifier +from ..utils.strings import StringIdentifier +from ..utils.local_constants import LocalsIdentifier +from ..utils.fptr import FptrIdentifier +from ..utils.switch_table import SwitchIdentifier ############################### # Architecture Configurations # diff --git a/src/thumbs_up/analyzers/mips.py b/src/karta/thumbs_up/analyzers/mips.py similarity index 96% rename from src/thumbs_up/analyzers/mips.py rename to src/karta/thumbs_up/analyzers/mips.py index 3333986..14b6e65 100644 --- a/src/thumbs_up/analyzers/mips.py +++ b/src/karta/thumbs_up/analyzers/mips.py @@ -1,11 +1,12 @@ import idc + from .analyzer import Analyzer from .analyzer_factory import registerAnalyzer -from utils.function import FunctionClassifier -from utils.strings import StringIdentifier -from utils.local_constants import LocalsIdentifier -from utils.fptr import FptrIdentifier -from utils.switch_table import SwitchIdentifier +from ..utils.function import FunctionClassifier +from ..utils.strings import StringIdentifier +from ..utils.local_constants import LocalsIdentifier +from ..utils.fptr import FptrIdentifier +from ..utils.switch_table import SwitchIdentifier ############################### # Architecture Configurations # diff --git a/src/thumbs_up/thumbs_up_ELF.py b/src/karta/thumbs_up/thumbs_up_ELF.py similarity index 98% rename from src/thumbs_up/thumbs_up_ELF.py rename to src/karta/thumbs_up/thumbs_up_ELF.py index 9db8f61..f934367 100644 --- a/src/thumbs_up/thumbs_up_ELF.py +++ b/src/karta/thumbs_up/thumbs_up_ELF.py @@ -2,8 +2,9 @@ import idc import logging from elementals import Logger -from analyzer_utils import * -from analyzers.analyzer_factory import createAnalyzer + +from .analyzer_utils import * +from .analyzers.analyzer_factory import createAnalyzer ## # Taken from Karta :) diff --git a/src/thumbs_up/thumbs_up_firmware.py b/src/karta/thumbs_up/thumbs_up_firmware.py similarity index 98% rename from src/thumbs_up/thumbs_up_firmware.py rename to src/karta/thumbs_up/thumbs_up_firmware.py index 7981488..a04098f 100644 --- a/src/thumbs_up/thumbs_up_firmware.py +++ b/src/karta/thumbs_up/thumbs_up_firmware.py @@ -2,8 +2,9 @@ import idc import logging from elementals import Logger -from analyzer_utils import * -from analyzers.analyzer_factory import createAnalyzer + +from .analyzer_utils import * +from .analyzers.analyzer_factory import createAnalyzer ## # Taken from Karta :) diff --git a/src/thumbs_up/utils/__init__.py b/src/karta/thumbs_up/utils/__init__.py similarity index 100% rename from src/thumbs_up/utils/__init__.py rename to src/karta/thumbs_up/utils/__init__.py diff --git a/src/thumbs_up/utils/code_metric.py b/src/karta/thumbs_up/utils/code_metric.py similarity index 100% rename from src/thumbs_up/utils/code_metric.py rename to src/karta/thumbs_up/utils/code_metric.py diff --git a/src/thumbs_up/utils/code_regions.py b/src/karta/thumbs_up/utils/code_regions.py similarity index 100% rename from src/thumbs_up/utils/code_regions.py rename to src/karta/thumbs_up/utils/code_regions.py diff --git a/src/thumbs_up/utils/fptr.py b/src/karta/thumbs_up/utils/fptr.py similarity index 99% rename from src/thumbs_up/utils/fptr.py rename to src/karta/thumbs_up/utils/fptr.py index 2d190a6..4163f39 100644 --- a/src/thumbs_up/utils/fptr.py +++ b/src/karta/thumbs_up/utils/fptr.py @@ -1,5 +1,3 @@ -from collections import defaultdict -from .pattern_observer import pad import pickle import struct import string @@ -7,6 +5,9 @@ import ida_bytes import ida_funcs import sark +from collections import defaultdict + +from .pattern_observer import pad ########################### ## Static Magic Constant ## diff --git a/src/thumbs_up/utils/function.py b/src/karta/thumbs_up/utils/function.py similarity index 100% rename from src/thumbs_up/utils/function.py rename to src/karta/thumbs_up/utils/function.py index e46b8cd..ca24271 100644 --- a/src/thumbs_up/utils/function.py +++ b/src/karta/thumbs_up/utils/function.py @@ -1,12 +1,12 @@ -from sklearn import metrics -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split import idc import ida_nalt import sark import numpy import struct import time +from sklearn import metrics +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import train_test_split ####################### ## Static Thresholds ## diff --git a/src/thumbs_up/utils/local_constants.py b/src/karta/thumbs_up/utils/local_constants.py similarity index 99% rename from src/thumbs_up/utils/local_constants.py rename to src/karta/thumbs_up/utils/local_constants.py index 4b1e0d3..e6058c7 100644 --- a/src/thumbs_up/utils/local_constants.py +++ b/src/karta/thumbs_up/utils/local_constants.py @@ -1,8 +1,9 @@ -from .pattern_observer import pad import idc import ida_bytes import sark +from .pattern_observer import pad + class LocalsIdentifier: """A class that collects the information and holds the knowledge we know about local (in-code) constants in the program. diff --git a/src/thumbs_up/utils/pattern_observer.py b/src/karta/thumbs_up/utils/pattern_observer.py similarity index 100% rename from src/thumbs_up/utils/pattern_observer.py rename to src/karta/thumbs_up/utils/pattern_observer.py diff --git a/src/thumbs_up/utils/strings.py b/src/karta/thumbs_up/utils/strings.py similarity index 99% rename from src/thumbs_up/utils/strings.py rename to src/karta/thumbs_up/utils/strings.py index 718a313..ca8c4fe 100644 --- a/src/thumbs_up/utils/strings.py +++ b/src/karta/thumbs_up/utils/strings.py @@ -1,10 +1,11 @@ -from .pattern_observer import AlignmentPattern, pad, padSize import idc import ida_bytes import idautils import string import sark +from .pattern_observer import AlignmentPattern, pad, padSize + ####################### ## String Heuristics ## ####################### diff --git a/src/thumbs_up/utils/switch_table.py b/src/karta/thumbs_up/utils/switch_table.py similarity index 99% rename from src/thumbs_up/utils/switch_table.py rename to src/karta/thumbs_up/utils/switch_table.py index 7764b96..a192f56 100644 --- a/src/thumbs_up/utils/switch_table.py +++ b/src/karta/thumbs_up/utils/switch_table.py @@ -3,6 +3,7 @@ import ida_bytes import idaapi import sark + from .pattern_observer import AlignmentPattern, CodePattern, pad ########################### diff --git a/src/thumbs_up/__init__.py b/src/thumbs_up/__init__.py deleted file mode 100644 index 3015824..0000000 --- a/src/thumbs_up/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import thumbs_up_ELF -import thumbs_up_firmware diff --git a/tests.py b/tests.py index eb09377..fc6054b 100644 --- a/tests.py +++ b/tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import pydocstyle import os