From 0f3ef2b1839a063cc04f074aaa21f594966ed80c Mon Sep 17 00:00:00 2001 From: Harelon Date: Fri, 21 Jan 2022 00:42:35 +0200 Subject: [PATCH 01/11] changed 'setup.py install' to 'pip3 install .' and changed shebang to python3 --- .gitignore | 22 ++-------------------- README.md | 2 +- docs/Installation.md | 4 ++-- setup.py | 4 ++-- src/karta_analyze_src.py | 2 +- src/karta_manual_anchor.py | 2 +- src/karta_manual_identifier.py | 2 +- tests.py | 2 +- 8 files changed, 11 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 4df859f..73aa6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,28 +1,10 @@ -src/*.vpj -src/*.vpw -src/*.vpwhist -src/*.vtg -src/*.pyc -src/libs/*.pyc -src/config/*.pyc -src/core/*.pyc -src/disassembler/*.pyc -src/disassembler/IDA/*.pyc +*.pyc +__pycache__ /build/* /dist/* /Karta.egg-info/* docs/_build/* -docs/_build/*/* -docs/_build/*/*/* -docs/_build/*/*/*/* docs/_static/* docs/_templates/* -/*.pyc -/*/*.pyc -/*/*/*.pyc -/*/*/*/*.pyc -/src/thumbs_up/analyzers/__pycache__/*.pyc -/src/*/*/__pycache__/*.pyc -/src/*/*/__pycache__/*.pyc diff --git a/README.md b/README.md index dfc776d..36dd2ef 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ We believe that there are 3 main use cases for this IDA plugin: https://karta.readthedocs.io/ ## Installation (Python 3 & IDA >= 7.4) -For the latest versions, using Python 3, simply git clone the repository and run the ```setup.py install``` script. +For the latest versions, using Python 3, simply git clone the repository and run ```pip3 install .```. Python 3 is supported since versions v2.0.0 and above. ## Installation (Python 2 & IDA < 7.4) diff --git a/docs/Installation.md b/docs/Installation.md index 292a793..e4057b7 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -7,9 +7,9 @@ Prerequisites * [sark](https://github.com/tmr232/Sark) Using the ```setup.py``` script, one can install all of these prerequisites, and be ready to go: -```./setup.py install``` +```pip3 install .``` Installing the Plugin ------------------------ -Nothing should be done :) +Nothing should be done :)aa There is no need to copy the directory to some IDA plugin folder. Instead, once your binary is loaded to IDA, simply start the desired plugin by loading it using the ```File->Script File...``` menu. That's it. \ No newline at end of file diff --git a/setup.py b/setup.py index 1801de7..a7950f2 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 from setuptools import setup, find_packages -from codecs import open with open("README.md", "r") as fh: long_description = fh.read() @@ -17,6 +16,7 @@ license='MIT', packages=find_packages(), install_requires=['elementals', 'sark', 'pydocstyle', 'flake8', 'click', 'scikit-learn'], + python_requires='>=3', classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License (MIT License)", diff --git a/src/karta_analyze_src.py b/src/karta_analyze_src.py index 6ed3ca2..6fefed9 100755 --- a/src/karta_analyze_src.py +++ b/src/karta_analyze_src.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from ar_parser import getArchiveFiles from config.utils import * diff --git a/src/karta_manual_anchor.py b/src/karta_manual_anchor.py index 9916fb8..53baa96 100755 --- a/src/karta_manual_anchor.py +++ b/src/karta_manual_anchor.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from config.utils import * from elementals import Prompter diff --git a/src/karta_manual_identifier.py b/src/karta_manual_identifier.py index bbc6c0d..b897792 100644 --- a/src/karta_manual_identifier.py +++ b/src/karta_manual_identifier.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from config.utils import * from elementals import Prompter diff --git a/tests.py b/tests.py index eb09377..fc6054b 100644 --- a/tests.py +++ b/tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import pydocstyle import os From 0f4dd82b52240194d02026f7fc626c8c7e408d7c Mon Sep 17 00:00:00 2001 From: Harelon Date: Fri, 21 Jan 2022 19:19:54 +0200 Subject: [PATCH 02/11] added checking that the obj and lib files exist beforehand --- src/karta_analyze_src.py | 67 ++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/src/karta_analyze_src.py b/src/karta_analyze_src.py index 6fefed9..fcc8fa9 100755 --- a/src/karta_analyze_src.py +++ b/src/karta_analyze_src.py @@ -8,7 +8,7 @@ from function_context import SourceContext, BinaryContext, IslandContext import config.anchor as anchor -import os +from os import path, walk import sys import argparse import logging @@ -36,14 +36,14 @@ def locateFiles(bin_dir, file_list, suffix): Return Value: Generator for a tuples of the form: (abs_path, compiled_file file name) """ - for root, dirs, files in os.walk(bin_dir): + for root, dirs, files in walk(bin_dir): if file_list is not None: for compiled_file in set(files) & set(file_list): - yield os.path.abspath(os.path.join(root, compiled_file)), compiled_file + yield path.abspath(path.join(root, compiled_file)), compiled_file file_list.remove(compiled_file) else: for file in filter(lambda x: x.endswith("." + suffix), files): - yield os.path.abspath(os.path.join(root, file)), file + yield path.abspath(path.join(root, file)), file def analyzeFile(full_file_path, is_windows): """Analyze a single file using analyzer script. @@ -220,12 +220,12 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): file_dict = {} # find a common file prefix, and remove it form the file path if len(src_file_mappings) > 1: - base_value = list(src_file_mappings.keys())[0].split(os.path.sep) - comparison_value = list(src_file_mappings.keys())[-1].split(os.path.sep) + base_value = list(src_file_mappings.keys())[0].split(path.sep) + comparison_value = list(src_file_mappings.keys())[-1].split(path.sep) for index in range(min(len(comparison_value), len(base_value))): if base_value[index] != comparison_value[index]: break - common_path_len = len(os.path.sep.join(base_value[:index])) + 1 + common_path_len = len(path.sep.join(base_value[:index])) + 1 else: common_path_len = len(bin_dirs[0]) + 1 @@ -243,6 +243,41 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): prompter.info(f"Anchor to function ratio is: {len(anchors_list)}/{len(src_functions_list)}") prompter.removeIndent() +def verify_archives_and_objects(using_archives, couples, prompter): + bin_dirs = [] + archive_paths = [] + error_occured = False + if using_archives: + for i in range(0, len(couples), 2): + if not path.exists(couples[i]): + prompter.error(f"Error the path {couples[i]} does not exist!") + error_occured = True + elif not path.isdir(couples[i]): + prompter.error(f"Error the path {couples[i]} exists but is not a directory!") + error_occured = True + else: + bin_dirs.append(couples[i]) + + if not path.exists(couples[i + 1]): + prompter.error(f"Error the path {couples[i + 1]} does not exist!") + error_occured = True + elif not path.isfile(couples[i + 1]): + prompter.error(f"Error the path {couples[i + 1]} exists but is not a file!") + error_occured = True + else: + archive_paths.append(couples[i + 1]) + else: + for bin_dir in couples: + if not path.exists(bin_dir): + prompter.error(f"Error the path {bin_dir} does not exist!") + error_occured = True + elif not path.isdir(bin_dir): + prompter.error(f"Error the path {bin_dir} exists but is not a directory!") + error_occured = True + bin_dirs = couples + + return (bin_dirs, archive_paths) if not error_occured else None + def main(args): """Create a .json configuration for the open source library version. @@ -272,19 +307,19 @@ def main(args): using_archives = args.no_archive couples = args.couples - bin_dirs = [] - archive_paths = [] + # open the log + prompter = Prompter(min_log_level=logging.INFO if not is_debug else logging.DEBUG) if using_archives: if len(couples) % 2 != 0: parser.error("Odd length in list of dir,archive couples, should be: [(directory, archive name), ...]") - for i in range(0, len(couples), 2): - bin_dirs.append(couples[i]) - archive_paths.append(couples[i + 1]) - else: - bin_dirs = couples + + paths = verify_archives_and_objects(using_archives, couples, prompter) + + if paths is None: + return + + bin_dirs, archive_paths = paths - # open the log - prompter = Prompter(min_log_level=logging.INFO if not is_debug else logging.DEBUG) prompter.info("Starting the Script") # requesting the path to the chosen disassembler From d30b2be78cf0bdeeb879d4729977209674ffc923 Mon Sep 17 00:00:00 2001 From: Harelon Date: Tue, 25 Jan 2022 20:08:27 +0200 Subject: [PATCH 03/11] added karta folder inside src --- src/{ => karta}/__init__.py | 0 src/{ => karta}/analyze_src_file.py | 0 src/{ => karta}/ar_parser.py | 0 src/{ => karta}/config/__init__.py | 0 src/{ => karta}/config/anchor.py | 0 src/{ => karta}/config/anchor_config.py | 0 src/{ => karta}/config/libc_config.py | 0 src/{ => karta}/config/score_config.py | 0 src/{ => karta}/config/utils.py | 0 src/{ => karta}/core/__init__.py | 0 src/{ => karta}/core/file_layer.py | 0 src/{ => karta}/core/function_context.py | 0 src/{ => karta}/core/matching_engine.py | 0 src/{ => karta}/disassembler/IDA/__init__.py | 0 src/{ => karta}/disassembler/IDA/ida_analysis_api.py | 0 src/{ => karta}/disassembler/IDA/ida_api.py | 0 src/{ => karta}/disassembler/IDA/ida_cmd_api.py | 0 src/{ => karta}/disassembler/IDA/ida_verifier_api.py | 0 src/{ => karta}/disassembler/__init__.py | 0 src/{ => karta}/disassembler/disas_api.py | 0 src/{ => karta}/disassembler/factory.py | 0 src/{ => karta}/file_layer.py | 0 src/{ => karta}/function_context.py | 0 src/{ => karta}/karta_analyze_src.py | 0 src/{ => karta}/karta_identifier.py | 0 src/{ => karta}/karta_manual_anchor.py | 0 src/{ => karta}/karta_manual_identifier.py | 0 src/{ => karta}/karta_matcher.py | 0 src/{ => karta}/libs/__init__.py | 0 src/{ => karta}/libs/gsoap.py | 0 src/{ => karta}/libs/icu.py | 0 src/{ => karta}/libs/lib_factory.py | 0 src/{ => karta}/libs/lib_template.py | 0 src/{ => karta}/libs/libjpeg.py | 0 src/{ => karta}/libs/libjpeg_turbo.py | 0 src/{ => karta}/libs/libpng.py | 0 src/{ => karta}/libs/libtiff.py | 0 src/{ => karta}/libs/libvpx.py | 0 src/{ => karta}/libs/libxml2.py | 0 src/{ => karta}/libs/mactelnet.py | 0 src/{ => karta}/libs/mdnsresponder.py | 0 src/{ => karta}/libs/netsnmp.py | 0 src/{ => karta}/libs/openssh.py | 0 src/{ => karta}/libs/openssl.py | 0 src/{ => karta}/libs/treck.py | 0 src/{ => karta}/libs/zlib.py | 0 src/{ => karta}/matching_engine.py | 0 src/{ => karta}/thumbs_up/README.md | 0 src/{ => karta}/thumbs_up/__init__.py | 0 src/{ => karta}/thumbs_up/analyzer_utils.py | 0 src/{ => karta}/thumbs_up/analyzers/__init__.py | 0 src/{ => karta}/thumbs_up/analyzers/analyzer.py | 0 src/{ => karta}/thumbs_up/analyzers/analyzer_factory.py | 0 src/{ => karta}/thumbs_up/analyzers/arm.py | 0 src/{ => karta}/thumbs_up/analyzers/intel.py | 0 src/{ => karta}/thumbs_up/analyzers/mips.py | 0 src/{ => karta}/thumbs_up/thumbs_up_ELF.py | 0 src/{ => karta}/thumbs_up/thumbs_up_firmware.py | 0 src/{ => karta}/thumbs_up/utils/__init__.py | 0 src/{ => karta}/thumbs_up/utils/code_metric.py | 0 src/{ => karta}/thumbs_up/utils/code_regions.py | 0 src/{ => karta}/thumbs_up/utils/fptr.py | 0 src/{ => karta}/thumbs_up/utils/function.py | 0 src/{ => karta}/thumbs_up/utils/local_constants.py | 0 src/{ => karta}/thumbs_up/utils/pattern_observer.py | 0 src/{ => karta}/thumbs_up/utils/strings.py | 0 src/{ => karta}/thumbs_up/utils/switch_table.py | 0 67 files changed, 0 insertions(+), 0 deletions(-) rename src/{ => karta}/__init__.py (100%) rename src/{ => karta}/analyze_src_file.py (100%) rename src/{ => karta}/ar_parser.py (100%) rename src/{ => karta}/config/__init__.py (100%) rename src/{ => karta}/config/anchor.py (100%) rename src/{ => karta}/config/anchor_config.py (100%) rename src/{ => karta}/config/libc_config.py (100%) rename src/{ => karta}/config/score_config.py (100%) rename src/{ => karta}/config/utils.py (100%) rename src/{ => karta}/core/__init__.py (100%) rename src/{ => karta}/core/file_layer.py (100%) rename src/{ => karta}/core/function_context.py (100%) rename src/{ => karta}/core/matching_engine.py (100%) rename src/{ => karta}/disassembler/IDA/__init__.py (100%) rename src/{ => karta}/disassembler/IDA/ida_analysis_api.py (100%) rename src/{ => karta}/disassembler/IDA/ida_api.py (100%) rename src/{ => karta}/disassembler/IDA/ida_cmd_api.py (100%) rename src/{ => karta}/disassembler/IDA/ida_verifier_api.py (100%) rename src/{ => karta}/disassembler/__init__.py (100%) rename src/{ => karta}/disassembler/disas_api.py (100%) rename src/{ => karta}/disassembler/factory.py (100%) rename src/{ => karta}/file_layer.py (100%) rename src/{ => karta}/function_context.py (100%) rename src/{ => karta}/karta_analyze_src.py (100%) mode change 100755 => 100644 rename src/{ => karta}/karta_identifier.py (100%) rename src/{ => karta}/karta_manual_anchor.py (100%) mode change 100755 => 100644 rename src/{ => karta}/karta_manual_identifier.py (100%) rename src/{ => karta}/karta_matcher.py (100%) rename src/{ => karta}/libs/__init__.py (100%) rename src/{ => karta}/libs/gsoap.py (100%) rename src/{ => karta}/libs/icu.py (100%) rename src/{ => karta}/libs/lib_factory.py (100%) rename src/{ => karta}/libs/lib_template.py (100%) rename src/{ => karta}/libs/libjpeg.py (100%) rename src/{ => karta}/libs/libjpeg_turbo.py (100%) rename src/{ => karta}/libs/libpng.py (100%) rename src/{ => karta}/libs/libtiff.py (100%) rename src/{ => karta}/libs/libvpx.py (100%) rename src/{ => karta}/libs/libxml2.py (100%) rename src/{ => karta}/libs/mactelnet.py (100%) rename src/{ => karta}/libs/mdnsresponder.py (100%) rename src/{ => karta}/libs/netsnmp.py (100%) rename src/{ => karta}/libs/openssh.py (100%) rename src/{ => karta}/libs/openssl.py (100%) rename src/{ => karta}/libs/treck.py (100%) rename src/{ => karta}/libs/zlib.py (100%) rename src/{ => karta}/matching_engine.py (100%) rename src/{ => karta}/thumbs_up/README.md (100%) rename src/{ => karta}/thumbs_up/__init__.py (100%) rename src/{ => karta}/thumbs_up/analyzer_utils.py (100%) rename src/{ => karta}/thumbs_up/analyzers/__init__.py (100%) rename src/{ => karta}/thumbs_up/analyzers/analyzer.py (100%) rename src/{ => karta}/thumbs_up/analyzers/analyzer_factory.py (100%) rename src/{ => karta}/thumbs_up/analyzers/arm.py (100%) rename src/{ => karta}/thumbs_up/analyzers/intel.py (100%) rename src/{ => karta}/thumbs_up/analyzers/mips.py (100%) rename src/{ => karta}/thumbs_up/thumbs_up_ELF.py (100%) rename src/{ => karta}/thumbs_up/thumbs_up_firmware.py (100%) rename src/{ => karta}/thumbs_up/utils/__init__.py (100%) rename src/{ => karta}/thumbs_up/utils/code_metric.py (100%) rename src/{ => karta}/thumbs_up/utils/code_regions.py (100%) rename src/{ => karta}/thumbs_up/utils/fptr.py (100%) rename src/{ => karta}/thumbs_up/utils/function.py (100%) rename src/{ => karta}/thumbs_up/utils/local_constants.py (100%) rename src/{ => karta}/thumbs_up/utils/pattern_observer.py (100%) rename src/{ => karta}/thumbs_up/utils/strings.py (100%) rename src/{ => karta}/thumbs_up/utils/switch_table.py (100%) diff --git a/src/__init__.py b/src/karta/__init__.py similarity index 100% rename from src/__init__.py rename to src/karta/__init__.py diff --git a/src/analyze_src_file.py b/src/karta/analyze_src_file.py similarity index 100% rename from src/analyze_src_file.py rename to src/karta/analyze_src_file.py diff --git a/src/ar_parser.py b/src/karta/ar_parser.py similarity index 100% rename from src/ar_parser.py rename to src/karta/ar_parser.py diff --git a/src/config/__init__.py b/src/karta/config/__init__.py similarity index 100% rename from src/config/__init__.py rename to src/karta/config/__init__.py diff --git a/src/config/anchor.py b/src/karta/config/anchor.py similarity index 100% rename from src/config/anchor.py rename to src/karta/config/anchor.py diff --git a/src/config/anchor_config.py b/src/karta/config/anchor_config.py similarity index 100% rename from src/config/anchor_config.py rename to src/karta/config/anchor_config.py diff --git a/src/config/libc_config.py b/src/karta/config/libc_config.py similarity index 100% rename from src/config/libc_config.py rename to src/karta/config/libc_config.py diff --git a/src/config/score_config.py b/src/karta/config/score_config.py similarity index 100% rename from src/config/score_config.py rename to src/karta/config/score_config.py diff --git a/src/config/utils.py b/src/karta/config/utils.py similarity index 100% rename from src/config/utils.py rename to src/karta/config/utils.py diff --git a/src/core/__init__.py b/src/karta/core/__init__.py similarity index 100% rename from src/core/__init__.py rename to src/karta/core/__init__.py diff --git a/src/core/file_layer.py b/src/karta/core/file_layer.py similarity index 100% rename from src/core/file_layer.py rename to src/karta/core/file_layer.py diff --git a/src/core/function_context.py b/src/karta/core/function_context.py similarity index 100% rename from src/core/function_context.py rename to src/karta/core/function_context.py diff --git a/src/core/matching_engine.py b/src/karta/core/matching_engine.py similarity index 100% rename from src/core/matching_engine.py rename to src/karta/core/matching_engine.py diff --git a/src/disassembler/IDA/__init__.py b/src/karta/disassembler/IDA/__init__.py similarity index 100% rename from src/disassembler/IDA/__init__.py rename to src/karta/disassembler/IDA/__init__.py diff --git a/src/disassembler/IDA/ida_analysis_api.py b/src/karta/disassembler/IDA/ida_analysis_api.py similarity index 100% rename from src/disassembler/IDA/ida_analysis_api.py rename to src/karta/disassembler/IDA/ida_analysis_api.py diff --git a/src/disassembler/IDA/ida_api.py b/src/karta/disassembler/IDA/ida_api.py similarity index 100% rename from src/disassembler/IDA/ida_api.py rename to src/karta/disassembler/IDA/ida_api.py diff --git a/src/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py similarity index 100% rename from src/disassembler/IDA/ida_cmd_api.py rename to src/karta/disassembler/IDA/ida_cmd_api.py diff --git a/src/disassembler/IDA/ida_verifier_api.py b/src/karta/disassembler/IDA/ida_verifier_api.py similarity index 100% rename from src/disassembler/IDA/ida_verifier_api.py rename to src/karta/disassembler/IDA/ida_verifier_api.py diff --git a/src/disassembler/__init__.py b/src/karta/disassembler/__init__.py similarity index 100% rename from src/disassembler/__init__.py rename to src/karta/disassembler/__init__.py diff --git a/src/disassembler/disas_api.py b/src/karta/disassembler/disas_api.py similarity index 100% rename from src/disassembler/disas_api.py rename to src/karta/disassembler/disas_api.py diff --git a/src/disassembler/factory.py b/src/karta/disassembler/factory.py similarity index 100% rename from src/disassembler/factory.py rename to src/karta/disassembler/factory.py diff --git a/src/file_layer.py b/src/karta/file_layer.py similarity index 100% rename from src/file_layer.py rename to src/karta/file_layer.py diff --git a/src/function_context.py b/src/karta/function_context.py similarity index 100% rename from src/function_context.py rename to src/karta/function_context.py diff --git a/src/karta_analyze_src.py b/src/karta/karta_analyze_src.py old mode 100755 new mode 100644 similarity index 100% rename from src/karta_analyze_src.py rename to src/karta/karta_analyze_src.py diff --git a/src/karta_identifier.py b/src/karta/karta_identifier.py similarity index 100% rename from src/karta_identifier.py rename to src/karta/karta_identifier.py diff --git a/src/karta_manual_anchor.py b/src/karta/karta_manual_anchor.py old mode 100755 new mode 100644 similarity index 100% rename from src/karta_manual_anchor.py rename to src/karta/karta_manual_anchor.py diff --git a/src/karta_manual_identifier.py b/src/karta/karta_manual_identifier.py similarity index 100% rename from src/karta_manual_identifier.py rename to src/karta/karta_manual_identifier.py diff --git a/src/karta_matcher.py b/src/karta/karta_matcher.py similarity index 100% rename from src/karta_matcher.py rename to src/karta/karta_matcher.py diff --git a/src/libs/__init__.py b/src/karta/libs/__init__.py similarity index 100% rename from src/libs/__init__.py rename to src/karta/libs/__init__.py diff --git a/src/libs/gsoap.py b/src/karta/libs/gsoap.py similarity index 100% rename from src/libs/gsoap.py rename to src/karta/libs/gsoap.py diff --git a/src/libs/icu.py b/src/karta/libs/icu.py similarity index 100% rename from src/libs/icu.py rename to src/karta/libs/icu.py diff --git a/src/libs/lib_factory.py b/src/karta/libs/lib_factory.py similarity index 100% rename from src/libs/lib_factory.py rename to src/karta/libs/lib_factory.py diff --git a/src/libs/lib_template.py b/src/karta/libs/lib_template.py similarity index 100% rename from src/libs/lib_template.py rename to src/karta/libs/lib_template.py diff --git a/src/libs/libjpeg.py b/src/karta/libs/libjpeg.py similarity index 100% rename from src/libs/libjpeg.py rename to src/karta/libs/libjpeg.py diff --git a/src/libs/libjpeg_turbo.py b/src/karta/libs/libjpeg_turbo.py similarity index 100% rename from src/libs/libjpeg_turbo.py rename to src/karta/libs/libjpeg_turbo.py diff --git a/src/libs/libpng.py b/src/karta/libs/libpng.py similarity index 100% rename from src/libs/libpng.py rename to src/karta/libs/libpng.py diff --git a/src/libs/libtiff.py b/src/karta/libs/libtiff.py similarity index 100% rename from src/libs/libtiff.py rename to src/karta/libs/libtiff.py diff --git a/src/libs/libvpx.py b/src/karta/libs/libvpx.py similarity index 100% rename from src/libs/libvpx.py rename to src/karta/libs/libvpx.py diff --git a/src/libs/libxml2.py b/src/karta/libs/libxml2.py similarity index 100% rename from src/libs/libxml2.py rename to src/karta/libs/libxml2.py diff --git a/src/libs/mactelnet.py b/src/karta/libs/mactelnet.py similarity index 100% rename from src/libs/mactelnet.py rename to src/karta/libs/mactelnet.py diff --git a/src/libs/mdnsresponder.py b/src/karta/libs/mdnsresponder.py similarity index 100% rename from src/libs/mdnsresponder.py rename to src/karta/libs/mdnsresponder.py diff --git a/src/libs/netsnmp.py b/src/karta/libs/netsnmp.py similarity index 100% rename from src/libs/netsnmp.py rename to src/karta/libs/netsnmp.py diff --git a/src/libs/openssh.py b/src/karta/libs/openssh.py similarity index 100% rename from src/libs/openssh.py rename to src/karta/libs/openssh.py diff --git a/src/libs/openssl.py b/src/karta/libs/openssl.py similarity index 100% rename from src/libs/openssl.py rename to src/karta/libs/openssl.py diff --git a/src/libs/treck.py b/src/karta/libs/treck.py similarity index 100% rename from src/libs/treck.py rename to src/karta/libs/treck.py diff --git a/src/libs/zlib.py b/src/karta/libs/zlib.py similarity index 100% rename from src/libs/zlib.py rename to src/karta/libs/zlib.py diff --git a/src/matching_engine.py b/src/karta/matching_engine.py similarity index 100% rename from src/matching_engine.py rename to src/karta/matching_engine.py diff --git a/src/thumbs_up/README.md b/src/karta/thumbs_up/README.md similarity index 100% rename from src/thumbs_up/README.md rename to src/karta/thumbs_up/README.md diff --git a/src/thumbs_up/__init__.py b/src/karta/thumbs_up/__init__.py similarity index 100% rename from src/thumbs_up/__init__.py rename to src/karta/thumbs_up/__init__.py diff --git a/src/thumbs_up/analyzer_utils.py b/src/karta/thumbs_up/analyzer_utils.py similarity index 100% rename from src/thumbs_up/analyzer_utils.py rename to src/karta/thumbs_up/analyzer_utils.py diff --git a/src/thumbs_up/analyzers/__init__.py b/src/karta/thumbs_up/analyzers/__init__.py similarity index 100% rename from src/thumbs_up/analyzers/__init__.py rename to src/karta/thumbs_up/analyzers/__init__.py diff --git a/src/thumbs_up/analyzers/analyzer.py b/src/karta/thumbs_up/analyzers/analyzer.py similarity index 100% rename from src/thumbs_up/analyzers/analyzer.py rename to src/karta/thumbs_up/analyzers/analyzer.py diff --git a/src/thumbs_up/analyzers/analyzer_factory.py b/src/karta/thumbs_up/analyzers/analyzer_factory.py similarity index 100% rename from src/thumbs_up/analyzers/analyzer_factory.py rename to src/karta/thumbs_up/analyzers/analyzer_factory.py diff --git a/src/thumbs_up/analyzers/arm.py b/src/karta/thumbs_up/analyzers/arm.py similarity index 100% rename from src/thumbs_up/analyzers/arm.py rename to src/karta/thumbs_up/analyzers/arm.py diff --git a/src/thumbs_up/analyzers/intel.py b/src/karta/thumbs_up/analyzers/intel.py similarity index 100% rename from src/thumbs_up/analyzers/intel.py rename to src/karta/thumbs_up/analyzers/intel.py diff --git a/src/thumbs_up/analyzers/mips.py b/src/karta/thumbs_up/analyzers/mips.py similarity index 100% rename from src/thumbs_up/analyzers/mips.py rename to src/karta/thumbs_up/analyzers/mips.py diff --git a/src/thumbs_up/thumbs_up_ELF.py b/src/karta/thumbs_up/thumbs_up_ELF.py similarity index 100% rename from src/thumbs_up/thumbs_up_ELF.py rename to src/karta/thumbs_up/thumbs_up_ELF.py diff --git a/src/thumbs_up/thumbs_up_firmware.py b/src/karta/thumbs_up/thumbs_up_firmware.py similarity index 100% rename from src/thumbs_up/thumbs_up_firmware.py rename to src/karta/thumbs_up/thumbs_up_firmware.py diff --git a/src/thumbs_up/utils/__init__.py b/src/karta/thumbs_up/utils/__init__.py similarity index 100% rename from src/thumbs_up/utils/__init__.py rename to src/karta/thumbs_up/utils/__init__.py diff --git a/src/thumbs_up/utils/code_metric.py b/src/karta/thumbs_up/utils/code_metric.py similarity index 100% rename from src/thumbs_up/utils/code_metric.py rename to src/karta/thumbs_up/utils/code_metric.py diff --git a/src/thumbs_up/utils/code_regions.py b/src/karta/thumbs_up/utils/code_regions.py similarity index 100% rename from src/thumbs_up/utils/code_regions.py rename to src/karta/thumbs_up/utils/code_regions.py diff --git a/src/thumbs_up/utils/fptr.py b/src/karta/thumbs_up/utils/fptr.py similarity index 100% rename from src/thumbs_up/utils/fptr.py rename to src/karta/thumbs_up/utils/fptr.py diff --git a/src/thumbs_up/utils/function.py b/src/karta/thumbs_up/utils/function.py similarity index 100% rename from src/thumbs_up/utils/function.py rename to src/karta/thumbs_up/utils/function.py diff --git a/src/thumbs_up/utils/local_constants.py b/src/karta/thumbs_up/utils/local_constants.py similarity index 100% rename from src/thumbs_up/utils/local_constants.py rename to src/karta/thumbs_up/utils/local_constants.py diff --git a/src/thumbs_up/utils/pattern_observer.py b/src/karta/thumbs_up/utils/pattern_observer.py similarity index 100% rename from src/thumbs_up/utils/pattern_observer.py rename to src/karta/thumbs_up/utils/pattern_observer.py diff --git a/src/thumbs_up/utils/strings.py b/src/karta/thumbs_up/utils/strings.py similarity index 100% rename from src/thumbs_up/utils/strings.py rename to src/karta/thumbs_up/utils/strings.py diff --git a/src/thumbs_up/utils/switch_table.py b/src/karta/thumbs_up/utils/switch_table.py similarity index 100% rename from src/thumbs_up/utils/switch_table.py rename to src/karta/thumbs_up/utils/switch_table.py From 4984faff8d22786cc2cce011b19de0bb0cf41024 Mon Sep 17 00:00:00 2001 From: Harelon Date: Tue, 25 Jan 2022 21:06:06 +0200 Subject: [PATCH 04/11] changed all imports to relative and changed src to karta in src folder --- setup.py | 6 ++++-- src/karta/__init__.py | 13 ++++++------- src/karta/analyze_src_file.py | 9 +++++---- src/karta/config/anchor.py | 2 +- src/karta/config/utils.py | 3 ++- src/karta/core/matching_engine.py | 8 +++++--- .../disassembler/IDA/ida_analysis_api.py | 3 ++- src/karta/disassembler/IDA/ida_api.py | 8 +++++--- src/karta/disassembler/IDA/ida_cmd_api.py | 5 +++-- .../disassembler/IDA/ida_verifier_api.py | 4 ++-- src/karta/file_layer.py | 6 +++--- src/karta/function_context.py | 7 ++++--- src/karta/karta_analyze_src.py | 19 ++++++++++--------- src/karta/karta_identifier.py | 10 +++++----- src/karta/karta_manual_anchor.py | 8 ++++---- src/karta/karta_manual_identifier.py | 9 +++++---- src/karta/karta_matcher.py | 12 ++++++------ src/karta/libs/lib_template.py | 3 ++- src/karta/libs/libjpeg.py | 3 ++- src/karta/libs/libpng.py | 2 +- src/karta/libs/openssl.py | 3 ++- src/karta/libs/zlib.py | 3 ++- src/karta/matching_engine.py | 16 +++++++++------- src/karta/thumbs_up/__init__.py | 4 ++-- src/karta/thumbs_up/analyzer_utils.py | 5 +++-- src/karta/thumbs_up/analyzers/arm.py | 11 ++++++----- src/karta/thumbs_up/analyzers/intel.py | 10 +++++----- src/karta/thumbs_up/analyzers/mips.py | 11 ++++++----- src/karta/thumbs_up/thumbs_up_ELF.py | 5 +++-- src/karta/thumbs_up/thumbs_up_firmware.py | 5 +++-- src/karta/thumbs_up/utils/fptr.py | 5 +++-- src/karta/thumbs_up/utils/function.py | 6 +++--- src/karta/thumbs_up/utils/local_constants.py | 3 ++- src/karta/thumbs_up/utils/strings.py | 3 ++- src/karta/thumbs_up/utils/switch_table.py | 1 + 35 files changed, 129 insertions(+), 102 deletions(-) diff --git a/setup.py b/setup.py index a7950f2..171378c 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,8 @@ long_description_content_type="text/markdown", url='https://github.com/CheckPointSW/Karta', license='MIT', - packages=find_packages(), + packages=find_packages(where="src"), + package_dir={"": "src"}, install_requires=['elementals', 'sark', 'pydocstyle', 'flake8', 'click', 'scikit-learn'], python_requires='>=3', classifiers=[ @@ -22,4 +23,5 @@ "License :: OSI Approved :: MIT License (MIT License)", "Operating System :: OS Independent", ], - zip_safe=False) + zip_safe=False +) diff --git a/src/karta/__init__.py b/src/karta/__init__.py index eeee0da..176b8ed 100644 --- a/src/karta/__init__.py +++ b/src/karta/__init__.py @@ -1,7 +1,6 @@ -import karta_analyze_src -import karta_identifier -import karta_matcher -import karta_manual_anchor -import karta_manual_identifier -import thumbs_up.thumbs_up_ELF -import thumbs_up.thumbs_up_firmware +from . import karta_analyze_src +from . import karta_identifier +from . import karta_matcher +from . import karta_manual_anchor +from . import karta_manual_identifier +from . import thumbs_up diff --git a/src/karta/analyze_src_file.py b/src/karta/analyze_src_file.py index ecf265b..7c4ba27 100644 --- a/src/karta/analyze_src_file.py +++ b/src/karta/analyze_src_file.py @@ -1,9 +1,10 @@ -from config.utils import * -from disassembler.factory import createDisassemblerHandler -from function_context import SourceContext, BinaryContext, IslandContext -from elementals import Logger import logging import traceback +from elementals import Logger + +from .config.utils import * +from .disassembler.factory import createDisassemblerHandler +from .function_context import SourceContext, BinaryContext, IslandContext def analyzeFile(): """Analyzes all of the (source) functions for a single compiled file.""" diff --git a/src/karta/config/anchor.py b/src/karta/config/anchor.py index 627e162..d032289 100644 --- a/src/karta/config/anchor.py +++ b/src/karta/config/anchor.py @@ -1,5 +1,5 @@ from .anchor_config import * -from config.utils import * +from .utils import * def isAnchor(context, seen_strings, seen_consts, functions_list, logger): """Check if the given context represents an Anchor function. diff --git a/src/karta/config/utils.py b/src/karta/config/utils.py index dde5c89..048d95c 100644 --- a/src/karta/config/utils.py +++ b/src/karta/config/utils.py @@ -1,7 +1,8 @@ -from .score_config import * import json import os +from .score_config import * + ################################# ## Basic Global Configurations ## ################################# diff --git a/src/karta/core/matching_engine.py b/src/karta/core/matching_engine.py index d7c7fe4..1af302e 100644 --- a/src/karta/core/matching_engine.py +++ b/src/karta/core/matching_engine.py @@ -1,7 +1,9 @@ -from config.utils import * -from collections import defaultdict -import config.anchor as anchor import time +from collections import defaultdict + +from ..config.utils import * +from ..config import anchor + class MatchEngine(object): """A class that handles the book-keeping for the matching process. diff --git a/src/karta/disassembler/IDA/ida_analysis_api.py b/src/karta/disassembler/IDA/ida_analysis_api.py index b8ecefc..4d91718 100644 --- a/src/karta/disassembler/IDA/ida_analysis_api.py +++ b/src/karta/disassembler/IDA/ida_analysis_api.py @@ -1,8 +1,9 @@ import idaapi import sark -from config.utils import * from hashlib import md5 +from ...config.utils import * + class AnalyzerIDA(object): """Logic instance for the IDA disassembler API. Contains the heart of Karta's canonical representation. diff --git a/src/karta/disassembler/IDA/ida_api.py b/src/karta/disassembler/IDA/ida_api.py index 581acc5..89f417d 100644 --- a/src/karta/disassembler/IDA/ida_api.py +++ b/src/karta/disassembler/IDA/ida_api.py @@ -1,3 +1,5 @@ +import logging + # Dependencies that only exist inside IDA import idautils import idaapi @@ -9,9 +11,9 @@ import sark from .ida_analysis_api import AnalyzerIDA # Basic dependencies (only basic python packages) -from config.utils import * -from disassembler.disas_api import DisasAPI -import logging +from ...config.utils import * +from ..disas_api import DisasAPI + class IdaLogHandler(logging.Handler): """Integrate the log messages with IDA's output window.""" diff --git a/src/karta/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py index e2083c5..14d5797 100644 --- a/src/karta/disassembler/IDA/ida_cmd_api.py +++ b/src/karta/disassembler/IDA/ida_cmd_api.py @@ -1,7 +1,8 @@ -from disassembler.disas_api import DisasCMD -from disassembler.factory import registerDisassemblerCMD import os +from ..disas_api import DisasCMD +from ..factory import registerDisassemblerCMD + class IdaCMD(DisasCMD): """DisasCMD implementation for the IDA disassembler.""" diff --git a/src/karta/disassembler/IDA/ida_verifier_api.py b/src/karta/disassembler/IDA/ida_verifier_api.py index a844b95..2dedf4e 100644 --- a/src/karta/disassembler/IDA/ida_verifier_api.py +++ b/src/karta/disassembler/IDA/ida_verifier_api.py @@ -1,5 +1,5 @@ -from disassembler.disas_api import DisasVerifier -from disassembler.factory import registerDisassembler +from ..disas_api import DisasVerifier +from ..factory import registerDisassembler from .ida_cmd_api import IdaCMD class IdaVerifier(DisasVerifier): diff --git a/src/karta/file_layer.py b/src/karta/file_layer.py index de29965..9df1d86 100644 --- a/src/karta/file_layer.py +++ b/src/karta/file_layer.py @@ -1,6 +1,6 @@ -from core.file_layer import * -from config.utils import * -import config.anchor as anchor +from .core.file_layer import * +from .config.utils import * +from .config import anchor class FileMatcher(FileMatch): """A wrapper for Matched Files with an additional matching logic layer. diff --git a/src/karta/function_context.py b/src/karta/function_context.py index 7aa31aa..ac66f6c 100644 --- a/src/karta/function_context.py +++ b/src/karta/function_context.py @@ -1,7 +1,8 @@ -from core.function_context import * -from config.utils import * from collections import defaultdict -import config.libc_config as libc + +from .core.function_context import * +from .config.utils import * +from .config import libc_config as libc class ExternalFunction(CodeContext): """A context that describes a source-code point of view of an external function, using references from/to it. diff --git a/src/karta/karta_analyze_src.py b/src/karta/karta_analyze_src.py index fcc8fa9..920d86b 100644 --- a/src/karta/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -1,17 +1,18 @@ #!/usr/bin/python3 -from ar_parser import getArchiveFiles -from config.utils import * -from elementals import Prompter, ProgressBar -from disassembler.factory import identifyDisassemblerHandler -from disassembler.IDA import ida_cmd_api -from function_context import SourceContext, BinaryContext, IslandContext -import config.anchor as anchor - -from os import path, walk import sys import argparse import logging +from os import path, walk +from elementals import Prompter, ProgressBar + +from .ar_parser import getArchiveFiles +from .config.utils import * +from .disassembler.factory import identifyDisassemblerHandler +from .disassembler.IDA import ida_cmd_api +from .function_context import SourceContext, BinaryContext, IslandContext +from .config import anchor + #################### ## Global Configs ## diff --git a/src/karta/karta_identifier.py b/src/karta/karta_identifier.py index 38bf731..74e0b70 100644 --- a/src/karta/karta_identifier.py +++ b/src/karta/karta_identifier.py @@ -1,9 +1,9 @@ -from config.utils import * -from disassembler.factory import createDisassemblerHandler -from libs import lib_factory - -from elementals import Logger import logging +from elementals import Logger + +from .config.utils import * +from .disassembler.factory import createDisassemblerHandler +from .libs import lib_factory #################### ## Global Configs ## diff --git a/src/karta/karta_manual_anchor.py b/src/karta/karta_manual_anchor.py index 53baa96..d6a06ab 100644 --- a/src/karta/karta_manual_anchor.py +++ b/src/karta/karta_manual_anchor.py @@ -1,14 +1,14 @@ #!/usr/bin/python3 -from config.utils import * -from elementals import Prompter -from function_context import SourceContext, BinaryContext, IslandContext - import os import sys import argparse import logging from collections import defaultdict +from elementals import Prompter + +from .config.utils import * +from .function_context import SourceContext, BinaryContext, IslandContext def recordManualAnchors(library_config, knowledge_config, lib_name, prompter): """Record the list of user defined manual anchor matches. diff --git a/src/karta/karta_manual_identifier.py b/src/karta/karta_manual_identifier.py index b897792..eba4021 100644 --- a/src/karta/karta_manual_identifier.py +++ b/src/karta/karta_manual_identifier.py @@ -1,13 +1,14 @@ #!/usr/bin/python3 -from config.utils import * -from elementals import Prompter - import sys import argparse import logging from collections import defaultdict -from libs import lib_factory +from elementals import Prompter + +from .config.utils import * +from .libs import lib_factory + def recordManualVersions(knowledge_config, prompter): """Record the list of user defined manual library versions. diff --git a/src/karta/karta_matcher.py b/src/karta/karta_matcher.py index 430cbe7..dc360e5 100644 --- a/src/karta/karta_matcher.py +++ b/src/karta/karta_matcher.py @@ -1,10 +1,10 @@ -from config.utils import * -from disassembler.factory import createDisassemblerHandler -from matching_engine import KartaMatcher -from libs import lib_factory - -from elementals import Logger import logging +from elementals import Logger + +from .config.utils import * +from .disassembler.factory import createDisassemblerHandler +from .matching_engine import KartaMatcher +from .libs import lib_factory ###################### ## Global Variables ## diff --git a/src/karta/libs/lib_template.py b/src/karta/libs/lib_template.py index f6ba387..626558d 100644 --- a/src/karta/libs/lib_template.py +++ b/src/karta/libs/lib_template.py @@ -1,6 +1,7 @@ -from .lib_factory import registerLibrary import string +from .lib_factory import registerLibrary + class Seeker(object): """Abstract class that represents a basic library seeker. diff --git a/src/karta/libs/libjpeg.py b/src/karta/libs/libjpeg.py index b7f2167..afd4c66 100644 --- a/src/karta/libs/libjpeg.py +++ b/src/karta/libs/libjpeg.py @@ -1,6 +1,7 @@ -from .lib_template import * import string +from .lib_template import * + class LibJPEGSeeker(Seeker): """Seeker (Identifier) for the libjpeg (ITU) open source library.""" diff --git a/src/karta/libs/libpng.py b/src/karta/libs/libpng.py index a91f232..9a0518b 100644 --- a/src/karta/libs/libpng.py +++ b/src/karta/libs/libpng.py @@ -1,5 +1,5 @@ from .lib_template import * -from config.utils import getDisas +from ..config.utils import getDisas class LibpngSeeker(Seeker): """Seeker (Identifier) for the libpng open source library.""" diff --git a/src/karta/libs/openssl.py b/src/karta/libs/openssl.py index abf76cf..82179d9 100644 --- a/src/karta/libs/openssl.py +++ b/src/karta/libs/openssl.py @@ -1,6 +1,7 @@ -from .lib_template import * import string +from .lib_template import * + class OpenSSLSeeker(Seeker): """Seeker (Identifier) for the OpenSSL open source library.""" diff --git a/src/karta/libs/zlib.py b/src/karta/libs/zlib.py index 7f255f0..20ba91e 100644 --- a/src/karta/libs/zlib.py +++ b/src/karta/libs/zlib.py @@ -1,6 +1,7 @@ -from .lib_template import * from collections import defaultdict +from .lib_template import * + class ZlibSeeker(Seeker): """Seeker (Identifier) for the zlib open source library.""" diff --git a/src/karta/matching_engine.py b/src/karta/matching_engine.py index ef59340..c83476c 100644 --- a/src/karta/matching_engine.py +++ b/src/karta/matching_engine.py @@ -1,11 +1,13 @@ -from core.matching_engine import MatchEngine -from core.file_layer import AssumptionException -from config.utils import * -from file_layer import FileMatcher -from function_context import ExternalFunction, SourceContext, BinaryContext, IslandContext -from collections import defaultdict -import config.libc_config as libc import sys +from collections import defaultdict + +from .core.matching_engine import MatchEngine +from .core.file_layer import AssumptionException +from .config.utils import * +from .file_layer import FileMatcher +from .function_context import ExternalFunction, SourceContext, BinaryContext, IslandContext +from .config import libc_config as libc + class KartaMatcher(MatchEngine): """Complete matching engine logic for Karta, based on the file matching logic of the base MatchEngine. diff --git a/src/karta/thumbs_up/__init__.py b/src/karta/thumbs_up/__init__.py index 3015824..f5c8111 100644 --- a/src/karta/thumbs_up/__init__.py +++ b/src/karta/thumbs_up/__init__.py @@ -1,2 +1,2 @@ -import thumbs_up_ELF -import thumbs_up_firmware +from . import thumbs_up_ELF +from . import thumbs_up_firmware diff --git a/src/karta/thumbs_up/analyzer_utils.py b/src/karta/thumbs_up/analyzer_utils.py index c0f1ed6..e94707a 100644 --- a/src/karta/thumbs_up/analyzer_utils.py +++ b/src/karta/thumbs_up/analyzer_utils.py @@ -4,8 +4,9 @@ import ida_funcs import idaapi from collections import defaultdict -from utils.code_metric import CodeMetric -from utils.code_regions import CodeRegion, CodeRegions + +from .utils.code_metric import CodeMetric +from .utils.code_regions import CodeRegion, CodeRegions def gatherIntel(analyzer, scs, sds): """Gather all of the intelligence about the program's different features. diff --git a/src/karta/thumbs_up/analyzers/arm.py b/src/karta/thumbs_up/analyzers/arm.py index 531dcf6..3583489 100644 --- a/src/karta/thumbs_up/analyzers/arm.py +++ b/src/karta/thumbs_up/analyzers/arm.py @@ -1,11 +1,12 @@ import idc + from .analyzer import Analyzer from .analyzer_factory import registerAnalyzer -from utils.function import FunctionClassifier -from utils.strings import StringIdentifier -from utils.local_constants import LocalsIdentifier -from utils.fptr import FptrIdentifier -from utils.switch_table import SwitchIdentifier +from ..utils.function import FunctionClassifier +from ..utils.strings import StringIdentifier +from ..utils.local_constants import LocalsIdentifier +from ..utils.fptr import FptrIdentifier +from ..utils.switch_table import SwitchIdentifier ############################### # Architecture Configurations # diff --git a/src/karta/thumbs_up/analyzers/intel.py b/src/karta/thumbs_up/analyzers/intel.py index de2ec5e..57188e7 100644 --- a/src/karta/thumbs_up/analyzers/intel.py +++ b/src/karta/thumbs_up/analyzers/intel.py @@ -1,10 +1,10 @@ from .analyzer import Analyzer from .analyzer_factory import registerAnalyzer -from utils.function import FunctionClassifier -from utils.strings import StringIdentifier -from utils.local_constants import LocalsIdentifier -from utils.fptr import FptrIdentifier -from utils.switch_table import SwitchIdentifier +from ..utils.function import FunctionClassifier +from ..utils.strings import StringIdentifier +from ..utils.local_constants import LocalsIdentifier +from ..utils.fptr import FptrIdentifier +from ..utils.switch_table import SwitchIdentifier ############################### # Architecture Configurations # diff --git a/src/karta/thumbs_up/analyzers/mips.py b/src/karta/thumbs_up/analyzers/mips.py index 3333986..14b6e65 100644 --- a/src/karta/thumbs_up/analyzers/mips.py +++ b/src/karta/thumbs_up/analyzers/mips.py @@ -1,11 +1,12 @@ import idc + from .analyzer import Analyzer from .analyzer_factory import registerAnalyzer -from utils.function import FunctionClassifier -from utils.strings import StringIdentifier -from utils.local_constants import LocalsIdentifier -from utils.fptr import FptrIdentifier -from utils.switch_table import SwitchIdentifier +from ..utils.function import FunctionClassifier +from ..utils.strings import StringIdentifier +from ..utils.local_constants import LocalsIdentifier +from ..utils.fptr import FptrIdentifier +from ..utils.switch_table import SwitchIdentifier ############################### # Architecture Configurations # diff --git a/src/karta/thumbs_up/thumbs_up_ELF.py b/src/karta/thumbs_up/thumbs_up_ELF.py index 9db8f61..f934367 100644 --- a/src/karta/thumbs_up/thumbs_up_ELF.py +++ b/src/karta/thumbs_up/thumbs_up_ELF.py @@ -2,8 +2,9 @@ import idc import logging from elementals import Logger -from analyzer_utils import * -from analyzers.analyzer_factory import createAnalyzer + +from .analyzer_utils import * +from .analyzers.analyzer_factory import createAnalyzer ## # Taken from Karta :) diff --git a/src/karta/thumbs_up/thumbs_up_firmware.py b/src/karta/thumbs_up/thumbs_up_firmware.py index 7981488..a04098f 100644 --- a/src/karta/thumbs_up/thumbs_up_firmware.py +++ b/src/karta/thumbs_up/thumbs_up_firmware.py @@ -2,8 +2,9 @@ import idc import logging from elementals import Logger -from analyzer_utils import * -from analyzers.analyzer_factory import createAnalyzer + +from .analyzer_utils import * +from .analyzers.analyzer_factory import createAnalyzer ## # Taken from Karta :) diff --git a/src/karta/thumbs_up/utils/fptr.py b/src/karta/thumbs_up/utils/fptr.py index 2d190a6..4163f39 100644 --- a/src/karta/thumbs_up/utils/fptr.py +++ b/src/karta/thumbs_up/utils/fptr.py @@ -1,5 +1,3 @@ -from collections import defaultdict -from .pattern_observer import pad import pickle import struct import string @@ -7,6 +5,9 @@ import ida_bytes import ida_funcs import sark +from collections import defaultdict + +from .pattern_observer import pad ########################### ## Static Magic Constant ## diff --git a/src/karta/thumbs_up/utils/function.py b/src/karta/thumbs_up/utils/function.py index e46b8cd..ca24271 100644 --- a/src/karta/thumbs_up/utils/function.py +++ b/src/karta/thumbs_up/utils/function.py @@ -1,12 +1,12 @@ -from sklearn import metrics -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split import idc import ida_nalt import sark import numpy import struct import time +from sklearn import metrics +from sklearn.ensemble import RandomForestClassifier +from sklearn.model_selection import train_test_split ####################### ## Static Thresholds ## diff --git a/src/karta/thumbs_up/utils/local_constants.py b/src/karta/thumbs_up/utils/local_constants.py index 4b1e0d3..e6058c7 100644 --- a/src/karta/thumbs_up/utils/local_constants.py +++ b/src/karta/thumbs_up/utils/local_constants.py @@ -1,8 +1,9 @@ -from .pattern_observer import pad import idc import ida_bytes import sark +from .pattern_observer import pad + class LocalsIdentifier: """A class that collects the information and holds the knowledge we know about local (in-code) constants in the program. diff --git a/src/karta/thumbs_up/utils/strings.py b/src/karta/thumbs_up/utils/strings.py index 718a313..ca8c4fe 100644 --- a/src/karta/thumbs_up/utils/strings.py +++ b/src/karta/thumbs_up/utils/strings.py @@ -1,10 +1,11 @@ -from .pattern_observer import AlignmentPattern, pad, padSize import idc import ida_bytes import idautils import string import sark +from .pattern_observer import AlignmentPattern, pad, padSize + ####################### ## String Heuristics ## ####################### diff --git a/src/karta/thumbs_up/utils/switch_table.py b/src/karta/thumbs_up/utils/switch_table.py index 7764b96..a192f56 100644 --- a/src/karta/thumbs_up/utils/switch_table.py +++ b/src/karta/thumbs_up/utils/switch_table.py @@ -3,6 +3,7 @@ import ida_bytes import idaapi import sark + from .pattern_observer import AlignmentPattern, CodePattern, pad ########################### From 2fd54b610858404ded2b4dc7833b93feee0c27be Mon Sep 17 00:00:00 2001 From: Harelon Date: Mon, 31 Jan 2022 18:09:07 +0200 Subject: [PATCH 05/11] Add use of default disassembler Add installer for ida Add plugin file for ida Add automation with default disassembler --- .gitignore | 6 +-- README.md | 2 + docs/Installation.md | 4 +- src/karta/__init__.py | 6 --- src/karta/analyze_src_file.py | 7 +-- src/karta/config/utils.py | 45 +++++++++++----- src/karta/disassembler/IDA/ida_cmd_api.py | 39 ++++++++++++-- src/karta/installers/__init__.py | 0 src/karta/installers/common_installer.py | 63 +++++++++++++++++++++++ src/karta/installers/ida_installer.py | 59 +++++++++++++++++++++ src/karta/karta_analyze_src.py | 9 +--- src/karta/plugins/__init__.py | 0 src/karta/plugins/ida_plugin.py | 1 + 13 files changed, 204 insertions(+), 37 deletions(-) create mode 100644 src/karta/installers/__init__.py create mode 100644 src/karta/installers/common_installer.py create mode 100644 src/karta/installers/ida_installer.py create mode 100644 src/karta/plugins/__init__.py create mode 100644 src/karta/plugins/ida_plugin.py diff --git a/.gitignore b/.gitignore index 73aa6a8..3199320 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ *.pyc __pycache__ -/build/* -/dist/* -/Karta.egg-info/* +/build +/dist +Karta.egg-info docs/_build/* docs/_static/* diff --git a/README.md b/README.md index 36dd2ef..10b5949 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ https://karta.readthedocs.io/ ## Installation (Python 3 & IDA >= 7.4) For the latest versions, using Python 3, simply git clone the repository and run ```pip3 install .```. +To install the plugin for use in ida run +```python -m karta.installers.ida_installer``` Python 3 is supported since versions v2.0.0 and above. ## Installation (Python 2 & IDA < 7.4) diff --git a/docs/Installation.md b/docs/Installation.md index e4057b7..68ac5ef 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -8,8 +8,10 @@ Prerequisites Using the ```setup.py``` script, one can install all of these prerequisites, and be ready to go: ```pip3 install .``` +To install the plugin for use in ida run +```python -m karta.installers.ida_installer``` Installing the Plugin ------------------------ -Nothing should be done :)aa +Nothing should be done :) There is no need to copy the directory to some IDA plugin folder. Instead, once your binary is loaded to IDA, simply start the desired plugin by loading it using the ```File->Script File...``` menu. That's it. \ No newline at end of file diff --git a/src/karta/__init__.py b/src/karta/__init__.py index 176b8ed..e69de29 100644 --- a/src/karta/__init__.py +++ b/src/karta/__init__.py @@ -1,6 +0,0 @@ -from . import karta_analyze_src -from . import karta_identifier -from . import karta_matcher -from . import karta_manual_anchor -from . import karta_manual_identifier -from . import thumbs_up diff --git a/src/karta/analyze_src_file.py b/src/karta/analyze_src_file.py index 7c4ba27..7443753 100644 --- a/src/karta/analyze_src_file.py +++ b/src/karta/analyze_src_file.py @@ -2,9 +2,10 @@ import traceback from elementals import Logger -from .config.utils import * -from .disassembler.factory import createDisassemblerHandler -from .function_context import SourceContext, BinaryContext, IslandContext +# cannot have realtive imports for a script running in ida +from karta.config.utils import * +from karta.disassembler.factory import createDisassemblerHandler +from karta.function_context import SourceContext, BinaryContext, IslandContext def analyzeFile(): """Analyzes all of the (source) functions for a single compiled file.""" diff --git a/src/karta/config/utils.py b/src/karta/config/utils.py index 048d95c..f6bf3cc 100644 --- a/src/karta/config/utils.py +++ b/src/karta/config/utils.py @@ -7,8 +7,10 @@ ## Basic Global Configurations ## ################################# -DISASSEMBLER_PATH = "/opt/ida-7.4/ida" -SCRIPT_PATH = os.path.abspath("analyze_src_file.py") +DISASSEMBLER_PATH = None +CONFIG_DIR = os.path.dirname(os.path.realpath(__file__)) +DEFAULT_DISASSEMBLER = os.path.join(CONFIG_DIR, "default_disassembler_path") +SCRIPT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","analyze_src_file.py") LIBRARY_NAME = "Karta" STATE_FILE_SUFFIX = "_file_state.json" @@ -456,24 +458,43 @@ def isMatching(): """ return matching_mode -def setDisassemblerPath(prompter): - """Update the disassembler path according to input from the user. +def addDisassembler(name, path): + with open(os.path.join(CONFIG_DIR, name), "w") as f: + f.write(path) - Args: - prompter (prompter): prompter elementals instance - """ - global DISASSEMBLER_PATH +def disassemblerInstallationExists(name): + return os.path.exists(os.path.join(CONFIG_DIR, name)) - new_path = prompter.input(f"Please insert the command (path) needed in order to execute your disassembler (IDA for instance) ({DISASSEMBLER_PATH}): ") - if len(new_path.strip()) != 0: - DISASSEMBLER_PATH = new_path +def getDisassembler(name): + if disassemblerInstallationExists(name): + with open(os.path.join(CONFIG_DIR, name), "r") as f: + return f.read() -def getDisasPath(): +def setDefaultDisassembler(name): + with open(os.path.join(CONFIG_DIR, DEFAULT_DISASSEMBLER), "w") as f: + if os.path.isfile(os.path.join(CONFIG_DIR, name)): + f.write(name) + +def getDisasPath(prompter): """Return the updated path to the disassembler. Return Value: The (updated) path to the disassembler program """ + global DISASSEMBLER_PATH + if DISASSEMBLER_PATH is not None: + pass + elif os.path.isfile(DEFAULT_DISASSEMBLER): + actual_disas_path = None + with open(DEFAULT_DISASSEMBLER, 'r') as f: + actual_disas_path = f.read() + full_disas_name_path = os.path.join(CONFIG_DIR ,actual_disas_path) + if os.path.isfile(full_disas_name_path): + with open(full_disas_name_path, 'r') as f: + DISASSEMBLER_PATH = f.read() + else: + DISASSEMBLER_PATH = prompter.input("Please enter a path to your disassembler: ") + prompter.info("I may suggest using one of the installers to set a default disassembler") return DISASSEMBLER_PATH def libraryName(): diff --git a/src/karta/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py index 14d5797..e68e08c 100644 --- a/src/karta/disassembler/IDA/ida_cmd_api.py +++ b/src/karta/disassembler/IDA/ida_cmd_api.py @@ -1,8 +1,16 @@ import os +import subprocess from ..disas_api import DisasCMD from ..factory import registerDisassemblerCMD +# machine type header of pe +# specified in that order, amd64, arm64, ia64, loongarch64, riscv64 +ARCH64PE = [b"\x64\x86", b"\x64\xaa", b"\x00\x02", b"\x64\x62", b"\x64\x50"] + +# machine type header of elf +ARCH64ELF64 = b"\x02" + class IdaCMD(DisasCMD): """DisasCMD implementation for the IDA disassembler.""" @@ -41,10 +49,33 @@ def createDatabase(self, binary_file, is_windows): path to the created database file """ type = "elf" if not is_windows else "coff" - suffix = ".i64" if self._path.endswith("64") else ".idb" - database_file = binary_file + suffix + # because we get the path to the folder of the dissasembler + # we need to define whether it is a 64 or 32 bit one + # so we check the respective fields in coff files and elf files + # and define the file ending and the ida to use based on tehm + if not hasattr(self, "is64"): + self._path = os.path.join(self._path, "ida") + self.is64 = False + with open(binary_file, 'rb') as f: + if is_windows: + # read machine type header and check whether it is in a list + # of 64 bit architectures + machine_type = f.read(2) + if machine_type in ARCH64PE: + self.is64 = True + else: + # read the 32bit / 64bit field + machine = f.read(5)[4:] + if machine == ARCH64ELF64: + self.is64 = True + self.suffix = ".i64" if self.is64 else ".idb" + self._path += "64" if self.is64 else "" + + database_file = binary_file + self.suffix + + # execute the program - os.system(f"{self._path} -A -B -T{type} -o{database_file} {binary_file}") + subprocess.run([self._path, "-A" , "-B", f"-T{type}" ,f"-o{database_file}", binary_file]) # return back the (should be) created database file path return database_file @@ -56,7 +87,7 @@ def executeScript(self, database, script): database (path): path to a database file created by the same program script (path): python script to be executed once the database is loaded """ - os.system(f"{self._path} -A -S{script} {database}") + subprocess.run([self._path, "-A", f"-S{script}", database]) # Don't forget to register at the factory diff --git a/src/karta/installers/__init__.py b/src/karta/installers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/karta/installers/common_installer.py b/src/karta/installers/common_installer.py new file mode 100644 index 0000000..2c80ded --- /dev/null +++ b/src/karta/installers/common_installer.py @@ -0,0 +1,63 @@ +import os +import platform + +from ..config.utils import addDisassembler, getDisassembler, setDefaultDisassembler + + +def common_locations(): + """ + Gets a list of common places where a disassembler may be installed + """ + location_list = list() + # based on the operating system installation location may vary + system_lower = platform.system().lower() + if system_lower == "windows".lower(): + # add drives + from ctypes import windll + kernel32 = windll.kernel32 + logical_drives = kernel32.GetLogicalDrives() + current_letter = 'A' + while logical_drives > 0: + if logical_drives & 1 == 1: + location_list.append(current_letter + ':\\') + logical_drives >>= 1 + current_letter = chr(ord(current_letter) + 1) + # add program files + location_list.append(os.environ['ProgramFiles']) + elif system_lower == "linux".lower(): + location_list.append("~") + location_list.append("/opt") + return location_list + + +def detect_installation(pattern, installation_file): + """ + If already installed get the installation path + Detect installation of a disassembler by a regex provided by its specific installer + Save it into a file so you may use it as a default + If its not in any of the common locations ask the user to enter it + """ + disassembler = getDisassembler(installation_file) + if disassembler is not None: + return disassembler + + for location in common_locations(): + for directory in next(os.walk(location))[1]: + if pattern.match(directory): + install_directory = os.path.join(location, directory) + addDisassembler(installation_file, install_directory) + return install_directory + try: + disass_path = input("Please enter the path to the folder of the disassembler: ") + if os.path.isdir(disass_path): + path = os.path.abspath(disass_path) + addDisassembler(installation_file, path) + return path + else: + print("The path you entered is not a directory") + except KeyboardInterrupt: + exit(0) + + +def set_default_disassembler(disassembler_name): + setDefaultDisassembler(disassembler_name) diff --git a/src/karta/installers/ida_installer.py b/src/karta/installers/ida_installer.py new file mode 100644 index 0000000..1969c5f --- /dev/null +++ b/src/karta/installers/ida_installer.py @@ -0,0 +1,59 @@ +import os +import re +from shutil import copytree, copyfile + +from .common_installer import detect_installation, set_default_disassembler + + +disassembler_name = "ida_path" + +def detect_ida(save_file): + """ + find where ida is installed by this regex, should work for both windows and linux + """ + pattern = re.compile("ida( pro )?-?\d\.\d", re.IGNORECASE) + return detect_installation(pattern, save_file) + +def wanted_ida_files(adir, filenames): + """ + list all files that will go into the ida plugins directory + no need to copy neither compiled python files nor files not relevent for ida + """ + filelist = list() + caches = re.compile("(__pycache__|.*\.pyc)") + non_needed_content = re.compile("(installers|plugins|.*_path)") + ida_valid = re.compile("(IDA|.*\.py)") + for filename in filenames: + if type(adir) != str and adir.name == "disassembler" and not ida_valid.match(filename): + filelist.append(filename) + continue + if caches.match(filename) or non_needed_content.match(filename): + filelist.append(filename) + return filelist + +def main(): + """ + detect ida copy library files into its plugins directory and add the plugin to be run on start + """ + path = detect_ida(disassembler_name) + set_default_disassembler(disassembler_name) + src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + ida_plugin = os.path.join(src_dir, "plugins", "ida_plugin.py") + plugin_dir = os.path.join(path, "plugins") + karta_dir = os.path.join(plugin_dir, "karta") + plugin_dst = os.path.join(plugin_dir, "ida_karta.py") + copytree( + src_dir, + karta_dir, + ignore=wanted_ida_files, + dirs_exist_ok=True + ) + copyfile( + ida_plugin, + plugin_dst + ) + + + +if __name__ == "__main__": + main() diff --git a/src/karta/karta_analyze_src.py b/src/karta/karta_analyze_src.py index 920d86b..5287dbc 100644 --- a/src/karta/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -124,7 +124,6 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): prompter.debug(f"{full_file_path} - {compiled_file}") if progress_bar is None: prompter.info(f"{compiled_file} - {full_file_path}") - # analyze the file analyzeFile(full_file_path, is_windows) # load the JSON data from it try: @@ -324,8 +323,7 @@ def main(args): prompter.info("Starting the Script") # requesting the path to the chosen disassembler - setDisassemblerPath(prompter) - disas_cmd = identifyDisassemblerHandler(getDisasPath(), prompter) + disas_cmd = identifyDisassemblerHandler(getDisasPath(prompter), prompter) if disas_cmd is None: return @@ -336,11 +334,6 @@ def main(args): if is_windows: setWindowsMode() - # Check if launched from the src directory - if not os.path.exists(SCRIPT_PATH): - prompter.error("The script should be executed from Karta's src directory!") - prompter.error("Exiting") - return # analyze the open source library analyzeLibrary(constructConfigPath(library_name, library_version), bin_dirs, archive_paths, prompter) diff --git a/src/karta/plugins/__init__.py b/src/karta/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/karta/plugins/ida_plugin.py b/src/karta/plugins/ida_plugin.py new file mode 100644 index 0000000..46117e2 --- /dev/null +++ b/src/karta/plugins/ida_plugin.py @@ -0,0 +1 @@ +pass From a13f318954343f15d8e18c40ef66aa3ada7c7051 Mon Sep 17 00:00:00 2001 From: Harelon Date: Tue, 1 Feb 2022 15:29:55 +0200 Subject: [PATCH 06/11] Improved performence in ida analyze src by analyzing and executing script in the same instance --- .../disassembler/IDA/ida_analysis_api.py | 1 + src/karta/disassembler/IDA/ida_cmd_api.py | 72 +++++++++++-------- src/karta/disassembler/disas_api.py | 6 +- src/karta/karta_analyze_src.py | 7 +- 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/src/karta/disassembler/IDA/ida_analysis_api.py b/src/karta/disassembler/IDA/ida_analysis_api.py index 4d91718..071b368 100644 --- a/src/karta/disassembler/IDA/ida_analysis_api.py +++ b/src/karta/disassembler/IDA/ida_analysis_api.py @@ -23,6 +23,7 @@ def __init__(self, disas): disas (disassembler): disassembler layer instance """ self.disas = disas + idaapi.auto_wait() def funcNameInner(self, raw_func_name): """Return the name of the function (including windows name fixes). diff --git a/src/karta/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py index e68e08c..8ec62a2 100644 --- a/src/karta/disassembler/IDA/ida_cmd_api.py +++ b/src/karta/disassembler/IDA/ida_cmd_api.py @@ -4,12 +4,6 @@ from ..disas_api import DisasCMD from ..factory import registerDisassemblerCMD -# machine type header of pe -# specified in that order, amd64, arm64, ia64, loongarch64, riscv64 -ARCH64PE = [b"\x64\x86", b"\x64\xaa", b"\x00\x02", b"\x64\x62", b"\x64\x50"] - -# machine type header of elf -ARCH64ELF64 = b"\x02" class IdaCMD(DisasCMD): """DisasCMD implementation for the IDA disassembler.""" @@ -49,31 +43,11 @@ def createDatabase(self, binary_file, is_windows): path to the created database file """ type = "elf" if not is_windows else "coff" - # because we get the path to the folder of the dissasembler - # we need to define whether it is a 64 or 32 bit one - # so we check the respective fields in coff files and elf files - # and define the file ending and the ida to use based on tehm - if not hasattr(self, "is64"): - self._path = os.path.join(self._path, "ida") - self.is64 = False - with open(binary_file, 'rb') as f: - if is_windows: - # read machine type header and check whether it is in a list - # of 64 bit architectures - machine_type = f.read(2) - if machine_type in ARCH64PE: - self.is64 = True - else: - # read the 32bit / 64bit field - machine = f.read(5)[4:] - if machine == ARCH64ELF64: - self.is64 = True - self.suffix = ".i64" if self.is64 else ".idb" - self._path += "64" if self.is64 else "" - - database_file = binary_file + self.suffix + if not hasattr(self, "is64"): + self.decideArchitecureChoices(binary_file, is_windows) + database_file = binary_file + self.suffix # execute the program subprocess.run([self._path, "-A" , "-B", f"-T{type}" ,f"-o{database_file}", binary_file]) # return back the (should be) created database file path @@ -88,7 +62,47 @@ def executeScript(self, database, script): script (path): python script to be executed once the database is loaded """ subprocess.run([self._path, "-A", f"-S{script}", database]) + + def isSupported(self, feature_name): + return hasattr(self, feature_name) + + def createAndExecute(self, binary_file, is_windows, script): + type = "elf" if not is_windows else "coff" + + if not hasattr(self, "is64"): + self.decideArchitecureChoices(binary_file, is_windows) + + # execute the program + subprocess.run([self._path, "-A" , f"-S{script}", f"-T{type}", binary_file]) + + def decideArchitecureChoices(self, binary_file, is_windows): + # machine type header of pe + # specified in that order, amd64, arm64, ia64, loongarch64, riscv64 + ARCH64PE = [b"\x64\x86", b"\x64\xaa", b"\x00\x02", b"\x64\x62", b"\x64\x50"] + # machine type header of elf + ARCH64ELF64 = b"\x02" + # because we get the path to the folder of the dissasembler + # we need to define whether it is a 64 or 32 bit one + # so we check the respective fields in coff files and elf files + # and define the file ending and the ida to use based on tehm + self._path = os.path.join(self._path, "ida") + self.is64 = False + with open(binary_file, 'rb') as f: + if is_windows: + # read machine type header and check whether it is in a list + # of 64 bit architectures + machine_type = f.read(2) + if machine_type in ARCH64PE: + self.is64 = True + else: + # read the 32bit / 64bit field + machine = f.read(5)[4:] + if machine == ARCH64ELF64: + self.is64 = True + self.suffix = ".i64" if self.is64 else ".idb" + self._path += "64" if self.is64 else "" + # Don't forget to register at the factory registerDisassemblerCMD(IdaCMD.identify, IdaCMD) diff --git a/src/karta/disassembler/disas_api.py b/src/karta/disassembler/disas_api.py index 1ea9752..f2320fb 100644 --- a/src/karta/disassembler/disas_api.py +++ b/src/karta/disassembler/disas_api.py @@ -2,6 +2,7 @@ ## Note: This API is an indirection point, so we could (maybe) add support for more disassemblers in the future ## ################################################################################################################## +import os from collections import defaultdict class DisasAPI(object): @@ -527,7 +528,7 @@ def __init__(self, path): Args: path (path): command line path for the program """ - self._path = path + self._path = os.path.normpath(path) @staticmethod def identify(path): @@ -570,6 +571,9 @@ def executeScript(self, database, script): script (path): python script to be executed once the database is loaded """ raise NotImplementedError("Subclasses should implement this!") + + def isSupported(self, feature_name): + return False class DisasVerifier(object): """Abstract class that verifies that we are running inside our matching disassembler. diff --git a/src/karta/karta_analyze_src.py b/src/karta/karta_analyze_src.py index 5287dbc..9e39de7 100644 --- a/src/karta/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -53,8 +53,11 @@ def analyzeFile(full_file_path, is_windows): full_file_path (str): full path to the specific (*.obj / *.o) file is_windows (bool): True iff a windows compilation (*.obj or *.o) """ - database_path = disas_cmd.createDatabase(full_file_path, is_windows) - disas_cmd.executeScript(database_path, SCRIPT_PATH) + if disas_cmd.isSupported("createAndExecute"): + disas_cmd.createAndExecute(full_file_path, is_windows, SCRIPT_PATH) + else: + database_path = disas_cmd.createDatabase(full_file_path, is_windows) + disas_cmd.executeScript(database_path, SCRIPT_PATH) def resolveUnknowns(): """Resolve "unknown" references between the different compiled files.""" From a14a2d2a433443b659ab317197dc95302805734d Mon Sep 17 00:00:00 2001 From: Harelon Date: Tue, 1 Feb 2022 23:50:01 +0200 Subject: [PATCH 07/11] Improve karta_analyze_src with async actions and multiprocessing --- src/karta/disassembler/IDA/ida_cmd_api.py | 17 +++-- src/karta/disassembler/disas_api.py | 5 +- src/karta/karta_analyze_src.py | 78 ++++++++++++++--------- 3 files changed, 61 insertions(+), 39 deletions(-) diff --git a/src/karta/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py index 8ec62a2..39d6848 100644 --- a/src/karta/disassembler/IDA/ida_cmd_api.py +++ b/src/karta/disassembler/IDA/ida_cmd_api.py @@ -1,5 +1,5 @@ import os -import subprocess +import asyncio from ..disas_api import DisasCMD from ..factory import registerDisassemblerCMD @@ -32,7 +32,7 @@ def name(): return "IDA" # Overridden base function - def createDatabase(self, binary_file, is_windows): + async def createDatabase(self, binary_file, is_windows): """Create a database file for the given binary file, compiled to windows or linux as specified. Args: @@ -49,31 +49,34 @@ def createDatabase(self, binary_file, is_windows): database_file = binary_file + self.suffix # execute the program - subprocess.run([self._path, "-A" , "-B", f"-T{type}" ,f"-o{database_file}", binary_file]) + process = await asyncio.create_subprocess_exec(self._path, "-A" , "-B", f"-T{type}" ,f"-o{database_file}", binary_file) + await process.wait() # return back the (should be) created database file path return database_file # Overridden base function - def executeScript(self, database, script): + async def executeScript(self, database, script): """Execute the given script over the given database file that was created earlier. Args: database (path): path to a database file created by the same program script (path): python script to be executed once the database is loaded """ - subprocess.run([self._path, "-A", f"-S{script}", database]) + process = await asyncio.create_subprocess_exec(self._path, "-A", f"-S{script}", database) + await process.wait() def isSupported(self, feature_name): return hasattr(self, feature_name) - def createAndExecute(self, binary_file, is_windows, script): + async def createAndExecute(self, binary_file, is_windows, script): type = "elf" if not is_windows else "coff" if not hasattr(self, "is64"): self.decideArchitecureChoices(binary_file, is_windows) # execute the program - subprocess.run([self._path, "-A" , f"-S{script}", f"-T{type}", binary_file]) + process = await asyncio.create_subprocess_exec(self._path, "-A", "-c" , f"-S{script}", f"-T{type}", binary_file) + await process.wait() def decideArchitecureChoices(self, binary_file, is_windows): # machine type header of pe diff --git a/src/karta/disassembler/disas_api.py b/src/karta/disassembler/disas_api.py index f2320fb..6302254 100644 --- a/src/karta/disassembler/disas_api.py +++ b/src/karta/disassembler/disas_api.py @@ -3,6 +3,7 @@ ################################################################################################################## import os +import asyncio from collections import defaultdict class DisasAPI(object): @@ -551,7 +552,7 @@ def name(): """ raise NotImplementedError("Subclasses should implement this!") - def createDatabase(self, binary_file, is_windows): + async def createDatabase(self, binary_file, is_windows): """Create a database file for the given binary file, compiled to windows or linux as specified. Args: @@ -563,7 +564,7 @@ def createDatabase(self, binary_file, is_windows): """ raise NotImplementedError("Subclasses should implement this!") - def executeScript(self, database, script): + async def executeScript(self, database, script): """Execute the given script over the given database file that was created earlier. Args: diff --git a/src/karta/karta_analyze_src.py b/src/karta/karta_analyze_src.py index 9e39de7..1ef8b9a 100644 --- a/src/karta/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -1,8 +1,10 @@ #!/usr/bin/python3 +from multiprocessing import Semaphore import sys import argparse import logging +import asyncio from os import path, walk from elementals import Prompter, ProgressBar @@ -46,7 +48,7 @@ def locateFiles(bin_dir, file_list, suffix): for file in filter(lambda x: x.endswith("." + suffix), files): yield path.abspath(path.join(root, file)), file -def analyzeFile(full_file_path, is_windows): +async def analyzeFile(full_file_path, is_windows): """Analyze a single file using analyzer script. Args: @@ -54,10 +56,40 @@ def analyzeFile(full_file_path, is_windows): is_windows (bool): True iff a windows compilation (*.obj or *.o) """ if disas_cmd.isSupported("createAndExecute"): - disas_cmd.createAndExecute(full_file_path, is_windows, SCRIPT_PATH) + await disas_cmd.createAndExecute(full_file_path, is_windows, SCRIPT_PATH) else: - database_path = disas_cmd.createDatabase(full_file_path, is_windows) - disas_cmd.executeScript(database_path, SCRIPT_PATH) + database_path = await disas_cmd.createDatabase(full_file_path, is_windows) + await disas_cmd.executeScript(database_path, SCRIPT_PATH) + +async def processFile(full_file_path, is_windows, compiled_file, progress_bar, prompter, semaphore): + # run the analsys on the files in an asynchrnous way + prompter.debug(f"{full_file_path} - {compiled_file}") + if progress_bar is None: + prompter.info(f"{compiled_file} - {full_file_path}") + # no need to analyze the file again if the process was canceled and the file state was saved + if not os.path.exists(full_file_path + STATE_FILE_SUFFIX): + await analyzeFile(full_file_path, is_windows) + # load the JSON data from it + try: + fd = open(full_file_path + STATE_FILE_SUFFIX, "r") + except IOError: + prompter.error(f"Failed to create the .JSON file for file: {compiled_file}") + prompter.error("Read the log file for more information:") + prompter.addIndent() + prompter.error(constructInitLogPath(full_file_path)) + prompter.error(constructLogPath(full_file_path)) + prompter.removeIndent() + prompter.removeIndent() + prompter.removeIndent() + prompter.error("Encountered an error, exiting") + exit(1) + # all was OK, can continue + parseFileStats(full_file_path, json.load(fd)) + fd.close() + if progress_bar is not None: + progress_bar.advance(1) + # release semaphore and allow an instance to run on another file + semaphore.release() def resolveUnknowns(): """Resolve "unknown" references between the different compiled files.""" @@ -71,7 +103,7 @@ def resolveUnknowns(): src_func_ctx.recordCall(resolved_call) src_func_ctx.unknown_fptrs.clear() -def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): +async def analyzeLibrary(config_name, bin_dirs, compiled_ars, concurrency, prompter): """Analyze the open source library, file-by-file and merge the results. Args: @@ -117,36 +149,20 @@ def analyzeLibrary(config_name, bin_dirs, compiled_ars, prompter): progress_bar = None # it makes more sense to have a sorted list archive_files.sort() - # start the work itself + # analyze many files at the same time, actions are parallel + semaphore = asyncio.Semaphore(concurrency) for full_file_path, compiled_file in archive_files: # ida has severe bugs, make sure to warn the user in advance if disas_cmd.name() == "IDA" and ' ' in full_file_path: prompter.error("IDA does not support spaces (' ') in the file's path (in script mode). Please move the binary directory accordingly (I feel your pain)") prompter.removeIndent() return - prompter.debug(f"{full_file_path} - {compiled_file}") - if progress_bar is None: - prompter.info(f"{compiled_file} - {full_file_path}") - analyzeFile(full_file_path, is_windows) - # load the JSON data from it - try: - fd = open(full_file_path + STATE_FILE_SUFFIX, "r") - except IOError: - prompter.error(f"Failed to create the .JSON file for file: {compiled_file}") - prompter.error("Read the log file for more information:") - prompter.addIndent() - prompter.error(constructInitLogPath(full_file_path)) - prompter.error(constructLogPath(full_file_path)) - prompter.removeIndent() - prompter.removeIndent() - prompter.removeIndent() - prompter.error("Encountered an error, exiting") - exit(1) - # all was OK, can continue - parseFileStats(full_file_path, json.load(fd)) - fd.close() - if progress_bar is not None: - progress_bar.advance(1) + await semaphore.acquire() + # start the work itself + asyncio.create_task(processFile(full_file_path, is_windows, compiled_file, progress_bar, prompter, semaphore)) + # we want to wait for all the disassemblers to finish + for _ in range(concurrency): + await semaphore.acquire() # wrap it up if progress_bar is not None: progress_bar.finish() @@ -297,6 +313,7 @@ def main(args): help="version string (case sensitive) as used by the identifier") parser.add_argument("couples", metavar="dir archive", type=str, nargs="+", help="directory with the compiled *.o / *.obj files + path to the matching *.a / *.lib file (if didn't use \"--no-archive\")") + parser.add_argument("-C", "--concurrency", default=5, type=int, help="number of disassemblers to run in parallel") parser.add_argument("-D", "--debug", action="store_true", help="set logging level to logging.DEBUG") parser.add_argument("-N", "--no-archive", action="store_false", help="extract data from all *.o / *.obj files in the directory") parser.add_argument("-W", "--windows", action="store_true", help="signals that the binary was compiled for Windows") @@ -309,6 +326,7 @@ def main(args): is_windows = args.windows using_archives = args.no_archive couples = args.couples + concurrency = args.concurrency # open the log prompter = Prompter(min_log_level=logging.INFO if not is_debug else logging.DEBUG) @@ -339,7 +357,7 @@ def main(args): # analyze the open source library - analyzeLibrary(constructConfigPath(library_name, library_version), bin_dirs, archive_paths, prompter) + asyncio.run(analyzeLibrary(constructConfigPath(library_name, library_version), bin_dirs, archive_paths, concurrency, prompter)) # finished prompter.info("Finished Successfully") From 0ba209ce39e2db4ffdad7e33b5a08530b98442cf Mon Sep 17 00:00:00 2001 From: Harelon Date: Sat, 12 Feb 2022 16:30:50 +0200 Subject: [PATCH 08/11] Change plugin installer to only copy plugin file --- src/karta/installers/ida_installer.py | 29 ++------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/karta/installers/ida_installer.py b/src/karta/installers/ida_installer.py index 1969c5f..0206bc8 100644 --- a/src/karta/installers/ida_installer.py +++ b/src/karta/installers/ida_installer.py @@ -1,6 +1,6 @@ import os import re -from shutil import copytree, copyfile +from shutil import copyfile from .common_installer import detect_installation, set_default_disassembler @@ -14,23 +14,6 @@ def detect_ida(save_file): pattern = re.compile("ida( pro )?-?\d\.\d", re.IGNORECASE) return detect_installation(pattern, save_file) -def wanted_ida_files(adir, filenames): - """ - list all files that will go into the ida plugins directory - no need to copy neither compiled python files nor files not relevent for ida - """ - filelist = list() - caches = re.compile("(__pycache__|.*\.pyc)") - non_needed_content = re.compile("(installers|plugins|.*_path)") - ida_valid = re.compile("(IDA|.*\.py)") - for filename in filenames: - if type(adir) != str and adir.name == "disassembler" and not ida_valid.match(filename): - filelist.append(filename) - continue - if caches.match(filename) or non_needed_content.match(filename): - filelist.append(filename) - return filelist - def main(): """ detect ida copy library files into its plugins directory and add the plugin to be run on start @@ -39,15 +22,7 @@ def main(): set_default_disassembler(disassembler_name) src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') ida_plugin = os.path.join(src_dir, "plugins", "ida_plugin.py") - plugin_dir = os.path.join(path, "plugins") - karta_dir = os.path.join(plugin_dir, "karta") - plugin_dst = os.path.join(plugin_dir, "ida_karta.py") - copytree( - src_dir, - karta_dir, - ignore=wanted_ida_files, - dirs_exist_ok=True - ) + plugin_dst = os.path.join(path, "plugins", "ida_karta.py") copyfile( ida_plugin, plugin_dst From dca629cd4f11f0b73dde74f560b8e1df647a3758 Mon Sep 17 00:00:00 2001 From: Harelon Date: Mon, 14 Feb 2022 20:14:16 +0200 Subject: [PATCH 09/11] Convert karta_analyze_src into a console script --- setup.py | 5 +++++ src/karta/karta_analyze_src.py | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 171378c..4679ea5 100755 --- a/setup.py +++ b/setup.py @@ -23,5 +23,10 @@ "License :: OSI Approved :: MIT License (MIT License)", "Operating System :: OS Independent", ], + entry_points={ + 'console_scripts': [ + 'karta_analyze_src = karta.karta_analyze_src:main' + ] + }, zip_safe=False ) diff --git a/src/karta/karta_analyze_src.py b/src/karta/karta_analyze_src.py index 1ef8b9a..cc504a8 100644 --- a/src/karta/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -1,6 +1,5 @@ #!/usr/bin/python3 -from multiprocessing import Semaphore import sys import argparse import logging @@ -297,7 +296,7 @@ def verify_archives_and_objects(using_archives, couples, prompter): return (bin_dirs, archive_paths) if not error_occured else None -def main(args): +def main(args=None): """Create a .json configuration for the open source library version. Args: From ef8b16d37e298f31b056b1d0840686a6a3ba2d2e Mon Sep 17 00:00:00 2001 From: Harelon Date: Mon, 14 Feb 2022 20:31:25 +0200 Subject: [PATCH 10/11] Add karta package reference into the scripts --- src/karta/karta_identifier.py | 6 +++--- src/karta/karta_manual_anchor.py | 4 ++-- src/karta/karta_manual_identifier.py | 4 ++-- src/karta/karta_matcher.py | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/karta/karta_identifier.py b/src/karta/karta_identifier.py index 74e0b70..aadc55d 100644 --- a/src/karta/karta_identifier.py +++ b/src/karta/karta_identifier.py @@ -1,9 +1,9 @@ import logging from elementals import Logger -from .config.utils import * -from .disassembler.factory import createDisassemblerHandler -from .libs import lib_factory +from karta.config.utils import * +from karta.disassembler.factory import createDisassemblerHandler +from karta.libs import lib_factory #################### ## Global Configs ## diff --git a/src/karta/karta_manual_anchor.py b/src/karta/karta_manual_anchor.py index d6a06ab..ba1065d 100644 --- a/src/karta/karta_manual_anchor.py +++ b/src/karta/karta_manual_anchor.py @@ -7,8 +7,8 @@ from collections import defaultdict from elementals import Prompter -from .config.utils import * -from .function_context import SourceContext, BinaryContext, IslandContext +from karta.config.utils import * +from karta.function_context import SourceContext, BinaryContext, IslandContext def recordManualAnchors(library_config, knowledge_config, lib_name, prompter): """Record the list of user defined manual anchor matches. diff --git a/src/karta/karta_manual_identifier.py b/src/karta/karta_manual_identifier.py index eba4021..c4bcacc 100644 --- a/src/karta/karta_manual_identifier.py +++ b/src/karta/karta_manual_identifier.py @@ -6,8 +6,8 @@ from collections import defaultdict from elementals import Prompter -from .config.utils import * -from .libs import lib_factory +from karta.config.utils import * +from karta.libs import lib_factory def recordManualVersions(knowledge_config, prompter): diff --git a/src/karta/karta_matcher.py b/src/karta/karta_matcher.py index dc360e5..c142c65 100644 --- a/src/karta/karta_matcher.py +++ b/src/karta/karta_matcher.py @@ -1,10 +1,10 @@ import logging from elementals import Logger -from .config.utils import * -from .disassembler.factory import createDisassemblerHandler -from .matching_engine import KartaMatcher -from .libs import lib_factory +from karta.config.utils import * +from karta.disassembler.factory import createDisassemblerHandler +from karta.matching_engine import KartaMatcher +from karta.libs import lib_factory ###################### ## Global Variables ## From fc18723e6ddfb27255dfc1a58d79986cd9bb16fc Mon Sep 17 00:00:00 2001 From: Harelon Date: Mon, 14 Feb 2022 22:06:44 +0200 Subject: [PATCH 11/11] Format files to pass tests --- setup.py | 8 +++--- src/karta/config/utils.py | 28 +++++++++++++++++-- src/karta/disassembler/IDA/ida_cmd_api.py | 34 +++++++++++++++++++---- src/karta/disassembler/disas_api.py | 12 +++++--- src/karta/installers/common_installer.py | 26 +++++++++-------- src/karta/installers/ida_installer.py | 26 +++++++++-------- src/karta/karta_analyze_src.py | 28 +++++++++++++++---- 7 files changed, 116 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index 4679ea5..d466585 100755 --- a/setup.py +++ b/setup.py @@ -24,9 +24,9 @@ "Operating System :: OS Independent", ], entry_points={ - 'console_scripts': [ - 'karta_analyze_src = karta.karta_analyze_src:main' - ] + 'console_scripts': [ + 'karta_analyze_src = karta.karta_analyze_src:main' + ] }, zip_safe=False -) + ) diff --git a/src/karta/config/utils.py b/src/karta/config/utils.py index f6bf3cc..674ebb9 100644 --- a/src/karta/config/utils.py +++ b/src/karta/config/utils.py @@ -10,7 +10,7 @@ DISASSEMBLER_PATH = None CONFIG_DIR = os.path.dirname(os.path.realpath(__file__)) DEFAULT_DISASSEMBLER = os.path.join(CONFIG_DIR, "default_disassembler_path") -SCRIPT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","analyze_src_file.py") +SCRIPT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "analyze_src_file.py") LIBRARY_NAME = "Karta" STATE_FILE_SUFFIX = "_file_state.json" @@ -459,18 +459,42 @@ def isMatching(): return matching_mode def addDisassembler(name, path): + """Add an installation of a dissasembler. + + Args: + name (str): name of the installation file + path (str): directory of the disassembler installtion + """ with open(os.path.join(CONFIG_DIR, name), "w") as f: f.write(path) def disassemblerInstallationExists(name): + """Check whether there is an existing installation with the filename. + + Args: + name (str): filename of the installtion to check + + Return Value: + return true if such installtion exists + """ return os.path.exists(os.path.join(CONFIG_DIR, name)) def getDisassembler(name): + """Get directory of disassembler from configuration by it's configuration file name. + + Args: + name (str): name of the disassembler to search for + """ if disassemblerInstallationExists(name): with open(os.path.join(CONFIG_DIR, name), "r") as f: return f.read() def setDefaultDisassembler(name): + """Set the default disassembler in the configuration file. + + Args: + name (str): name of the file that contains the default disassembler to use. + """ with open(os.path.join(CONFIG_DIR, DEFAULT_DISASSEMBLER), "w") as f: if os.path.isfile(os.path.join(CONFIG_DIR, name)): f.write(name) @@ -488,7 +512,7 @@ def getDisasPath(prompter): actual_disas_path = None with open(DEFAULT_DISASSEMBLER, 'r') as f: actual_disas_path = f.read() - full_disas_name_path = os.path.join(CONFIG_DIR ,actual_disas_path) + full_disas_name_path = os.path.join(CONFIG_DIR, actual_disas_path) if os.path.isfile(full_disas_name_path): with open(full_disas_name_path, 'r') as f: DISASSEMBLER_PATH = f.read() diff --git a/src/karta/disassembler/IDA/ida_cmd_api.py b/src/karta/disassembler/IDA/ida_cmd_api.py index 39d6848..1fc62ea 100644 --- a/src/karta/disassembler/IDA/ida_cmd_api.py +++ b/src/karta/disassembler/IDA/ida_cmd_api.py @@ -33,7 +33,7 @@ def name(): # Overridden base function async def createDatabase(self, binary_file, is_windows): - """Create a database file for the given binary file, compiled to windows or linux as specified. + """Create a database file asynchonously for the given binary file , compiled to windows or linux as specified. Args: binary_file (path): path to the input binary (*.o / *.obj) file @@ -49,14 +49,14 @@ async def createDatabase(self, binary_file, is_windows): database_file = binary_file + self.suffix # execute the program - process = await asyncio.create_subprocess_exec(self._path, "-A" , "-B", f"-T{type}" ,f"-o{database_file}", binary_file) + process = await asyncio.create_subprocess_exec(self._path, "-A", "-B", f"-T{type}", f"-o{database_file}", binary_file) await process.wait() # return back the (should be) created database file path return database_file # Overridden base function async def executeScript(self, database, script): - """Execute the given script over the given database file that was created earlier. + """Execute the given script asynchonously over the given database file that was created earlier. Args: database (path): path to a database file created by the same program @@ -64,21 +64,42 @@ async def executeScript(self, database, script): """ process = await asyncio.create_subprocess_exec(self._path, "-A", f"-S{script}", database) await process.wait() - + def isSupported(self, feature_name): + """Check if feature is enabled. + + Args: + feature_name (str): name of the feature to check if enabled + + Return Value: + returns whether the current class has a member name "feature_name" + """ return hasattr(self, feature_name) async def createAndExecute(self, binary_file, is_windows, script): + """Execute the given script over the given database asynchonously without closing it first. + + Args: + binary_file (str): filename of the binary to analyze + is_windows (bool): whether the file is a windows compiled file or linux compiled file + script (str): filename of the script to use on the binary in ida + """ type = "elf" if not is_windows else "coff" if not hasattr(self, "is64"): self.decideArchitecureChoices(binary_file, is_windows) # execute the program - process = await asyncio.create_subprocess_exec(self._path, "-A", "-c" , f"-S{script}", f"-T{type}", binary_file) + process = await asyncio.create_subprocess_exec(self._path, "-A", "-c", f"-S{script}", f"-T{type}", binary_file) await process.wait() def decideArchitecureChoices(self, binary_file, is_windows): + """Automate deciding whether to send the files to ida64 or ida. + + Args: + binary_file (str): filename of the file to be a test case for analysis + is_windows (bool): whether the test file is a linux or windows binary + """ # machine type header of pe # specified in that order, amd64, arm64, ia64, loongarch64, riscv64 ARCH64PE = [b"\x64\x86", b"\x64\xaa", b"\x00\x02", b"\x64\x62", b"\x64\x50"] @@ -106,6 +127,7 @@ def decideArchitecureChoices(self, binary_file, is_windows): self.is64 = True self.suffix = ".i64" if self.is64 else ".idb" self._path += "64" if self.is64 else "" - + + # Don't forget to register at the factory registerDisassemblerCMD(IdaCMD.identify, IdaCMD) diff --git a/src/karta/disassembler/disas_api.py b/src/karta/disassembler/disas_api.py index 6302254..18261dd 100644 --- a/src/karta/disassembler/disas_api.py +++ b/src/karta/disassembler/disas_api.py @@ -3,7 +3,6 @@ ################################################################################################################## import os -import asyncio from collections import defaultdict class DisasAPI(object): @@ -553,7 +552,7 @@ def name(): raise NotImplementedError("Subclasses should implement this!") async def createDatabase(self, binary_file, is_windows): - """Create a database file for the given binary file, compiled to windows or linux as specified. + """Create a database file asynchonously for the given binary file, compiled to windows or linux as specified. Args: binary_file (path): path to the input binary (*.o / *.obj) file @@ -565,15 +564,20 @@ async def createDatabase(self, binary_file, is_windows): raise NotImplementedError("Subclasses should implement this!") async def executeScript(self, database, script): - """Execute the given script over the given database file that was created earlier. + """Execute the given script asynchonously over the given database file that was created earlier. Args: database (path): path to a database file created by the same program script (path): python script to be executed once the database is loaded """ raise NotImplementedError("Subclasses should implement this!") - + def isSupported(self, feature_name): + """Allow script to query whether a feature is enabled in it's disas_api. + + Return Value: + False, no featue is supported on this Abstract class + """ return False class DisasVerifier(object): diff --git a/src/karta/installers/common_installer.py b/src/karta/installers/common_installer.py index 2c80ded..692ab8d 100644 --- a/src/karta/installers/common_installer.py +++ b/src/karta/installers/common_installer.py @@ -4,10 +4,8 @@ from ..config.utils import addDisassembler, getDisassembler, setDefaultDisassembler -def common_locations(): - """ - Gets a list of common places where a disassembler may be installed - """ +def commonLocations(): + """Get a list of common places where a disassembler may be installed on any os.""" location_list = list() # based on the operating system installation location may vary system_lower = platform.system().lower() @@ -30,18 +28,21 @@ def common_locations(): return location_list -def detect_installation(pattern, installation_file): - """ - If already installed get the installation path - Detect installation of a disassembler by a regex provided by its specific installer - Save it into a file so you may use it as a default - If its not in any of the common locations ask the user to enter it +def detectInstallation(pattern, installation_file): + """Detect a disassembler's installation by regex, if not found get it with input. + + Args: + pattern (re.Pattern): pattern to detect an installation + installation_file (str): name of the file to save the disassembler name in + + Return Value: + Directory of the disassembler file """ disassembler = getDisassembler(installation_file) if disassembler is not None: return disassembler - for location in common_locations(): + for location in commonLocations(): for directory in next(os.walk(location))[1]: if pattern.match(directory): install_directory = os.path.join(location, directory) @@ -59,5 +60,6 @@ def detect_installation(pattern, installation_file): exit(0) -def set_default_disassembler(disassembler_name): +def commonSetDefaultDisassembler(disassembler_name): + """Call config utils setDefaultDisassembler function.""" setDefaultDisassembler(disassembler_name) diff --git a/src/karta/installers/ida_installer.py b/src/karta/installers/ida_installer.py index 0206bc8..b4882d4 100644 --- a/src/karta/installers/ida_installer.py +++ b/src/karta/installers/ida_installer.py @@ -2,24 +2,27 @@ import re from shutil import copyfile -from .common_installer import detect_installation, set_default_disassembler +from .common_installer import detectInstallation, commonSetDefaultDisassembler disassembler_name = "ida_path" -def detect_ida(save_file): - """ - find where ida is installed by this regex, should work for both windows and linux +def detectIda(save_file): + """Find where ida is installed by this regex, should work for both windows and linux. + + Args: + save_file (str): name of the file which will save the path to the disassembler directory + + Return Value: + Installation folder for the ida pro disassembler """ - pattern = re.compile("ida( pro )?-?\d\.\d", re.IGNORECASE) - return detect_installation(pattern, save_file) + pattern = re.compile("ida( pro )?-?\\d\\.\\d", re.IGNORECASE) + return detectInstallation(pattern, save_file) def main(): - """ - detect ida copy library files into its plugins directory and add the plugin to be run on start - """ - path = detect_ida(disassembler_name) - set_default_disassembler(disassembler_name) + """Detect ida copy plugin file into its plugins directory.""" + path = detectIda(disassembler_name) + commonSetDefaultDisassembler(disassembler_name) src_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') ida_plugin = os.path.join(src_dir, "plugins", "ida_plugin.py") plugin_dst = os.path.join(path, "plugins", "ida_karta.py") @@ -29,6 +32,5 @@ def main(): ) - if __name__ == "__main__": main() diff --git a/src/karta/karta_analyze_src.py b/src/karta/karta_analyze_src.py index cc504a8..e395939 100644 --- a/src/karta/karta_analyze_src.py +++ b/src/karta/karta_analyze_src.py @@ -48,7 +48,7 @@ def locateFiles(bin_dir, file_list, suffix): yield path.abspath(path.join(root, file)), file async def analyzeFile(full_file_path, is_windows): - """Analyze a single file using analyzer script. + """Analyze a single file asynchonously using analyzer script. Args: full_file_path (str): full path to the specific (*.obj / *.o) file @@ -61,6 +61,16 @@ async def analyzeFile(full_file_path, is_windows): await disas_cmd.executeScript(database_path, SCRIPT_PATH) async def processFile(full_file_path, is_windows, compiled_file, progress_bar, prompter, semaphore): + """Analyze a file asynchonously using a disassembler and parse the file stats. + + Args: + full_file_path (str): full path to the specific (*.obj / *.o) file + is_windows (bool): True iff a windows compilation (*.obj or *.o) + compiled_file (str): file name of the compiled_file + progress_bar (progressBar): progress bar to update once file processing is finished + prompter (prompter): notify the user when an error occured + semaphore (semaphore): release it when the disassembler finished processing our file + """ # run the analsys on the files in an asynchrnous way prompter.debug(f"{full_file_path} - {compiled_file}") if progress_bar is None: @@ -261,7 +271,14 @@ async def analyzeLibrary(config_name, bin_dirs, compiled_ars, concurrency, promp prompter.info(f"Anchor to function ratio is: {len(anchors_list)}/{len(src_functions_list)}") prompter.removeIndent() -def verify_archives_and_objects(using_archives, couples, prompter): +def verifyArchivesAndObjects(using_archives, couples, prompter): + """Verify that the archive and object files karta processes are valid. + + Args: + using_archives (bool): is karta analyzing object files only or library files too + couples (list): couples of dir and lib files or only bin dirs according to the using_archives parameter + prompter (prompter): prompter to notify the user if any error occures + """ bin_dirs = [] archive_paths = [] error_occured = False @@ -275,7 +292,7 @@ def verify_archives_and_objects(using_archives, couples, prompter): error_occured = True else: bin_dirs.append(couples[i]) - + if not path.exists(couples[i + 1]): prompter.error(f"Error the path {couples[i + 1]} does not exist!") error_occured = True @@ -332,8 +349,8 @@ def main(args=None): if using_archives: if len(couples) % 2 != 0: parser.error("Odd length in list of dir,archive couples, should be: [(directory, archive name), ...]") - - paths = verify_archives_and_objects(using_archives, couples, prompter) + + paths = verifyArchivesAndObjects(using_archives, couples, prompter) if paths is None: return @@ -354,7 +371,6 @@ def main(args=None): if is_windows: setWindowsMode() - # analyze the open source library asyncio.run(analyzeLibrary(constructConfigPath(library_name, library_version), bin_dirs, archive_paths, concurrency, prompter))