From 5308cbc55d60bfb44342078527e1c01f661b94c7 Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Thu, 12 Sep 2024 14:59:13 +0200 Subject: [PATCH 1/6] refactor: Use proper module strucutre This allows us to properly import the extractor via `import fact_extract` rather than importing all submodules individually. Also this is most definitely more correct. --- Dockerfile | 2 +- fact_extractor/__init__.py | 0 fact_extractor/docker_extraction.py | 8 ++++---- fact_extractor/fact_extract.py | 4 ++-- fact_extractor/helperFunctions/config.py | 2 +- fact_extractor/helperFunctions/hash.py | 2 +- fact_extractor/helperFunctions/plugin.py | 2 +- fact_extractor/helperFunctions/program_setup.py | 14 ++++++++++++-- fact_extractor/helperFunctions/statistics.py | 2 +- fact_extractor/install.py | 6 +++--- fact_extractor/install/common.py | 4 ++-- fact_extractor/install/unpacker.py | 2 +- .../plugins/unpacking/ambarella/code/ambarella.py | 2 +- .../unpacking/ambarella/code/ambarella_romfs.py | 2 +- .../ambarella/test/test_plugin_ambarella_romfs.py | 2 +- .../android_simg/test/test_plugin_android_simg.py | 2 +- .../plugins/unpacking/arj/test/test_arj.py | 2 +- .../unpacking/ascii85/test/test_plugin_adobe.py | 4 ++-- .../avm_kernel_image/code/avm_kernel_image.py | 2 +- .../avm_kernel_image/test/test_avm_kernel.py | 2 +- .../unpacking/avm_sqfs_fake/test/test_plugin.py | 2 +- .../unpacking/boschtool/test/test_boschtool.py | 4 ++-- .../plugins/unpacking/dahua/test/test_dahua.py | 2 +- .../plugins/unpacking/dji/code/dji_drones.py | 2 +- .../unpacking/dji/test/test_plugin_dji_drones.py | 2 +- .../unpacking/dlink_shrs/test/test_dlink_shrs.py | 2 +- .../test/test_draytek_vigor_167.py | 4 ++-- .../test/test_plugin_generic_carver_binwalk.py | 6 +++--- .../generic_fs/test/test_plugin_generic_fs.py | 4 ++-- .../plugins/unpacking/hp/test/test_pjl.py | 2 +- .../unpacking/hp/test/test_update_stream.py | 2 +- .../unpacking/intel_hex/test/test_intel_hex.py | 6 +++--- .../unpacking/jffs2/test/test_plugin_jffs2.py | 2 +- .../linuxkernel/test/test_plugin_linuxkernel.py | 2 +- .../plugins/unpacking/mikrotik/test/test_npk.py | 6 +++--- .../unpacking/patool/test/test_plugin_patool.py | 4 ++-- .../plugins/unpacking/raw/test/test_raw.py | 2 +- .../plugins/unpacking/ros/test/test_ros.py | 4 ++-- .../plugins/unpacking/sevenz/code/sevenz.py | 2 +- .../unpacking/sevenz/test/test_plugin_sevenz.py | 6 +++--- fact_extractor/plugins/unpacking/sfx/code/sfx.py | 6 +++++- .../plugins/unpacking/sfx/test/test_sfx.py | 2 +- .../plugins/unpacking/squashFS/code/squash_fs.py | 2 +- .../squashFS/test/test_plugin_squashfs.py | 2 +- .../plugins/unpacking/srec/test/test_srec.py | 6 +++--- .../plugins/unpacking/stuffit/test/test_sit.py | 2 +- .../unpacking/tektronix/test/test_plugin_tek.py | 6 +++--- .../unpacking/tektronix/test/test_plugin_xtek.py | 6 +++--- .../plugins/unpacking/tpl/code/TPWRN702N.py | 2 +- .../plugins/unpacking/tpl/code/tpltool.py | 2 +- .../plugins/unpacking/tpl/test/test_tp_wr702n.py | 4 ++-- .../plugins/unpacking/tpl/test/test_tpl.py | 2 +- fact_extractor/plugins/unpacking/trx/code/untrx.py | 2 +- .../unpacking/trx/test/test_plugin_untrx.py | 2 +- .../unpacking/ubi/test/test_plugin_ubi_image.py | 2 +- .../unpacking/ubi/test/test_plugin_ubifs.py | 2 +- .../plugins/unpacking/uboot/code/uboot.py | 2 +- .../unpacking/uboot/test/test_plugin_uboot.py | 2 +- .../unpacking/uefi/test/test_plugin_uefi.py | 2 +- .../plugins/unpacking/xerox/code/postscript.py | 2 +- .../unpacking/xerox/internal/dsk_container.py | 2 +- .../unpacking/xerox/test/test_container_dsk_one.py | 4 ++-- .../plugins/unpacking/xerox/test/test_dlm.py | 4 ++-- .../plugins/unpacking/xerox/test/test_dsk.py | 2 +- .../unpacking/xerox/test/test_dsk_extended.py | 2 +- .../unpacking/xerox/test/test_postscript.py | 2 +- .../plugins/unpacking/yaffs/code/yaffs.py | 2 +- .../unpacking/yaffs/test/test_plugin_yaffs.py | 2 +- .../unpacking/zlib/test/test_plugin_zlib.py | 4 ++-- fact_extractor/server.py | 8 ++++---- .../test/unit/helperFunctions/test_config.py | 10 +++++----- .../unit/helperFunctions/test_data_conversion.py | 2 +- .../test/unit/helperFunctions/test_file_system.py | 2 +- .../test/unit/helperFunctions/test_hash.py | 2 +- .../test/unit/helperFunctions/test_install.py | 2 +- .../test/unit/helperFunctions/test_plugin.py | 2 +- .../unit/helperFunctions/test_program_setup.py | 4 ++-- .../test/unit/helperFunctions/test_statistics.py | 4 ++-- .../test/unit/unpacker/test_carved_area.py | 2 +- fact_extractor/test/unit/unpacker/test_unpacker.py | 4 ++-- fact_extractor/unpacker/unpack.py | 8 ++++---- fact_extractor/unpacker/unpackBase.py | 6 +++--- pyproject.toml | 3 --- 83 files changed, 142 insertions(+), 131 deletions(-) create mode 100644 fact_extractor/__init__.py diff --git a/Dockerfile b/Dockerfile index c3f14c53..d973747a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ apt update && apt install -y \ RUN python3.11 -m venv /venv ENV PATH=/venv/bin:$PATH \ VIRTUAL_ENV=/venv \ - PYTHONPATH=/app/fact_extractor + PYTHONPATH=/app/ ADD ./fact_extractor/install/pre_install.sh /app/fact_extractor/install/pre_install.sh RUN --mount=type=cache,target=/var/cache/apt \ diff --git a/fact_extractor/__init__.py b/fact_extractor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fact_extractor/docker_extraction.py b/fact_extractor/docker_extraction.py index f38946d4..58dd260b 100755 --- a/fact_extractor/docker_extraction.py +++ b/fact_extractor/docker_extraction.py @@ -20,10 +20,10 @@ from pathlib import Path import sys -from helperFunctions.config import get_config_dir -from helperFunctions.file_system import change_owner_of_output_files -from helperFunctions.program_setup import check_ulimits, load_config, setup_logging -from unpacker.unpack import unpack +from fact_extractor.helperFunctions.config import get_config_dir +from fact_extractor.helperFunctions.file_system import change_owner_of_output_files +from fact_extractor.helperFunctions.program_setup import check_ulimits, load_config, setup_logging +from fact_extractor.unpacker.unpack import unpack def _parse_args(): diff --git a/fact_extractor/fact_extract.py b/fact_extractor/fact_extract.py index 078d57fe..86721f7e 100755 --- a/fact_extractor/fact_extract.py +++ b/fact_extractor/fact_extract.py @@ -20,8 +20,8 @@ import sys from pathlib import Path -from helperFunctions.program_setup import setup_argparser, setup_logging, load_config -from unpacker.unpack import unpack +from fact_extractor.helperFunctions.program_setup import setup_argparser, setup_logging, load_config +from fact_extractor.unpacker.unpack import unpack def main(): diff --git a/fact_extractor/helperFunctions/config.py b/fact_extractor/helperFunctions/config.py index 85be1a1a..90a8e88f 100644 --- a/fact_extractor/helperFunctions/config.py +++ b/fact_extractor/helperFunctions/config.py @@ -1,7 +1,7 @@ import os from configparser import ConfigParser, NoOptionError, NoSectionError -from helperFunctions.file_system import get_src_dir +from fact_extractor.helperFunctions.file_system import get_src_dir def load_config(config_file_name): diff --git a/fact_extractor/helperFunctions/hash.py b/fact_extractor/helperFunctions/hash.py index 184da85e..2a6a984c 100644 --- a/fact_extractor/helperFunctions/hash.py +++ b/fact_extractor/helperFunctions/hash.py @@ -1,6 +1,6 @@ from hashlib import new -from helperFunctions.dataConversion import make_bytes +from fact_extractor.helperFunctions.dataConversion import make_bytes def get_hash(hash_function, binary): diff --git a/fact_extractor/helperFunctions/plugin.py b/fact_extractor/helperFunctions/plugin.py index f2c08c69..3d9ac84a 100644 --- a/fact_extractor/helperFunctions/plugin.py +++ b/fact_extractor/helperFunctions/plugin.py @@ -4,7 +4,7 @@ from common_helper_files import get_dirs_in_dir from pluginbase import PluginBase -from helperFunctions.file_system import get_src_dir +from fact_extractor.helperFunctions.file_system import get_src_dir def import_plugins(plugin_mount, plugin_base_dir): diff --git a/fact_extractor/helperFunctions/program_setup.py b/fact_extractor/helperFunctions/program_setup.py index 994bdcec..f031e343 100644 --- a/fact_extractor/helperFunctions/program_setup.py +++ b/fact_extractor/helperFunctions/program_setup.py @@ -5,8 +5,8 @@ from common_helper_files import create_dir_for_file -from helperFunctions.config import get_config_dir -from version import __VERSION__ +from fact_extractor.helperFunctions.config import get_config_dir +from fact_extractor.version import __VERSION__ def setup_argparser(name, description, command_line_options, version=__VERSION__): @@ -17,6 +17,16 @@ def setup_argparser(name, description, command_line_options, version=__VERSION__ parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages') parser.add_argument('-C', '--config_file', help='set path to config File', default='{}/main.cfg'.format(get_config_dir())) parser.add_argument('FILE_PATH', type=str, help='Path to file that should be extracted') + parser.add_argument( + '--chown', type=str, default=None, help='change back ownership of output files to :' + ) + parser.add_argument( + '--extract_everything', + action='store_true', + default=False, + help='change the behavior of the extractor: extract also empty files', + ) + return parser.parse_args(command_line_options[1:]) diff --git a/fact_extractor/helperFunctions/statistics.py b/fact_extractor/helperFunctions/statistics.py index 1333e70d..ae865775 100644 --- a/fact_extractor/helperFunctions/statistics.py +++ b/fact_extractor/helperFunctions/statistics.py @@ -8,7 +8,7 @@ avg_entropy, get_binary_size_without_padding, is_compressed ) from fact_helper_file import get_file_type_from_path -from helperFunctions.config import read_list_from_config +from fact_extractor.helperFunctions.config import read_list_from_config def add_unpack_statistics(extraction_dir: Path, meta_data: Dict): diff --git a/fact_extractor/install.py b/fact_extractor/install.py index a19df4d6..0160fd82 100755 --- a/fact_extractor/install.py +++ b/fact_extractor/install.py @@ -27,9 +27,9 @@ try: import distro - from helperFunctions.install import OperateInDirectory - from install.common import main as common - from install.unpacker import main as unpacker + from fact_extractor.helperFunctions.install import OperateInDirectory + from fact_extractor.install.common import main as common + from fact_extractor.install.unpacker import main as unpacker except ImportError: sys.exit('Could not import install dependencies. Please (re-)run install/pre_install.sh') diff --git a/fact_extractor/install/common.py b/fact_extractor/install/common.py index 7bd026ee..60a97861 100644 --- a/fact_extractor/install/common.py +++ b/fact_extractor/install/common.py @@ -3,8 +3,8 @@ from contextlib import suppress from pathlib import Path -from helperFunctions.config import load_config -from helperFunctions.install import ( +from fact_extractor.helperFunctions.config import load_config +from fact_extractor.helperFunctions.install import ( apt_install_packages, apt_update_sources, pip_install_packages, load_requirements_file ) diff --git a/fact_extractor/install/unpacker.py b/fact_extractor/install/unpacker.py index 9304ddba..23b93a71 100644 --- a/fact_extractor/install/unpacker.py +++ b/fact_extractor/install/unpacker.py @@ -10,7 +10,7 @@ from common_helper_process import execute_shell_command_get_return_code -from helperFunctions.install import ( +from fact_extractor.helperFunctions.install import ( InstallationError, OperateInDirectory, apt_install_packages, diff --git a/fact_extractor/plugins/unpacking/ambarella/code/ambarella.py b/fact_extractor/plugins/unpacking/ambarella/code/ambarella.py index ee8e406c..4a9decc6 100644 --- a/fact_extractor/plugins/unpacking/ambarella/code/ambarella.py +++ b/fact_extractor/plugins/unpacking/ambarella/code/ambarella.py @@ -5,7 +5,7 @@ from common_helper_files import get_files_in_dir from common_helper_process import execute_shell_command -from helperFunctions.file_system import get_src_dir +from fact_extractor.helperFunctions.file_system import get_src_dir NAME = 'Ambarella' MIME_PATTERNS = ['firmware/ambarella'] diff --git a/fact_extractor/plugins/unpacking/ambarella/code/ambarella_romfs.py b/fact_extractor/plugins/unpacking/ambarella/code/ambarella_romfs.py index 57dbe3b6..0e22580d 100644 --- a/fact_extractor/plugins/unpacking/ambarella/code/ambarella_romfs.py +++ b/fact_extractor/plugins/unpacking/ambarella/code/ambarella_romfs.py @@ -2,7 +2,7 @@ from os import path from common_helper_process import execute_shell_command -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir NAME = 'Ambarella_RomFS' MIME_PATTERNS = ['filesystem/ambarella-romfs'] diff --git a/fact_extractor/plugins/unpacking/ambarella/test/test_plugin_ambarella_romfs.py b/fact_extractor/plugins/unpacking/ambarella/test/test_plugin_ambarella_romfs.py index 0d57c130..46f3452b 100644 --- a/fact_extractor/plugins/unpacking/ambarella/test/test_plugin_ambarella_romfs.py +++ b/fact_extractor/plugins/unpacking/ambarella/test/test_plugin_ambarella_romfs.py @@ -1,5 +1,5 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/android_simg/test/test_plugin_android_simg.py b/fact_extractor/plugins/unpacking/android_simg/test/test_plugin_android_simg.py index 27277e29..0606b54d 100644 --- a/fact_extractor/plugins/unpacking/android_simg/test/test_plugin_android_simg.py +++ b/fact_extractor/plugins/unpacking/android_simg/test/test_plugin_android_simg.py @@ -6,7 +6,7 @@ print(root_dir, [str(d) for d in root_dir.iterdir() if d.is_dir()]) sys.path.append(str(root_dir)) -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/arj/test/test_arj.py b/fact_extractor/plugins/unpacking/arj/test/test_arj.py index 38f7223a..3312ebc3 100644 --- a/fact_extractor/plugins/unpacking/arj/test/test_arj.py +++ b/fact_extractor/plugins/unpacking/arj/test/test_arj.py @@ -1,6 +1,6 @@ from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_FILE = Path(__file__).parent / 'data' / 'test.arj' diff --git a/fact_extractor/plugins/unpacking/ascii85/test/test_plugin_adobe.py b/fact_extractor/plugins/unpacking/ascii85/test/test_plugin_adobe.py index 4d0b5598..e8038d81 100755 --- a/fact_extractor/plugins/unpacking/ascii85/test/test_plugin_adobe.py +++ b/fact_extractor/plugins/unpacking/ascii85/test/test_plugin_adobe.py @@ -3,8 +3,8 @@ from tempfile import TemporaryDirectory from unittest.mock import patch -from plugins.unpacking.ascii85.code.adobe import unpack_function -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.plugins.unpacking.ascii85.code.adobe import unpack_function +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/avm_kernel_image/code/avm_kernel_image.py b/fact_extractor/plugins/unpacking/avm_kernel_image/code/avm_kernel_image.py index 64fde16f..7bc49496 100644 --- a/fact_extractor/plugins/unpacking/avm_kernel_image/code/avm_kernel_image.py +++ b/fact_extractor/plugins/unpacking/avm_kernel_image/code/avm_kernel_image.py @@ -1,6 +1,6 @@ from common_helper_process.fail_safe_subprocess import execute_shell_command from pathlib import Path -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir NAME = 'avm_kernel_image' diff --git a/fact_extractor/plugins/unpacking/avm_kernel_image/test/test_avm_kernel.py b/fact_extractor/plugins/unpacking/avm_kernel_image/test/test_avm_kernel.py index 134f9bf8..0757b369 100644 --- a/fact_extractor/plugins/unpacking/avm_kernel_image/test/test_avm_kernel.py +++ b/fact_extractor/plugins/unpacking/avm_kernel_image/test/test_avm_kernel.py @@ -1,4 +1,4 @@ -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.avm_kernel_image import FIND_SQUASHFS_TOOL_PATH, UNPACK_KERNEL_TOOL_PATH diff --git a/fact_extractor/plugins/unpacking/avm_sqfs_fake/test/test_plugin.py b/fact_extractor/plugins/unpacking/avm_sqfs_fake/test/test_plugin.py index fa4aa56a..7e1875de 100644 --- a/fact_extractor/plugins/unpacking/avm_sqfs_fake/test/test_plugin.py +++ b/fact_extractor/plugins/unpacking/avm_sqfs_fake/test/test_plugin.py @@ -1,7 +1,7 @@ import os from common_helper_files import get_binary_from_file -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/boschtool/test/test_boschtool.py b/fact_extractor/plugins/unpacking/boschtool/test/test_boschtool.py index caede32d..e1ab8cd5 100644 --- a/fact_extractor/plugins/unpacking/boschtool/test/test_boschtool.py +++ b/fact_extractor/plugins/unpacking/boschtool/test/test_boschtool.py @@ -1,8 +1,8 @@ from pathlib import Path from subprocess import check_output -from plugins.unpacking.boschtool.code.boschtool import TOOL_PATH -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.plugins.unpacking.boschtool.code.boschtool import TOOL_PATH +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_FILE = Path(__file__).parent / 'data' / 'test.fw' diff --git a/fact_extractor/plugins/unpacking/dahua/test/test_dahua.py b/fact_extractor/plugins/unpacking/dahua/test/test_dahua.py index 553752de..658bcfaa 100644 --- a/fact_extractor/plugins/unpacking/dahua/test/test_dahua.py +++ b/fact_extractor/plugins/unpacking/dahua/test/test_dahua.py @@ -1,6 +1,6 @@ from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(Path(__file__).parent, 'data') diff --git a/fact_extractor/plugins/unpacking/dji/code/dji_drones.py b/fact_extractor/plugins/unpacking/dji/code/dji_drones.py index 534f64f2..6c8660e0 100644 --- a/fact_extractor/plugins/unpacking/dji/code/dji_drones.py +++ b/fact_extractor/plugins/unpacking/dji/code/dji_drones.py @@ -4,7 +4,7 @@ from common_helper_files import delete_file, get_files_in_dir from common_helper_process import execute_shell_command -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir NAME = 'DJI_drones' MIME_PATTERNS = ['firmware/dji-drone'] diff --git a/fact_extractor/plugins/unpacking/dji/test/test_plugin_dji_drones.py b/fact_extractor/plugins/unpacking/dji/test/test_plugin_dji_drones.py index 3616b7ae..0afe5ce3 100644 --- a/fact_extractor/plugins/unpacking/dji/test/test_plugin_dji_drones.py +++ b/fact_extractor/plugins/unpacking/dji/test/test_plugin_dji_drones.py @@ -1,5 +1,5 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/dlink_shrs/test/test_dlink_shrs.py b/fact_extractor/plugins/unpacking/dlink_shrs/test/test_dlink_shrs.py index 2641e0b5..e67064b7 100644 --- a/fact_extractor/plugins/unpacking/dlink_shrs/test/test_dlink_shrs.py +++ b/fact_extractor/plugins/unpacking/dlink_shrs/test/test_dlink_shrs.py @@ -1,5 +1,5 @@ from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/draytek_vigor_167/test/test_draytek_vigor_167.py b/fact_extractor/plugins/unpacking/draytek_vigor_167/test/test_draytek_vigor_167.py index 65e5e676..d4bbb825 100644 --- a/fact_extractor/plugins/unpacking/draytek_vigor_167/test/test_draytek_vigor_167.py +++ b/fact_extractor/plugins/unpacking/draytek_vigor_167/test/test_draytek_vigor_167.py @@ -1,7 +1,7 @@ from pathlib import Path -from helperFunctions.hash import get_sha256 +from fact_extractor.helperFunctions.hash import get_sha256 from common_helper_files import get_binary_from_file -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/generic_carver/test/test_plugin_generic_carver_binwalk.py b/fact_extractor/plugins/unpacking/generic_carver/test/test_plugin_generic_carver_binwalk.py index d21bc4b6..b1852d56 100644 --- a/fact_extractor/plugins/unpacking/generic_carver/test/test_plugin_generic_carver_binwalk.py +++ b/fact_extractor/plugins/unpacking/generic_carver/test/test_plugin_generic_carver_binwalk.py @@ -5,9 +5,9 @@ import pytest from contextlib import contextmanager -from plugins.unpacking.generic_carver.code.generic_carver import ArchivesFilter -from test.unit.unpacker.test_unpacker import TestUnpackerBase -from helperFunctions.file_system import get_test_data_dir +from fact_extractor.plugins.unpacking.generic_carver.code.generic_carver import ArchivesFilter +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir # pylint: disable=protected-access diff --git a/fact_extractor/plugins/unpacking/generic_fs/test/test_plugin_generic_fs.py b/fact_extractor/plugins/unpacking/generic_fs/test/test_plugin_generic_fs.py index 28441e45..d9302e99 100644 --- a/fact_extractor/plugins/unpacking/generic_fs/test/test_plugin_generic_fs.py +++ b/fact_extractor/plugins/unpacking/generic_fs/test/test_plugin_generic_fs.py @@ -1,8 +1,8 @@ from pathlib import Path from tempfile import TemporaryDirectory -from helperFunctions.file_system import decompress_test_file -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import decompress_test_file +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.generic_fs import _extract_loop_devices, _mount_single_filesystem, TYPES TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/hp/test/test_pjl.py b/fact_extractor/plugins/unpacking/hp/test/test_pjl.py index 0ec208e3..879b2d07 100644 --- a/fact_extractor/plugins/unpacking/hp/test/test_pjl.py +++ b/fact_extractor/plugins/unpacking/hp/test/test_pjl.py @@ -1,5 +1,5 @@ from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from common_helper_files import get_binary_from_file diff --git a/fact_extractor/plugins/unpacking/hp/test/test_update_stream.py b/fact_extractor/plugins/unpacking/hp/test/test_update_stream.py index d703b245..ff513946 100644 --- a/fact_extractor/plugins/unpacking/hp/test/test_update_stream.py +++ b/fact_extractor/plugins/unpacking/hp/test/test_update_stream.py @@ -1,5 +1,5 @@ from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from common_helper_files.fail_safe_file_operations import get_binary_from_file diff --git a/fact_extractor/plugins/unpacking/intel_hex/test/test_intel_hex.py b/fact_extractor/plugins/unpacking/intel_hex/test/test_intel_hex.py index e2ab3e78..85e3eaac 100644 --- a/fact_extractor/plugins/unpacking/intel_hex/test/test_intel_hex.py +++ b/fact_extractor/plugins/unpacking/intel_hex/test/test_intel_hex.py @@ -5,9 +5,9 @@ from intelhex import IntelHexError -from helperFunctions.file_system import get_test_data_dir -from plugins.unpacking.intel_hex.code.intel_hex import unpack_function -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.plugins.unpacking.intel_hex.code.intel_hex import unpack_function +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/jffs2/test/test_plugin_jffs2.py b/fact_extractor/plugins/unpacking/jffs2/test/test_plugin_jffs2.py index cb8c7985..1458adf6 100644 --- a/fact_extractor/plugins/unpacking/jffs2/test/test_plugin_jffs2.py +++ b/fact_extractor/plugins/unpacking/jffs2/test/test_plugin_jffs2.py @@ -1,6 +1,6 @@ from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/linuxkernel/test/test_plugin_linuxkernel.py b/fact_extractor/plugins/unpacking/linuxkernel/test/test_plugin_linuxkernel.py index 74c4c0cb..d3c7f784 100644 --- a/fact_extractor/plugins/unpacking/linuxkernel/test/test_plugin_linuxkernel.py +++ b/fact_extractor/plugins/unpacking/linuxkernel/test/test_plugin_linuxkernel.py @@ -2,7 +2,7 @@ import pytest -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/mikrotik/test/test_npk.py b/fact_extractor/plugins/unpacking/mikrotik/test/test_npk.py index 230fa598..c5ac622f 100644 --- a/fact_extractor/plugins/unpacking/mikrotik/test/test_npk.py +++ b/fact_extractor/plugins/unpacking/mikrotik/test/test_npk.py @@ -2,9 +2,9 @@ from pathlib import Path from tempfile import TemporaryDirectory -from helperFunctions.file_system import get_test_data_dir -from plugins.unpacking.mikrotik.code.npk import unpack_function -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.plugins.unpacking.mikrotik.code.npk import unpack_function +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py b/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py index 5600db7e..712a7aea 100644 --- a/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py +++ b/fact_extractor/plugins/unpacking/patool/test/test_plugin_patool.py @@ -3,8 +3,8 @@ import pytest -from helperFunctions.hash import get_sha256 -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.hash import get_sha256 +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/raw/test/test_raw.py b/fact_extractor/plugins/unpacking/raw/test/test_raw.py index c1d1932b..3462cb18 100644 --- a/fact_extractor/plugins/unpacking/raw/test/test_raw.py +++ b/fact_extractor/plugins/unpacking/raw/test/test_raw.py @@ -1,6 +1,6 @@ # pylint: disable=protected-access from pathlib import Path -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(Path(__file__).parent, 'data') diff --git a/fact_extractor/plugins/unpacking/ros/test/test_ros.py b/fact_extractor/plugins/unpacking/ros/test/test_ros.py index 3684f15f..33386524 100644 --- a/fact_extractor/plugins/unpacking/ros/test/test_ros.py +++ b/fact_extractor/plugins/unpacking/ros/test/test_ros.py @@ -3,8 +3,8 @@ import pytest -from helperFunctions.file_system import get_test_data_dir -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.ros import infer_header_size_from_version, infer_endianness_from_file_count, unpack_function TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py b/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py index 87fd7753..a89c9e6e 100644 --- a/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py +++ b/fact_extractor/plugins/unpacking/sevenz/code/sevenz.py @@ -7,7 +7,7 @@ from common_helper_passwords import get_merged_password_set from common_helper_process import execute_shell_command -from helperFunctions.file_system import get_src_dir +from fact_extractor.helperFunctions.file_system import get_src_dir NAME = '7z' MIME_PATTERNS = [ diff --git a/fact_extractor/plugins/unpacking/sevenz/test/test_plugin_sevenz.py b/fact_extractor/plugins/unpacking/sevenz/test/test_plugin_sevenz.py index c1b391ae..c378946a 100644 --- a/fact_extractor/plugins/unpacking/sevenz/test/test_plugin_sevenz.py +++ b/fact_extractor/plugins/unpacking/sevenz/test/test_plugin_sevenz.py @@ -2,9 +2,9 @@ import pytest -from helperFunctions.file_system import decompress_test_file -from plugins.unpacking.sevenz.code.sevenz import MIME_PATTERNS -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import decompress_test_file +from fact_extractor.plugins.unpacking.sevenz.code.sevenz import MIME_PATTERNS +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/sfx/code/sfx.py b/fact_extractor/plugins/unpacking/sfx/code/sfx.py index c3b0db43..7ada3170 100644 --- a/fact_extractor/plugins/unpacking/sfx/code/sfx.py +++ b/fact_extractor/plugins/unpacking/sfx/code/sfx.py @@ -1,6 +1,10 @@ from pathlib import Path -from plugins.unpacking.sevenz.code.sevenz import unpack_function as sevenz +import fact_extractor.plugins.unpacking.sevenz +import fact_extractor +# This is not an actual import due to wired pluginbase shenanigans. +# The 'sevenz' module is actually the 'code.sevenz' module. +from fact_extractor.plugins.unpacking.sevenz import unpack_function as sevenz NAME = 'SFX' MIME_PATTERNS = ['application/x-executable', 'application/x-dosexec'] diff --git a/fact_extractor/plugins/unpacking/sfx/test/test_sfx.py b/fact_extractor/plugins/unpacking/sfx/test/test_sfx.py index abb197a0..5a1f9315 100644 --- a/fact_extractor/plugins/unpacking/sfx/test/test_sfx.py +++ b/fact_extractor/plugins/unpacking/sfx/test/test_sfx.py @@ -1,5 +1,5 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py b/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py index 1286b212..a0e06103 100755 --- a/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py +++ b/fact_extractor/plugins/unpacking/squashFS/code/squash_fs.py @@ -5,7 +5,7 @@ from common_helper_process import execute_shell_command from pathlib import Path -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir SASQUATCH = Path('/usr/bin/sasquatch') SASQUATCH_BE = Path('/usr/bin/sasquatch-v4be') diff --git a/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py b/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py index 111c5cae..496a87a9 100755 --- a/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py +++ b/fact_extractor/plugins/unpacking/squashFS/test/test_plugin_squashfs.py @@ -3,7 +3,7 @@ import pytest from tempfile import TemporaryDirectory -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.squash_fs import _unpack_success, unpack_function, SQUASH_UNPACKER diff --git a/fact_extractor/plugins/unpacking/srec/test/test_srec.py b/fact_extractor/plugins/unpacking/srec/test/test_srec.py index de5bd7d4..e63a6856 100755 --- a/fact_extractor/plugins/unpacking/srec/test/test_srec.py +++ b/fact_extractor/plugins/unpacking/srec/test/test_srec.py @@ -2,9 +2,9 @@ from tempfile import TemporaryDirectory from unittest.mock import patch -from helperFunctions.file_system import get_test_data_dir -from plugins.unpacking.srec.code.srec import unpack_function -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.plugins.unpacking.srec.code.srec import unpack_function +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = Path(__file__).parent / 'data' diff --git a/fact_extractor/plugins/unpacking/stuffit/test/test_sit.py b/fact_extractor/plugins/unpacking/stuffit/test/test_sit.py index 580be8ca..fcc8e148 100644 --- a/fact_extractor/plugins/unpacking/stuffit/test/test_sit.py +++ b/fact_extractor/plugins/unpacking/stuffit/test/test_sit.py @@ -1,5 +1,5 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_tek.py b/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_tek.py index 7cf73ee7..61dcc276 100755 --- a/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_tek.py +++ b/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_tek.py @@ -5,9 +5,9 @@ from binascii import Error -from helperFunctions.file_system import get_test_data_dir -from plugins.unpacking.tektronix.code.tek import unpack_function -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.plugins.unpacking.tektronix.code.tek import unpack_function +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_xtek.py b/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_xtek.py index b5b66ac0..33a1fb25 100755 --- a/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_xtek.py +++ b/fact_extractor/plugins/unpacking/tektronix/test/test_plugin_xtek.py @@ -5,9 +5,9 @@ from binascii import Error -from helperFunctions.file_system import get_test_data_dir -from plugins.unpacking.tektronix.code.xtek import unpack_function -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.plugins.unpacking.tektronix.code.xtek import unpack_function +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/tpl/code/TPWRN702N.py b/fact_extractor/plugins/unpacking/tpl/code/TPWRN702N.py index adfe276b..0396a2a0 100644 --- a/fact_extractor/plugins/unpacking/tpl/code/TPWRN702N.py +++ b/fact_extractor/plugins/unpacking/tpl/code/TPWRN702N.py @@ -2,7 +2,7 @@ from struct import unpack from common_helper_files import write_binary_to_file -from unpacker.helper.carving import Carver +from fact_extractor.unpacker.helper.carving import Carver NAME = 'TP-WR702N' MIME_PATTERNS = ['firmware/tp-wr702n'] diff --git a/fact_extractor/plugins/unpacking/tpl/code/tpltool.py b/fact_extractor/plugins/unpacking/tpl/code/tpltool.py index fafcc9f2..c4ac8525 100644 --- a/fact_extractor/plugins/unpacking/tpl/code/tpltool.py +++ b/fact_extractor/plugins/unpacking/tpl/code/tpltool.py @@ -2,7 +2,7 @@ from shutil import copyfile from common_helper_process import execute_shell_command -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir NAME = 'tpl-tool' MIME_PATTERNS = ['firmware/tp-link'] diff --git a/fact_extractor/plugins/unpacking/tpl/test/test_tp_wr702n.py b/fact_extractor/plugins/unpacking/tpl/test/test_tp_wr702n.py index 98d18c85..c718111a 100644 --- a/fact_extractor/plugins/unpacking/tpl/test/test_tp_wr702n.py +++ b/fact_extractor/plugins/unpacking/tpl/test/test_tp_wr702n.py @@ -4,8 +4,8 @@ from common_helper_files import get_binary_from_file -from helperFunctions.hash import get_sha256 -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.hash import get_sha256 +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.TPWRN702N import TPWR702N TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/tpl/test/test_tpl.py b/fact_extractor/plugins/unpacking/tpl/test/test_tpl.py index e6d2a2e0..864848c6 100644 --- a/fact_extractor/plugins/unpacking/tpl/test/test_tpl.py +++ b/fact_extractor/plugins/unpacking/tpl/test/test_tpl.py @@ -1,6 +1,6 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/trx/code/untrx.py b/fact_extractor/plugins/unpacking/trx/code/untrx.py index 3ebea59c..a13d4f98 100644 --- a/fact_extractor/plugins/unpacking/trx/code/untrx.py +++ b/fact_extractor/plugins/unpacking/trx/code/untrx.py @@ -2,7 +2,7 @@ from tempfile import NamedTemporaryFile from common_helper_process.fail_safe_subprocess import execute_shell_command -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir NAME = 'untrx' MIME_PATTERNS = ['firmware/trx'] diff --git a/fact_extractor/plugins/unpacking/trx/test/test_plugin_untrx.py b/fact_extractor/plugins/unpacking/trx/test/test_plugin_untrx.py index 1c197064..8b4adccc 100644 --- a/fact_extractor/plugins/unpacking/trx/test/test_plugin_untrx.py +++ b/fact_extractor/plugins/unpacking/trx/test/test_plugin_untrx.py @@ -1,7 +1,7 @@ import os import zipfile -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubi_image.py b/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubi_image.py index b2638104..74333779 100644 --- a/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubi_image.py +++ b/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubi_image.py @@ -1,6 +1,6 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubifs.py b/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubifs.py index 2f3c2b40..2133b9e0 100644 --- a/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubifs.py +++ b/fact_extractor/plugins/unpacking/ubi/test/test_plugin_ubifs.py @@ -1,6 +1,6 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/uboot/code/uboot.py b/fact_extractor/plugins/unpacking/uboot/code/uboot.py index afaf9604..b4ca948e 100644 --- a/fact_extractor/plugins/unpacking/uboot/code/uboot.py +++ b/fact_extractor/plugins/unpacking/uboot/code/uboot.py @@ -5,7 +5,7 @@ from common_helper_process import execute_shell_command -from unpacker.helper.carving import Carver +from fact_extractor.unpacker.helper.carving import Carver THIS_FILE = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(THIS_FILE, '..', 'internal')) diff --git a/fact_extractor/plugins/unpacking/uboot/test/test_plugin_uboot.py b/fact_extractor/plugins/unpacking/uboot/test/test_plugin_uboot.py index 6da2e209..10336b4c 100644 --- a/fact_extractor/plugins/unpacking/uboot/test/test_plugin_uboot.py +++ b/fact_extractor/plugins/unpacking/uboot/test/test_plugin_uboot.py @@ -1,6 +1,6 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/uefi/test/test_plugin_uefi.py b/fact_extractor/plugins/unpacking/uefi/test/test_plugin_uefi.py index 359b1b84..84c945fb 100644 --- a/fact_extractor/plugins/unpacking/uefi/test/test_plugin_uefi.py +++ b/fact_extractor/plugins/unpacking/uefi/test/test_plugin_uefi.py @@ -1,5 +1,5 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/xerox/code/postscript.py b/fact_extractor/plugins/unpacking/xerox/code/postscript.py index 682c213f..05bdfd76 100644 --- a/fact_extractor/plugins/unpacking/xerox/code/postscript.py +++ b/fact_extractor/plugins/unpacking/xerox/code/postscript.py @@ -5,7 +5,7 @@ from base64 import a85decode from common_helper_files import get_binary_from_file, write_binary_to_file -from helperFunctions.dataConversion import ( +from fact_extractor.helperFunctions.dataConversion import ( make_bytes, make_unicode_string, remove_uneccessary_spaces ) diff --git a/fact_extractor/plugins/unpacking/xerox/internal/dsk_container.py b/fact_extractor/plugins/unpacking/xerox/internal/dsk_container.py index d7eb4a5b..801b8841 100644 --- a/fact_extractor/plugins/unpacking/xerox/internal/dsk_container.py +++ b/fact_extractor/plugins/unpacking/xerox/internal/dsk_container.py @@ -5,7 +5,7 @@ from struct import unpack import sys -from helperFunctions.dataConversion import make_unicode_string +from fact_extractor.helperFunctions.dataConversion import make_unicode_string ENCODING_OVERHEAD = 0.33 diff --git a/fact_extractor/plugins/unpacking/xerox/test/test_container_dsk_one.py b/fact_extractor/plugins/unpacking/xerox/test/test_container_dsk_one.py index c2346441..9227b42d 100644 --- a/fact_extractor/plugins/unpacking/xerox/test/test_container_dsk_one.py +++ b/fact_extractor/plugins/unpacking/xerox/test/test_container_dsk_one.py @@ -1,10 +1,10 @@ import os import gc -from helperFunctions.hash import get_sha256 +from fact_extractor.helperFunctions.hash import get_sha256 from ..internal.dsk_container import DskOne -from plugins.unpacking.xerox.internal.dsk_container import ExtendedDskOne +from fact_extractor.plugins.unpacking.xerox.internal.dsk_container import ExtendedDskOne TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/xerox/test/test_dlm.py b/fact_extractor/plugins/unpacking/xerox/test/test_dlm.py index b653c2fd..2149b11b 100644 --- a/fact_extractor/plugins/unpacking/xerox/test/test_dlm.py +++ b/fact_extractor/plugins/unpacking/xerox/test/test_dlm.py @@ -4,8 +4,8 @@ from common_helper_files import get_binary_from_file -from helperFunctions.hash import get_sha256 -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.hash import get_sha256 +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.dlm import XeroxDLM TEST_DATA_DIR = Path(Path(__file__).parent, 'data') diff --git a/fact_extractor/plugins/unpacking/xerox/test/test_dsk.py b/fact_extractor/plugins/unpacking/xerox/test/test_dsk.py index 0463cf7c..89a7fd8b 100644 --- a/fact_extractor/plugins/unpacking/xerox/test/test_dsk.py +++ b/fact_extractor/plugins/unpacking/xerox/test/test_dsk.py @@ -1,6 +1,6 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/xerox/test/test_dsk_extended.py b/fact_extractor/plugins/unpacking/xerox/test/test_dsk_extended.py index 6faa5b3a..5328f398 100644 --- a/fact_extractor/plugins/unpacking/xerox/test/test_dsk_extended.py +++ b/fact_extractor/plugins/unpacking/xerox/test/test_dsk_extended.py @@ -1,6 +1,6 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/xerox/test/test_postscript.py b/fact_extractor/plugins/unpacking/xerox/test/test_postscript.py index 66b23256..c3b5a255 100644 --- a/fact_extractor/plugins/unpacking/xerox/test/test_postscript.py +++ b/fact_extractor/plugins/unpacking/xerox/test/test_postscript.py @@ -2,7 +2,7 @@ from common_helper_files import get_binary_from_file -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase from ..code.postscript import _get_raw_payloads, _convert_payloads TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/yaffs/code/yaffs.py b/fact_extractor/plugins/unpacking/yaffs/code/yaffs.py index b742b454..cc25cb2f 100644 --- a/fact_extractor/plugins/unpacking/yaffs/code/yaffs.py +++ b/fact_extractor/plugins/unpacking/yaffs/code/yaffs.py @@ -1,7 +1,7 @@ from os import path from common_helper_process import execute_shell_command -from helperFunctions.file_system import get_fact_bin_dir +from fact_extractor.helperFunctions.file_system import get_fact_bin_dir NAME = 'YAFFS' MIME_PATTERNS = ['filesystem/yaffs'] diff --git a/fact_extractor/plugins/unpacking/yaffs/test/test_plugin_yaffs.py b/fact_extractor/plugins/unpacking/yaffs/test/test_plugin_yaffs.py index 22316686..586f242b 100644 --- a/fact_extractor/plugins/unpacking/yaffs/test/test_plugin_yaffs.py +++ b/fact_extractor/plugins/unpacking/yaffs/test/test_plugin_yaffs.py @@ -1,5 +1,5 @@ import os -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/plugins/unpacking/zlib/test/test_plugin_zlib.py b/fact_extractor/plugins/unpacking/zlib/test/test_plugin_zlib.py index 6659ddd8..2c3eb3df 100644 --- a/fact_extractor/plugins/unpacking/zlib/test/test_plugin_zlib.py +++ b/fact_extractor/plugins/unpacking/zlib/test/test_plugin_zlib.py @@ -2,8 +2,8 @@ from common_helper_files import get_binary_from_file -from helperFunctions.hash import get_sha256 -from test.unit.unpacker.test_unpacker import TestUnpackerBase +from fact_extractor.helperFunctions.hash import get_sha256 +from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') diff --git a/fact_extractor/server.py b/fact_extractor/server.py index a0625c50..d4363169 100644 --- a/fact_extractor/server.py +++ b/fact_extractor/server.py @@ -6,10 +6,10 @@ from flask import Flask from flask_restful import Api, Resource -from helperFunctions.config import load_config -from helperFunctions.file_system import change_owner_of_output_files -from helperFunctions.program_setup import setup_logging -from unpacker.unpack import unpack +from fact_extractor.helperFunctions.config import load_config +from fact_extractor.helperFunctions.file_system import change_owner_of_output_files +from fact_extractor.helperFunctions.program_setup import setup_logging +from fact_extractor.unpacker.unpack import unpack app = Flask(__name__) api = Api(app) diff --git a/fact_extractor/test/unit/helperFunctions/test_config.py b/fact_extractor/test/unit/helperFunctions/test_config.py index 7bbe7503..59f0d71d 100644 --- a/fact_extractor/test/unit/helperFunctions/test_config.py +++ b/fact_extractor/test/unit/helperFunctions/test_config.py @@ -2,8 +2,8 @@ import pytest -from helperFunctions.file_system import get_test_data_dir -from helperFunctions.config import get_config_dir, load_config, read_list_from_config +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.helperFunctions.config import get_config_dir, load_config, read_list_from_config def test_get_config_dir(): @@ -11,7 +11,7 @@ def test_get_config_dir(): def test_load_config(monkeypatch): - monkeypatch.setattr('helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) + monkeypatch.setattr('fact_extractor.helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) test_config = load_config('test.cfg') assert test_config['test']['test'] == 'test_config', 'config not correct' @@ -24,7 +24,7 @@ def test_load_config(monkeypatch): (' item1 , item2 , item3 ', ['item1', 'item2', 'item3']), ]) def test_read_list_from_config(monkeypatch, input_data, expected): - monkeypatch.setattr('helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) + monkeypatch.setattr('fact_extractor.helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) test_config = load_config('test.cfg') test_config.add_section('test_section') test_config.set('test_section', 'test_option', input_data) @@ -33,7 +33,7 @@ def test_read_list_from_config(monkeypatch, input_data, expected): def test_read_list_from_config__key_not_in_config(monkeypatch): - monkeypatch.setattr('helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) + monkeypatch.setattr('fact_extractor.helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) test_config = load_config('test.cfg') result = read_list_from_config(test_config, 'foo', 'bar') assert result == [] diff --git a/fact_extractor/test/unit/helperFunctions/test_data_conversion.py b/fact_extractor/test/unit/helperFunctions/test_data_conversion.py index d7e1b84f..55a25d54 100644 --- a/fact_extractor/test/unit/helperFunctions/test_data_conversion.py +++ b/fact_extractor/test/unit/helperFunctions/test_data_conversion.py @@ -2,7 +2,7 @@ import pytest -from helperFunctions.dataConversion import make_bytes, make_unicode_string, remove_uneccessary_spaces, ReportEncoder +from fact_extractor.helperFunctions.dataConversion import make_bytes, make_unicode_string, remove_uneccessary_spaces, ReportEncoder @pytest.mark.parametrize('input_data', [ diff --git a/fact_extractor/test/unit/helperFunctions/test_file_system.py b/fact_extractor/test/unit/helperFunctions/test_file_system.py index 9435a03f..99a787e1 100644 --- a/fact_extractor/test/unit/helperFunctions/test_file_system.py +++ b/fact_extractor/test/unit/helperFunctions/test_file_system.py @@ -3,7 +3,7 @@ import pytest from common_helper_files import get_files_in_dir -from helperFunctions.file_system import ( +from fact_extractor.helperFunctions.file_system import ( file_is_empty, get_fact_bin_dir, get_src_dir, diff --git a/fact_extractor/test/unit/helperFunctions/test_hash.py b/fact_extractor/test/unit/helperFunctions/test_hash.py index ba0e8ba3..92ff3c84 100644 --- a/fact_extractor/test/unit/helperFunctions/test_hash.py +++ b/fact_extractor/test/unit/helperFunctions/test_hash.py @@ -3,7 +3,7 @@ @author: weidenba ''' -from helperFunctions.hash import get_sha256 +from fact_extractor.helperFunctions.hash import get_sha256 class TestHashGeneration: diff --git a/fact_extractor/test/unit/helperFunctions/test_install.py b/fact_extractor/test/unit/helperFunctions/test_install.py index 4057dc49..fdb7b850 100644 --- a/fact_extractor/test/unit/helperFunctions/test_install.py +++ b/fact_extractor/test/unit/helperFunctions/test_install.py @@ -1,6 +1,6 @@ import pytest -from helperFunctions.install import InstallationError, run_shell_command_raise_on_return_code +from fact_extractor.helperFunctions.install import InstallationError, run_shell_command_raise_on_return_code def _patch_shell_command(patch, mock_output: str, mock_return_code: int): diff --git a/fact_extractor/test/unit/helperFunctions/test_plugin.py b/fact_extractor/test/unit/helperFunctions/test_plugin.py index 19de25ef..bdc00e3b 100644 --- a/fact_extractor/test/unit/helperFunctions/test_plugin.py +++ b/fact_extractor/test/unit/helperFunctions/test_plugin.py @@ -1,4 +1,4 @@ -from helperFunctions.plugin import _get_plugin_src_dirs, import_plugins +from fact_extractor.helperFunctions.plugin import _get_plugin_src_dirs, import_plugins TEST_PLUGINS_BASE_PATH = 'test/data/plugin_system' diff --git a/fact_extractor/test/unit/helperFunctions/test_program_setup.py b/fact_extractor/test/unit/helperFunctions/test_program_setup.py index 8a8e3dae..e897d45e 100644 --- a/fact_extractor/test/unit/helperFunctions/test_program_setup.py +++ b/fact_extractor/test/unit/helperFunctions/test_program_setup.py @@ -1,8 +1,8 @@ import logging from pathlib import Path -from helperFunctions.config import get_config_dir -from helperFunctions.program_setup import ( +from fact_extractor.helperFunctions.config import get_config_dir +from fact_extractor.helperFunctions.program_setup import ( load_config, setup_logging, setup_argparser ) diff --git a/fact_extractor/test/unit/helperFunctions/test_statistics.py b/fact_extractor/test/unit/helperFunctions/test_statistics.py index facc5009..e6b47202 100644 --- a/fact_extractor/test/unit/helperFunctions/test_statistics.py +++ b/fact_extractor/test/unit/helperFunctions/test_statistics.py @@ -3,8 +3,8 @@ import pytest -from helperFunctions.file_system import get_test_data_dir -from helperFunctions.statistics import get_unpack_status, _detect_unpack_loss +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.helperFunctions.statistics import get_unpack_status, _detect_unpack_loss @pytest.fixture(scope='function') diff --git a/fact_extractor/test/unit/unpacker/test_carved_area.py b/fact_extractor/test/unit/unpacker/test_carved_area.py index 09d535f4..a8d46956 100644 --- a/fact_extractor/test/unit/unpacker/test_carved_area.py +++ b/fact_extractor/test/unit/unpacker/test_carved_area.py @@ -1,6 +1,6 @@ import gc -from unpacker.helper.carving import CarvedArea +from fact_extractor.unpacker.helper.carving import CarvedArea class TestCarvedArea: diff --git a/fact_extractor/test/unit/unpacker/test_unpacker.py b/fact_extractor/test/unit/unpacker/test_unpacker.py index 9c9506ed..0e81d416 100644 --- a/fact_extractor/test/unit/unpacker/test_unpacker.py +++ b/fact_extractor/test/unit/unpacker/test_unpacker.py @@ -11,8 +11,8 @@ from tempfile import TemporaryDirectory from unittest.mock import Mock, patch -from helperFunctions.file_system import get_test_data_dir -from unpacker.unpack import Unpacker +from fact_extractor.helperFunctions.file_system import get_test_data_dir +from fact_extractor.unpacker.unpack import Unpacker class TestUnpackerBase: diff --git a/fact_extractor/unpacker/unpack.py b/fact_extractor/unpacker/unpack.py index 98fb3901..aee0edc9 100644 --- a/fact_extractor/unpacker/unpack.py +++ b/fact_extractor/unpacker/unpack.py @@ -7,10 +7,10 @@ from tempfile import TemporaryDirectory from typing import List, Dict, Tuple -from helperFunctions.dataConversion import ReportEncoder -from helperFunctions.file_system import file_is_empty -from helperFunctions.statistics import get_unpack_status, add_unpack_statistics -from unpacker.unpackBase import UnpackBase +from fact_extractor.helperFunctions.dataConversion import ReportEncoder +from fact_extractor.helperFunctions.file_system import file_is_empty +from fact_extractor.helperFunctions.statistics import get_unpack_status, add_unpack_statistics +from fact_extractor.unpacker.unpackBase import UnpackBase class Unpacker(UnpackBase): diff --git a/fact_extractor/unpacker/unpackBase.py b/fact_extractor/unpacker/unpackBase.py index fb0d629f..59d9cc20 100644 --- a/fact_extractor/unpacker/unpackBase.py +++ b/fact_extractor/unpacker/unpackBase.py @@ -7,8 +7,8 @@ from common_helper_files import get_files_in_dir from fact_helper_file import get_file_type_from_path -from helperFunctions.config import read_list_from_config -from helperFunctions.plugin import import_plugins +from fact_extractor.helperFunctions.config import read_list_from_config +from fact_extractor.helperFunctions.plugin import import_plugins class UnpackBase(object): @@ -29,7 +29,7 @@ def _setup_plugins(self): self._set_whitelist() def load_plugins(self): - self.source = import_plugins('unpacker.plugins', 'plugins/unpacking') + self.source = import_plugins('fact_extractor.plugins.unpacking', 'plugins/unpacking') for plugin_name in self.source.list_plugins(): plugin = self.source.load_plugin(plugin_name) plugin.setup(self) diff --git a/pyproject.toml b/pyproject.toml index 748a318a..3ab2348b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,9 +56,6 @@ fixable = ["ALL"] "test*.py" = ["ARG002"] "conftest.py" = ["ARG002"] -[tool.ruff.lint.isort] -known-first-party = ["config", "helperFunctions", "install", "plugins", "test", "unpacker", "version"] - [tool.ruff.lint.pylint] max-args=7 max-public-methods = 40 From 000abb1adc27757994aecce11ed95d44bb87d41e Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Thu, 12 Sep 2024 15:15:57 +0200 Subject: [PATCH 2/6] chore: Remove unhelpful error message Not telling the user which modules failed is arguably worse than just erroring out with the ImportError --- fact_extractor/install.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fact_extractor/install.py b/fact_extractor/install.py index 0160fd82..74690ebf 100755 --- a/fact_extractor/install.py +++ b/fact_extractor/install.py @@ -24,14 +24,11 @@ from version import __VERSION__ -try: - import distro - - from fact_extractor.helperFunctions.install import OperateInDirectory - from fact_extractor.install.common import main as common - from fact_extractor.install.unpacker import main as unpacker -except ImportError: - sys.exit('Could not import install dependencies. Please (re-)run install/pre_install.sh') +import distro + +from fact_extractor.helperFunctions.install import OperateInDirectory +from fact_extractor.install.common import main as common +from fact_extractor.install.unpacker import main as unpacker PROGRAM_NAME = 'FACT_extractor Installer' PROGRAM_VERSION = __VERSION__ From 8f4e75546df4021b533073414b221b0a96727289 Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Thu, 12 Sep 2024 15:26:29 +0200 Subject: [PATCH 3/6] refactor: Move install.py As the python interpreter only adds the parent directory of the executed script as module search path and we depend on `fact_extractor` the file must be placed next to the `fact_extractor` module. --- .github/workflows/build_ci.yml | 2 +- Dockerfile | 2 +- fact_extractor/install.py => install.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename fact_extractor/install.py => install.py (96%) diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml index 99d4cebe..248dd0f6 100644 --- a/.github/workflows/build_ci.yml +++ b/.github/workflows/build_ci.yml @@ -39,7 +39,7 @@ jobs: run: ./fact_extractor/install/pre_install.sh - name: Install fact_extractor shell: 'script -q -e -c "bash {0}"' - run: ./fact_extractor/install.py + run: ./install.py - name: Unit Tests shell: 'script -q -e -c "bash {0}"' run: pytest diff --git a/Dockerfile b/Dockerfile index d973747a..2fa30dcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN --mount=type=cache,target=/var/cache/apt \ ADD . /app RUN --mount=type=cache,target=/var/cache/apt \ --mount=type=cache,target=/root/.cache/pip \ -/app/fact_extractor/install.py +/app/install.py ENTRYPOINT ["/app/fact_extractor/docker_extraction.py"] diff --git a/fact_extractor/install.py b/install.py similarity index 96% rename from fact_extractor/install.py rename to install.py index 74690ebf..4a708905 100755 --- a/fact_extractor/install.py +++ b/install.py @@ -22,7 +22,7 @@ import sys from pathlib import Path -from version import __VERSION__ +from fact_extractor.version import __VERSION__ import distro @@ -98,7 +98,7 @@ def main(): distribution = check_distribution() logging.info(f'{PROGRAM_NAME} {PROGRAM_VERSION}') - installation_directory = str(Path(__file__).parent / 'install') + installation_directory = str(Path(__file__).parent / 'fact_extractor' / 'install') with OperateInDirectory(installation_directory): common(distribution) From be0279784723282926e4cc4545332a2416fa20d2 Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Thu, 19 Sep 2024 08:22:04 +0200 Subject: [PATCH 4/6] chore: Add `.dockerignore` Despite us using ADD, the gitignore does not seem to be honored --- .dockerignore | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..43038255 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +# Contents of .gitignore +.cache +.coverage +.idea +.project +.pyc +.pydevproject +.pytest_cache +.settings +__pycache__ +bin/ +install.log + +# Other files +.dockerignore +.git +.github +.gitignore +.ruff_cache From 2192d04173df4b576c92175d4ebd86628fbdd13c Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Thu, 19 Sep 2024 10:24:04 +0200 Subject: [PATCH 5/6] refactor: Unify startup scripts Previously we had three different startup scripts: - The docker entrypoint (fact_extractor/docker_extract.py) - The python module entrypoint (fact_extractor/fact_extract.py) - The docker wrapper (extract.py) This commit removes the docker entrypoint and moves the module entrypoint to __main__.py while changing as little code as possible. --- Dockerfile | 4 +- README.md | 2 +- docker/entrypoint.sh | 5 ++ extract.py | 5 +- .../{fact_extract.py => __main__.py} | 19 ++++-- fact_extractor/docker_extraction.py | 61 ------------------- .../helperFunctions/program_setup.py | 2 +- 7 files changed, 26 insertions(+), 72 deletions(-) create mode 100755 docker/entrypoint.sh rename fact_extractor/{fact_extract.py => __main__.py} (73%) delete mode 100755 fact_extractor/docker_extraction.py diff --git a/Dockerfile b/Dockerfile index 2fa30dcc..01481b3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,10 @@ RUN --mount=type=cache,target=/var/cache/apt \ --mount=type=cache,target=/root/.cache/pip \ /app/fact_extractor/install/pre_install.sh -ADD . /app +ADD . /app/ RUN --mount=type=cache,target=/var/cache/apt \ --mount=type=cache,target=/root/.cache/pip \ /app/install.py -ENTRYPOINT ["/app/fact_extractor/docker_extraction.py"] +ENTRYPOINT ["/app/docker/entrypoint.py"] diff --git a/README.md b/README.md index b04eb0a2..15786593 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ sudo apt install linux-modules-extra-$(uname -r) The tool can then be run with ```bash -fact_extractor/fact_extract.py [OPTIONS] PATH_TO_FIRMWARE +python -m fact_extractor [OPTIONS] PATH_TO_FIRMWARE ``` The tool is build with docker in mind. To that end it extracts all files into a directory specified in the config. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..eadff1b3 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +python -m fact_extractor \ + --config_file /app/fact_extractor/config/main.cfg \ + "$@" diff --git a/extract.py b/extract.py index dd065af6..f72f691c 100755 --- a/extract.py +++ b/extract.py @@ -18,7 +18,10 @@ def parse_arguments(): parser = argparse.ArgumentParser( - description='Command line interface for FACT_extractor.\nExtract arbitrary container or compression formats with one utility.' + description=( + 'Command line wrapper for FACT_extractor docker container.\n' + ' Extract arbitrary container or compression formats with one utility.' + ) ) parser.add_argument('-v', '--version', action='version', version=set_version()) parser.add_argument('-c', '--container', help='docker container', default=DEFAULT_CONTAINER) diff --git a/fact_extractor/fact_extract.py b/fact_extractor/__main__.py similarity index 73% rename from fact_extractor/fact_extract.py rename to fact_extractor/__main__.py index 86721f7e..5a260d72 100755 --- a/fact_extractor/fact_extract.py +++ b/fact_extractor/__main__.py @@ -20,7 +20,8 @@ import sys from pathlib import Path -from fact_extractor.helperFunctions.program_setup import setup_argparser, setup_logging, load_config +from fact_extractor.helperFunctions.file_system import change_owner_of_output_files +from fact_extractor.helperFunctions.program_setup import setup_argparser, setup_logging, load_config, check_ulimits from fact_extractor.unpacker.unpack import unpack @@ -28,14 +29,20 @@ def main(): arguments = setup_argparser('FACT extractor', 'Standalone extraction utility', sys.argv) config = load_config(arguments.config_file) setup_logging(arguments.debug, log_file=arguments.log_file, log_level=arguments.log_level) + check_ulimits() + + data_dir = Path(config.get('unpack', 'data_folder')) + report_dir = data_dir / 'reports' + files_dir = data_dir / 'files' # Make sure report folder exists some meta.json can be written - report_folder = Path(config.get('unpack', 'data_folder'), 'reports') - report_folder.mkdir(parents=True, exist_ok=True) + report_dir.mkdir(parents=True, exist_ok=True) + unpack(arguments.FILE_PATH, config) - return 0 + if arguments.chown is not None: + change_owner_of_output_files(files_dir, arguments.chown) + return 0 -if __name__ == '__main__': - exit(main()) +sys.exit(main()) diff --git a/fact_extractor/docker_extraction.py b/fact_extractor/docker_extraction.py deleted file mode 100755 index 58dd260b..00000000 --- a/fact_extractor/docker_extraction.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -''' - fact_extractor - Copyright (C) 2015-2019 Fraunhofer FKIE - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import argparse -from pathlib import Path -import sys - -from fact_extractor.helperFunctions.config import get_config_dir -from fact_extractor.helperFunctions.file_system import change_owner_of_output_files -from fact_extractor.helperFunctions.program_setup import check_ulimits, load_config, setup_logging -from fact_extractor.unpacker.unpack import unpack - - -def _parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--chown', type=str, default='', help='change back ownership of output files to :' - ) - parser.add_argument( - '--extract_everything', - action='store_true', - default=False, - help='change the behavior of the extractor: extract also empty files', - ) - return parser.parse_args() - - -def main(args): - config = load_config(f'{get_config_dir()}/main.cfg') - setup_logging(debug=False) - check_ulimits() - - input_dir = Path(config.get('unpack', 'data_folder'), 'input') - input_file = list(input_dir.iterdir())[0] - - unpack(input_file, config, args.extract_everything) - - if args.chown: - output_dir = Path(config.get('unpack', 'data_folder'), 'files') - return change_owner_of_output_files(output_dir, args.chown) - - return 0 - - -if __name__ == '__main__': - sys.exit(main(_parse_args())) diff --git a/fact_extractor/helperFunctions/program_setup.py b/fact_extractor/helperFunctions/program_setup.py index f031e343..e0c39f87 100644 --- a/fact_extractor/helperFunctions/program_setup.py +++ b/fact_extractor/helperFunctions/program_setup.py @@ -10,7 +10,7 @@ def setup_argparser(name, description, command_line_options, version=__VERSION__): - parser = argparse.ArgumentParser(description='{} - {}'.format(name, description)) + parser = argparse.ArgumentParser(prog=name, description='{} - {}'.format(name, description)) parser.add_argument('-V', '--version', action='version', version='{} {}'.format(name, version)) parser.add_argument('-l', '--log_file', help='path to log file', default=None) parser.add_argument('-L', '--log_level', help='define the log level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], default=None) From 9ee65bbba8feacfa52b0e93bad97b13063602fa9 Mon Sep 17 00:00:00 2001 From: Marten Ringwelski Date: Thu, 15 Aug 2024 17:38:54 +0200 Subject: [PATCH 6/6] chore: Remove dependency on fact_helper_file --- fact_extractor/__init__.py | 6 ++++++ fact_extractor/helperFunctions/statistics.py | 4 ++-- fact_extractor/install/common.py | 18 ++++++++++++++++++ fact_extractor/install/pre_install.sh | 2 +- .../generic_carver/code/generic_carver.py | 4 ++-- .../unpacking/generic_fs/code/generic_fs.py | 5 ++--- fact_extractor/test/data/ros_header | Bin 0 -> 288 bytes fact_extractor/test/unit/test_mime.py | 9 +++++++++ fact_extractor/unpacker/unpackBase.py | 4 ++-- requirements-unpackers.txt | 2 +- 10 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 fact_extractor/test/data/ros_header create mode 100644 fact_extractor/test/unit/test_mime.py diff --git a/fact_extractor/__init__.py b/fact_extractor/__init__.py index e69de29b..cff4742f 100644 --- a/fact_extractor/__init__.py +++ b/fact_extractor/__init__.py @@ -0,0 +1,6 @@ +import os +import pathlib as pl + +firmware_magic_path = pl.Path(__file__).parent.parent / "bin" / "firmware" + +os.environ["MAGIC"] = f'/usr/lib/file/magic.mgc:{firmware_magic_path}' diff --git a/fact_extractor/helperFunctions/statistics.py b/fact_extractor/helperFunctions/statistics.py index ae865775..1dc74d01 100644 --- a/fact_extractor/helperFunctions/statistics.py +++ b/fact_extractor/helperFunctions/statistics.py @@ -1,3 +1,4 @@ +import magic from configparser import ConfigParser from contextlib import suppress from pathlib import Path @@ -7,7 +8,6 @@ from common_helper_unpacking_classifier import ( avg_entropy, get_binary_size_without_padding, is_compressed ) -from fact_helper_file import get_file_type_from_path from fact_extractor.helperFunctions.config import read_list_from_config @@ -28,7 +28,7 @@ def get_unpack_status(file_path: str, binary: bytes, extracted_files: List[Path] meta_data['entropy'] = avg_entropy(binary) if not extracted_files and meta_data.get('number_of_excluded_files', 0) == 0: - if get_file_type_from_path(file_path)['mime'] in read_list_from_config(config, 'ExpertSettings', 'compressed_file_types')\ + if magic.from_file(file_path, mime=True) in read_list_from_config(config, 'ExpertSettings', 'compressed_file_types')\ or not is_compressed(binary, compress_entropy_threshold=config.getfloat('ExpertSettings', 'unpack_threshold'), classifier=avg_entropy): meta_data['summary'] = ['unpacked'] else: diff --git a/fact_extractor/install/common.py b/fact_extractor/install/common.py index 60a97861..7eebe73f 100644 --- a/fact_extractor/install/common.py +++ b/fact_extractor/install/common.py @@ -1,4 +1,5 @@ import logging +import subprocess as sp import os from contextlib import suppress from pathlib import Path @@ -49,6 +50,23 @@ def main(distribution): with suppress(FileExistsError): os.mkdir('../bin') + sp.run( + [ + "wget", + "--output-document", + "../bin/firmware.xz", + "https://github.com/fkie-cad/firmware-magic-database/releases/download/v0.2.1/firmware.xz", + ], + check=True, + ) + sp.run( + [ + "unxz", + "--force", + "../bin/firmware.xz", + ] + ) + config = load_config('main.cfg') data_folder = config.get('unpack', 'data_folder') os.makedirs(str(Path(data_folder, 'files')), exist_ok=True) diff --git a/fact_extractor/install/pre_install.sh b/fact_extractor/install/pre_install.sh index c6035c55..d5b41ce0 100755 --- a/fact_extractor/install/pre_install.sh +++ b/fact_extractor/install/pre_install.sh @@ -4,7 +4,7 @@ echo "Install Pre-Install Requirements" (apt-get update && apt-get install sudo) || true sudo apt-get update -sudo apt-get -y install git apt-transport-https ca-certificates curl software-properties-common wget libmagic-dev +sudo apt-get -y install git apt-transport-https ca-certificates curl software-properties-common wget libmagic-dev xz-utils IS_VENV=$(python3 -c 'import sys; print(sys.exec_prefix!=sys.base_prefix)') if [[ $IS_VENV == "False" ]] diff --git a/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py b/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py index a11b7eee..9f21867d 100644 --- a/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py +++ b/fact_extractor/plugins/unpacking/generic_carver/code/generic_carver.py @@ -3,13 +3,13 @@ ''' from __future__ import annotations +import magic import logging import re import shutil from pathlib import Path from common_helper_process import execute_shell_command -from fact_helper_file import get_file_type_from_path NAME = 'generic_carver' MIME_PATTERNS = ['generic/carver'] @@ -45,7 +45,7 @@ def remove_false_positive_archives(self) -> str: for file_path in self.unpack_directory.glob('**/*'): if not file_path.is_file(): continue - file_type = get_file_type_from_path(file_path)['mime'] + file_type = magic.from_file(file_path, mime=True) if file_type == 'application/x-tar' or self._is_possible_tar(file_type, file_path): self._remove_invalid_archives(file_path, 'tar -tvf {}', 'does not look like a tar archive') diff --git a/fact_extractor/plugins/unpacking/generic_fs/code/generic_fs.py b/fact_extractor/plugins/unpacking/generic_fs/code/generic_fs.py index f35fd9f4..fdc213b0 100644 --- a/fact_extractor/plugins/unpacking/generic_fs/code/generic_fs.py +++ b/fact_extractor/plugins/unpacking/generic_fs/code/generic_fs.py @@ -2,13 +2,12 @@ This plugin mounts filesystem images and extracts their content ''' import re +import magic from shlex import split from subprocess import run, PIPE, STDOUT from tempfile import TemporaryDirectory from time import sleep -from fact_helper_file import get_file_type_from_path - NAME = 'genericFS' MIME_PATTERNS = [ 'filesystem/btrfs', 'filesystem/dosmbr', 'filesystem/f2fs', 'filesystem/jfs', 'filesystem/minix', @@ -28,7 +27,7 @@ def unpack_function(file_path, tmp_dir): - mime_type = get_file_type_from_path(file_path)['mime'] + mime_type = magic.from_file(file_path, mime=True) if mime_type == 'filesystem/dosmbr': output = _mount_from_boot_record(file_path, tmp_dir) else: diff --git a/fact_extractor/test/data/ros_header b/fact_extractor/test/data/ros_header new file mode 100644 index 0000000000000000000000000000000000000000..286fc793a94ec281fe5b46071dff41592a2e001c GIT binary patch literal 288 zcmWIY3kfjPGcaT^6kw2Hy1{;!)09C*C0#1O(b<~;2-tvF0K|X-XCKdaH%}i|82lfFu`O{u(Q7l+;t2L1(m!E%@fq&`h$S}HPSOM burdJom;D9<10zFo2nXDRSO$g$B!LV7_ogVu literal 0 HcmV?d00001 diff --git a/fact_extractor/test/unit/test_mime.py b/fact_extractor/test/unit/test_mime.py new file mode 100644 index 00000000..58d5c61c --- /dev/null +++ b/fact_extractor/test/unit/test_mime.py @@ -0,0 +1,9 @@ +import magic +from fact_extractor.helperFunctions.file_system import get_test_data_dir + +def test_magic(): + # Ensures that all submodules of the extractor use the custom mime types + import fact_extractor + assert magic.from_file(f"{get_test_data_dir()}/ros_header", mime=True) == "firmware/ros", "firmware-magic-database is not loaded" + + assert magic.from_file(f"{get_test_data_dir()}/container/test.zip", mime=True) == "application/zip" diff --git a/fact_extractor/unpacker/unpackBase.py b/fact_extractor/unpacker/unpackBase.py index 59d9cc20..6fedfc3c 100644 --- a/fact_extractor/unpacker/unpackBase.py +++ b/fact_extractor/unpacker/unpackBase.py @@ -4,9 +4,9 @@ from time import time import fnmatch from typing import Callable, Dict, List, Tuple +import magic from common_helper_files import get_files_in_dir -from fact_helper_file import get_file_type_from_path from fact_extractor.helperFunctions.config import read_list_from_config from fact_extractor.helperFunctions.plugin import import_plugins @@ -50,7 +50,7 @@ def get_unpacker(self, mime_type: str): return self.unpacker_plugins['generic/carver'] def extract_files_from_file(self, file_path: str, tmp_dir) -> Tuple[List, Dict]: - current_unpacker = self.get_unpacker(get_file_type_from_path(file_path)['mime']) + current_unpacker = self.get_unpacker(magic.from_file(file_path, mime=True)) return self._extract_files_from_file_using_specific_unpacker(file_path, tmp_dir, current_unpacker) def unpacking_fallback(self, file_path, tmp_dir, old_meta, fallback_plugin_mime) -> Tuple[List, Dict]: diff --git a/requirements-unpackers.txt b/requirements-unpackers.txt index 85a3b75d..8956be14 100644 --- a/requirements-unpackers.txt +++ b/requirements-unpackers.txt @@ -1,7 +1,7 @@ # FixMe: deprecated pluginbase~=1.0.1 git+https://github.com/fkie-cad/common_helper_unpacking_classifier.git -git+https://github.com/fkie-cad/fact_helper_file.git +python-magic patool~=2.2.0 # jffs2: jefferson + deps git+https://github.com/sviehb/jefferson.git@v0.4.1