Drop dependency on fact_helper_file

.dockerignore

@@ -0,0 +1,19 @@
+# Contents of .gitignore
+.cache
+.coverage
+.idea
+.project
+.pyc
+.pydevproject
+.pytest_cache
+.settings
+__pycache__
+bin/
+install.log
+
+# Other files
+.dockerignore
+.git
+.github
+.gitignore
+.ruff_cache

.github/workflows/build_ci.yml

@@ -39,7 +39,7 @@ jobs:
         run: ./fact_extractor/install/pre_install.sh
       - name: Install fact_extractor
         shell: 'script -q -e -c "bash {0}"'
-        run: ./fact_extractor/install.py
+        run: ./install.py
       - name: Unit Tests
         shell: 'script -q -e -c "bash {0}"'
         run: pytest

Dockerfile

@@ -10,17 +10,17 @@ apt update && apt install -y \
 RUN python3.11 -m venv /venv
 ENV PATH=/venv/bin:$PATH \
     VIRTUAL_ENV=/venv \
-    PYTHONPATH=/app/fact_extractor
+    PYTHONPATH=/app/
 
 ADD ./fact_extractor/install/pre_install.sh /app/fact_extractor/install/pre_install.sh
 RUN --mount=type=cache,target=/var/cache/apt \
     --mount=type=cache,target=/root/.cache/pip \
 /app/fact_extractor/install/pre_install.sh
 
-ADD . /app
+ADD . /app/
 RUN --mount=type=cache,target=/var/cache/apt \
     --mount=type=cache,target=/root/.cache/pip \
-/app/fact_extractor/install.py
+/app/install.py
 
 
-ENTRYPOINT ["/app/fact_extractor/docker_extraction.py"]
+ENTRYPOINT ["/app/docker/entrypoint.py"]

README.md

@@ -42,7 +42,7 @@ sudo apt install linux-modules-extra-$(uname -r)
 The tool can then be run with
 
 ```bash
-fact_extractor/fact_extract.py [OPTIONS] PATH_TO_FIRMWARE
+python -m fact_extractor [OPTIONS] PATH_TO_FIRMWARE
 ```
 The tool is build with docker in mind.
 To that end it extracts all files into a directory specified in the config.

docker/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+python -m fact_extractor \
+    --config_file /app/fact_extractor/config/main.cfg \
+    "$@"

extract.py

@@ -18,7 +18,10 @@
 
 def parse_arguments():
     parser = argparse.ArgumentParser(
-        description='Command line interface for FACT_extractor.\nExtract arbitrary container or compression formats with one utility.'
+        description=(
+            'Command line wrapper for FACT_extractor docker container.\n'
+            ' Extract arbitrary container or compression formats with one utility.'
+        )
     )
     parser.add_argument('-v', '--version', action='version', version=set_version())
     parser.add_argument('-c', '--container', help='docker container', default=DEFAULT_CONTAINER)

fact_extractor/__init__.py

@@ -0,0 +1,6 @@
+import os
+import pathlib as pl
+
+firmware_magic_path = pl.Path(__file__).parent.parent / "bin" / "firmware"
+
+os.environ["MAGIC"] = f'/usr/lib/file/magic.mgc:{firmware_magic_path}'

fact_extractor/fact_extract.py → fact_extractor/__main__.py

@@ -20,22 +20,29 @@
 import sys
 from pathlib import Path
 
-from helperFunctions.program_setup import setup_argparser, setup_logging, load_config
-from unpacker.unpack import unpack
+from fact_extractor.helperFunctions.file_system import change_owner_of_output_files
+from fact_extractor.helperFunctions.program_setup import setup_argparser, setup_logging, load_config, check_ulimits
+from fact_extractor.unpacker.unpack import unpack
 
 
 def main():
     arguments = setup_argparser('FACT extractor', 'Standalone extraction utility', sys.argv)
     config = load_config(arguments.config_file)
     setup_logging(arguments.debug, log_file=arguments.log_file, log_level=arguments.log_level)
+    check_ulimits()
+
+    data_dir = Path(config.get('unpack', 'data_folder'))
+    report_dir = data_dir / 'reports'
+    files_dir = data_dir / 'files'
 
     # Make sure report folder exists some meta.json can be written
-    report_folder = Path(config.get('unpack', 'data_folder'), 'reports')
-    report_folder.mkdir(parents=True, exist_ok=True)
+    report_dir.mkdir(parents=True, exist_ok=True)
+
     unpack(arguments.FILE_PATH, config)
 
-    return 0
+    if arguments.chown is not None:
+        change_owner_of_output_files(files_dir, arguments.chown)
 
+    return 0
 
-if __name__ == '__main__':
-    exit(main())
+sys.exit(main())

fact_extractor/helperFunctions/config.py

@@ -1,7 +1,7 @@
 import os
 from configparser import ConfigParser, NoOptionError, NoSectionError
 
-from helperFunctions.file_system import get_src_dir
+from fact_extractor.helperFunctions.file_system import get_src_dir
 
 
 def load_config(config_file_name):

fact_extractor/helperFunctions/hash.py

@@ -1,6 +1,6 @@
 from hashlib import new
 
-from helperFunctions.dataConversion import make_bytes
+from fact_extractor.helperFunctions.dataConversion import make_bytes
 
 
 def get_hash(hash_function, binary):

fact_extractor/helperFunctions/plugin.py

@@ -4,7 +4,7 @@
 from common_helper_files import get_dirs_in_dir
 from pluginbase import PluginBase
 
-from helperFunctions.file_system import get_src_dir
+from fact_extractor.helperFunctions.file_system import get_src_dir
 
 
 def import_plugins(plugin_mount, plugin_base_dir):

fact_extractor/helperFunctions/program_setup.py

@@ -5,18 +5,28 @@
 
 from common_helper_files import create_dir_for_file
 
-from helperFunctions.config import get_config_dir
-from version import __VERSION__
+from fact_extractor.helperFunctions.config import get_config_dir
+from fact_extractor.version import __VERSION__
 
 
 def setup_argparser(name, description, command_line_options, version=__VERSION__):
-    parser = argparse.ArgumentParser(description='{} - {}'.format(name, description))
+    parser = argparse.ArgumentParser(prog=name, description='{} - {}'.format(name, description))
     parser.add_argument('-V', '--version', action='version', version='{} {}'.format(name, version))
     parser.add_argument('-l', '--log_file', help='path to log file', default=None)
     parser.add_argument('-L', '--log_level', help='define the log level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], default=None)
     parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages')
     parser.add_argument('-C', '--config_file', help='set path to config File', default='{}/main.cfg'.format(get_config_dir()))
     parser.add_argument('FILE_PATH', type=str, help='Path to file that should be extracted')
+    parser.add_argument(
+        '--chown', type=str, default=None, help='change back ownership of output files to <user id>:<group id>'
+    )
+    parser.add_argument(
+        '--extract_everything',
+        action='store_true',
+        default=False,
+        help='change the behavior of the extractor: extract also empty files',
+    )
+
     return parser.parse_args(command_line_options[1:])
 
 

fact_extractor/helperFunctions/statistics.py

@@ -1,3 +1,4 @@
+import magic
 from configparser import ConfigParser
 from contextlib import suppress
 from pathlib import Path
@@ -7,8 +8,7 @@
 from common_helper_unpacking_classifier import (
     avg_entropy, get_binary_size_without_padding, is_compressed
 )
-from fact_helper_file import get_file_type_from_path
-from helperFunctions.config import read_list_from_config
+from fact_extractor.helperFunctions.config import read_list_from_config
 
 
 def add_unpack_statistics(extraction_dir: Path, meta_data: Dict):
@@ -28,7 +28,7 @@ def get_unpack_status(file_path: str, binary: bytes, extracted_files: List[Path]
     meta_data['entropy'] = avg_entropy(binary)
 
     if not extracted_files and meta_data.get('number_of_excluded_files', 0) == 0:
-        if get_file_type_from_path(file_path)['mime'] in read_list_from_config(config, 'ExpertSettings', 'compressed_file_types')\
+        if magic.from_file(file_path, mime=True) in read_list_from_config(config, 'ExpertSettings', 'compressed_file_types')\
                 or not is_compressed(binary, compress_entropy_threshold=config.getfloat('ExpertSettings', 'unpack_threshold'), classifier=avg_entropy):
             meta_data['summary'] = ['unpacked']
         else:

fact_extractor/install/common.py

@@ -1,10 +1,11 @@
 import logging
+import subprocess as sp
 import os
 from contextlib import suppress
 from pathlib import Path
 
-from helperFunctions.config import load_config
-from helperFunctions.install import (
+from fact_extractor.helperFunctions.config import load_config
+from fact_extractor.helperFunctions.install import (
     apt_install_packages, apt_update_sources, pip_install_packages, load_requirements_file
 )
 
@@ -49,6 +50,23 @@ def main(distribution):
     with suppress(FileExistsError):
         os.mkdir('../bin')
 
+    sp.run(
+        [
+            "wget",
+            "--output-document",
+            "../bin/firmware.xz",
+            "https://github.com/fkie-cad/firmware-magic-database/releases/download/v0.2.1/firmware.xz",
+        ],
+        check=True,
+    )
+    sp.run(
+        [
+            "unxz",
+            "--force",
+            "../bin/firmware.xz",
+        ]
+    )
+
     config = load_config('main.cfg')
     data_folder = config.get('unpack', 'data_folder')
     os.makedirs(str(Path(data_folder, 'files')), exist_ok=True)

fact_extractor/install/pre_install.sh

@@ -4,7 +4,7 @@ echo "Install Pre-Install Requirements"
 (apt-get update && apt-get install sudo) || true
 
 sudo apt-get update
-sudo apt-get -y install git apt-transport-https ca-certificates curl software-properties-common wget libmagic-dev
+sudo apt-get -y install git apt-transport-https ca-certificates curl software-properties-common wget libmagic-dev xz-utils
 
 IS_VENV=$(python3 -c 'import sys; print(sys.exec_prefix!=sys.base_prefix)')
 if [[ $IS_VENV == "False" ]]

fact_extractor/install/unpacker.py

@@ -10,7 +10,7 @@
 
 from common_helper_process import execute_shell_command_get_return_code
 
-from helperFunctions.install import (
+from fact_extractor.helperFunctions.install import (
     InstallationError,
     OperateInDirectory,
     apt_install_packages,

fact_extractor/plugins/unpacking/ambarella/code/ambarella.py

@@ -5,7 +5,7 @@
 
 from common_helper_files import get_files_in_dir
 from common_helper_process import execute_shell_command
-from helperFunctions.file_system import get_src_dir
+from fact_extractor.helperFunctions.file_system import get_src_dir
 
 NAME = 'Ambarella'
 MIME_PATTERNS = ['firmware/ambarella']

fact_extractor/plugins/unpacking/ambarella/code/ambarella_romfs.py

@@ -2,7 +2,7 @@
 from os import path
 
 from common_helper_process import execute_shell_command
-from helperFunctions.file_system import get_fact_bin_dir
+from fact_extractor.helperFunctions.file_system import get_fact_bin_dir
 
 NAME = 'Ambarella_RomFS'
 MIME_PATTERNS = ['filesystem/ambarella-romfs']

fact_extractor/plugins/unpacking/ambarella/test/test_plugin_ambarella_romfs.py

@@ -1,5 +1,5 @@
 import os
-from test.unit.unpacker.test_unpacker import TestUnpackerBase
+from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase
 
 
 TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')

fact_extractor/plugins/unpacking/android_simg/test/test_plugin_android_simg.py

@@ -6,7 +6,7 @@
 print(root_dir, [str(d) for d in root_dir.iterdir() if d.is_dir()])
 sys.path.append(str(root_dir))
 
-from test.unit.unpacker.test_unpacker import TestUnpackerBase
+from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase
 
 TEST_DATA_DIR = Path(__file__).parent / 'data'
 

fact_extractor/plugins/unpacking/arj/test/test_arj.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from test.unit.unpacker.test_unpacker import TestUnpackerBase
+from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase
 
 TEST_FILE = Path(__file__).parent / 'data' / 'test.arj'
 

fact_extractor/plugins/unpacking/ascii85/test/test_plugin_adobe.py

@@ -3,8 +3,8 @@
 from tempfile import TemporaryDirectory
 from unittest.mock import patch
 
-from plugins.unpacking.ascii85.code.adobe import unpack_function
-from test.unit.unpacker.test_unpacker import TestUnpackerBase
+from fact_extractor.plugins.unpacking.ascii85.code.adobe import unpack_function
+from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase
 
 TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')
 

fact_extractor/plugins/unpacking/avm_kernel_image/code/avm_kernel_image.py

@@ -1,6 +1,6 @@
 from common_helper_process.fail_safe_subprocess import execute_shell_command
 from pathlib import Path
-from helperFunctions.file_system import get_fact_bin_dir
+from fact_extractor.helperFunctions.file_system import get_fact_bin_dir
 
 
 NAME = 'avm_kernel_image'

fact_extractor/plugins/unpacking/avm_kernel_image/test/test_avm_kernel.py

@@ -1,4 +1,4 @@
-from test.unit.unpacker.test_unpacker import TestUnpackerBase
+from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase
 from ..code.avm_kernel_image import FIND_SQUASHFS_TOOL_PATH, UNPACK_KERNEL_TOOL_PATH
 
 

fact_extractor/plugins/unpacking/avm_sqfs_fake/test/test_plugin.py

@@ -1,7 +1,7 @@
 import os
 from common_helper_files import get_binary_from_file
 
-from test.unit.unpacker.test_unpacker import TestUnpackerBase
+from fact_extractor.test.unit.unpacker.test_unpacker import TestUnpackerBase
 
 
 TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')