Mach-O File Format Parsing (#1562)

* Adds first pass at the Macho-O parser

* Ports Plaso Mach-0 parsing code

* Base 20240912

* Adds directory walk

* Cleaned up logging

* Outputs parsing result JSON objects

* Writes Mach-O parse result JSON output file

* Adds segments and sections

* Pretty prints JSON

* Remove RawDisk for the time being as it is not supported (yet)

* First pass at JSON formated report

* Adds symbols

* Adds architecture info and parsing time

* Adds magic and flags

* Adds Signature CodeDirectory items to report

* Completes signature report

* Adds tgz with three FatBinaries, cleans up debug logging

* Adds TLSH

* Switches Segment size to hex

* Puts safety net around signature parsing

* Fixes SymHash calculation

* Adds ssdeep

* Adds ppdeep and py-tlsh to server and api-server

* Adds imports

* Adds section details

* Clean up evidence type and remove iocs section from report

* Places output reports in location corresponding to input evidence

* Adds MachoExtraction Evidence type

* Update poetry.lock

* Introduces Mach-O test

* Adds Mach-O test case

* Switches from ppdeep to pyssdeep

* Adds cd_hash calculation

* Removing test data that is no longer used

* Cleans up log statements

* Add comment to explain lambda for json.dumps()

* Fix output formatting

* Formatting cleanups and os.path.relpath for simpler rel path calc

* Adds assertions to unit test

* Reduces scope of imported libraries to worker

* Removes dev docker compose file

* Updates poetry to the latest status

Author: daschwanden

.gitignore

@@ -13,6 +13,8 @@
 /_sources/
 /.tox
 charts/
+conf/
+evidence/
 
 # And don't care about the 'egg'.
 /turbinia.egg-info

docker/server/Dockerfile

@@ -56,4 +56,3 @@ ENV TURBINIA_DEBUG_PORT ${TURBINIA_DEBUG_PORT:-20000}
 CMD ["/home/turbinia/start.sh"]
 # Expose Prometheus endpoint.
 EXPOSE 9200/tcp
-

poetry.lock

@@ -15,6 +15,7 @@ turbiniactl = "turbinia.turbiniactl:main"
 [tool.poetry.dependencies]
 python = "^3.10"
 acstore = { version = "20240128" }
+asn1crypto = { version = ">= 0.24.0", optional = true }
 backoff = { version = ">=2.2.1" }
 celery = { version = "^5.2.2" }
 dfDewey = { version = "^20231016", optional = true }
@@ -25,14 +26,17 @@ filelock = { version = "*" }
 google-api-core = { version = "<3.0.0", optional = true }
 google-generativeai = { version = ">=0.8.2" }
 libcloudforensics = "^20240325"
+lief = { version = ">= 0.15.1", optional = true }
 opensearch-py = {version = "2.4.2"}
 pandas = { version = "^2.1.0" }
 plaso = { version = "20240308", optional = true }
+pyssdeep = { version = "1.0.0", optional = true }
 prometheus_client = { version = "^0.17.1" }
 protobuf = { version = ">=3.19.0", optional = true }
 pydantic = { version = "^1.10.5,<2"}
 pyglove = { version = ">=0.4.4" }
 pyhindsight = { version = "^20230327.0", optional = true }
+py-tlsh = { version = ">= 4.7.2", optional = true }
 ratelimit = { version = ">=2.2.1" }
 redis = { version = "^4.4.4" }
 urllib3 = [
@@ -58,10 +62,14 @@ yapf = "*"
 
 [tool.poetry.extras]
 worker = [
+  "asn1crypto",
   "dfimagetools",
   "dfDewey",
+  "lief",
   "plaso",
+  "pyssdeep",
   "pyhindsight",
+  "py-tlsh",
 ]
 
 [tool.yapfignore]

pyproject.toml

@@ -219,6 +219,11 @@
     'programs': ['hashcat', 'john'],
     'docker_image': None,
     'timeout': 1200
+}, {
+    'job': 'MachoAnalysisJob',
+    'programs': ['grep'],
+    'docker_image': None,
+    'timeout': 3600
 }, {
     'job': 'YaraAnalysisJob',
     'programs': ['/opt/fraken/fraken'],

test_data/macho-3.tgz

@@ -1201,6 +1201,11 @@ class BinaryExtraction(CompressedDirectory):
   pass
 
 
+class MachoExtraction(CompressedDirectory):
+  """Mach-O details extracted from evidence."""
+  pass
+
+
 class DockerContainer(Evidence):
   """Evidence object for a DockerContainer filesystem.
 

turbinia/config/turbinia_config_tmpl.py

@@ -29,6 +29,7 @@
 from turbinia.jobs import jupyter
 from turbinia.jobs import linux_acct
 from turbinia.jobs import llm_artifacts_analyzer
+from turbinia.jobs import macho
 from turbinia.jobs import yara
 from turbinia.jobs import partitions
 from turbinia.jobs import photorec

turbinia/evidence.py

@@ -0,0 +1,31 @@
+"""Job to execute macho analysis task."""
+
+from turbinia.evidence import MachoExtraction
+from turbinia.evidence import Directory
+from turbinia.evidence import RawDisk
+from turbinia.evidence import ReportText
+from turbinia.jobs import interface
+from turbinia.jobs import manager
+from turbinia.workers.analysis import macho
+
+
+class MachoAnalysisJob(interface.TurbiniaJob):
+  """Mach-O analysis job."""
+
+  evidence_input = [MachoExtraction]
+  evidence_output = [ReportText]
+
+  NAME = 'MachoAnalysisJob'
+
+  def create_tasks(self, evidence):
+    """Create task.
+    Args:
+      evidence: List of evidence objects to process
+    Returns:
+        A list of tasks to schedule.
+    """
+    tasks = [macho.MachoAnalysisTask() for _ in evidence]
+    return tasks
+
+
+manager.JobsManager.RegisterJob(MachoAnalysisJob)

turbinia/jobs/__init__.py

@@ -54,6 +54,7 @@ class TaskLoader():
       'LinuxAccountAnalysisTask',
       'LinuxSSHAnalysisTask',
       'LLMAnalyzerTask',
+      'MachoAnalysisTask',
       'YaraAnalysisTask',
       'PartitionEnumerationTask',
       'PhotorecTask',
@@ -106,6 +107,7 @@ def get_task(self, task_name):
     from turbinia.workers.analysis.jupyter import JupyterAnalysisTask
     from turbinia.workers.analysis.linux_acct import LinuxAccountAnalysisTask
     from turbinia.workers.analysis.llm_analyzer import LLMAnalyzerTask
+    from turbinia.workers.analysis.macho import MachoAnalysisTask
     from turbinia.workers.analysis.postgresql_acct import PostgresAccountAnalysisTask
     from turbinia.workers.analysis.redis import RedisAnalysisTask
     from turbinia.workers.analysis.ssh_analyzer import LinuxSSHAnalysisTask