Merge pull request #903 from JamesHabben/lava-output

update device_info

admin/docs/device_info_values.md

@@ -0,0 +1,90 @@
+# Device Information Collection
+
+This document outlines the various device information collected by iLEAPP modules using the `device_info()` and `logdevinfo()` functions.
+
+> **Note**: The information below is automatically generated by `admin/scripts/device_info_values.py`
+
+## Categories and Sources
+
+### device_info() Usage
+<!-- DEVICE_INFO_START -->
+| Category | Label | Source Modules |
+|-----------|-------|----------------|
+| Advertising Identifier | Apple Advertising Identifier | advertisingID |
+| Airdrop | Airdrop ID | airdropId |
+| Backup Settings | Cloud Backup Enabled | backupSettings |
+| Backup Settings | Last Cloud iTunes Backup Date | backupSettings |
+| Backup Settings | Last Cloud iTunes Backup TZ | backupSettings |
+| Backup Settings | Last iTunes Backup Date | backupSettings |
+| Backup Settings | Last iTunes Backup TZ | backupSettings |
+| Device Information | Product | lastBuild |
+| Device Information | ProductBuildVersion | lastBuild |
+| Device Information | iOS version | lastBuild |
+| Settings | Last System Version | appleLocationd |
+| Settings | Location Services Enabled | appleLocationd |
+| WiFi | Keep Wifi Powered Airplane Mode | appleWifiPlist |
+<!-- DEVICE_INFO_END -->
+
+### logdevinfo() Usage
+<!-- LOGDEVINFO_START -->
+| Key | Source Modules |
+|-----|----------------|
+| Bluetooth Address: {x[1]} | deviceActivator |
+| BuildID: {val} | Ph99SystemVersionPlist |
+| BuildVersion: {val} | Ph100UFEDdevcievaluesplist, Ph99SystemVersionPlist |
+| CDMA Network Phone Number ICCID: {val} | celWireless |
+| Device Name: {val} | deviceName |
+| Device/Computer Name: {computername} | preferencesPlist |
+| DeviceName: {val} | Ph100UFEDdevcievaluesplist |
+| Ethernet Mac Address: {x[1]} | deviceActivator |
+| Find My iPhone Add Time: {addtime} | findMy |
+| Find My iPhone: Enabled | findMy |
+| HardwareModel: {val} | Ph100UFEDdevcievaluesplist |
+| Host Name: {hostname} | preferencesPlist |
+| IMEI: {val} | celWireless |
+| IMEIs: {imeis} | deviceDatam |
+| InternationalMobileEquipmentIdentity: {val} | Ph100UFEDdevcievaluesplist |
+| Last Bootstrap Date: {times} | timezoneInfo |
+| Last Bootstrap Timezone: {val} | timezoneInfo |
+| Last Good IMSI: {lastgoodimsi} | imeiImsi |
+| Last Known ICCI: {lastknownicci} | imeiImsi |
+| Last Known ICCID: {val} | celWireless |
+| MAC Address: {hexstring} - User Defined Name: {userdefinedname} - BSD Name: {bsdname} | wifiIdent |
+| MEID: {val} | celWireless |
+| Model Number: {x[1]} | deviceActivator |
+| Model: {localhostname} | preferencesPlist |
+| Model: {val} | preferencesPlist |
+| Obliterated Timestamp: {utc_modified_date} | obliterated |
+| PasswordProtected: {val} | Ph100UFEDdevcievaluesplist |
+| Phone Number: {val} | imeiImsi |
+| ProductName: {val} | Ph99SystemVersionPlist |
+| ProductType: {val} | Ph100UFEDdevcievaluesplist |
+| ReleaseType: {val} | Ph99SystemVersionPlist |
+| Reported Phone Number: {val} | celWireless, deviceDatam |
+| Self Registration Update IMEI: {selfregistrationupdateimei} | imeiImsi |
+| Self Registration Update IMSI: {selfregitrationupdateimsi} | imeiImsi |
+| Serial Number: {row[0]} | serialNumber |
+| SerialNumber: {val} | Ph100UFEDdevcievaluesplist |
+| SystemImageID: {val} | Ph99SystemVersionPlist |
+| TimeZone: {val} | Ph100UFEDdevcievaluesplist |
+| Timezone Set: {val} | timezoneset |
+| UDID: {uid} | carCD |
+| Vehicle - Last Connected: {connected} - Last Disconnected: {disconnected} - Type: {contype} | carCD |
+| Wifi Address: {x[1]} | deviceActivator |
+| com.apple.MobileSMS.plist - Keep Messages for Days (iOS 17+): {val} | messageRetention |
+| com.apple.MobileSMS.plist - Keep Messages for Days (iOS {val} | messageRetention |
+| com.apple.MobileSMS.plist - Keep Messages for Days: No Value | messageRetention |
+| com.apple.mobileSMS.plist - Keep Messages for Days (iOS 17+): {keep_val} | messageRetention |
+| com.apple.mobileSMS.plist - Keep Messages for Days (iOS {keep_val} | messageRetention |
+| com.apple.mobileSMS.plist - Keep Messages for Days: No Value | messageRetention |
+| comapplemobileslideshowplist-PhotosSharedLibrarySyncingIsActive: {val} | Ph80comappleMobileSlideShowPlist |
+| comapplemobileslideshowplist-downloadAndKeepOriginals: {val} | Ph80comappleMobileSlideShowPlist |
+| comapplepurplebuddyplist-SetupState: {val} | Ph83comapplePurpleBuddyPlist |
+| iOS version: {val} | Ph100UFEDdevcievaluesplist, Ph99SystemVersionPlist |
+| {base_file} - Keep Message for Days: No Value | messageRetention |
+| {base_file} - Keep Messages for Days (iOS 17+): {keep_val} | messageRetention |
+| {base_file} - Keep Messages for Days (iOS {keep_val} | messageRetention |
+| {info_key}: {value_key} | iTunesBackupInfo |
+| {key}: {val} | timezoneset |
+<!-- LOGDEVINFO_END -->
+

admin/docs/module_updates.md

@@ -10,6 +10,7 @@ This guide outlines the process of updating existing xLEAPP modules to use the n
 4. Adjust the main function
 5. Remove manual report generation code
 6. Add chat parameters if the artifact should support a threaded type view
+7. Update Device Information Collection
 
 ## Detailed Process
 
@@ -158,6 +159,38 @@ Example (From googleChat.py artifact):
         }
     }
 
+### 7. Update Device Information Collection
+
+The `logdevinfo()` function is being deprecated in favor of the new `device_info()` function. This new function provides better organization and structure for device-related information. Not every module uses these functions, so this section is only applicable to modules that do.
+
+#### Old Method (logdevinfo):
+```python
+logdevinfo(f'<b>IMEI: </b>{imei}')
+logdevinfo(f'<b>Serial Number: </b>{serial}')
+```
+
+#### New Method (device_info):
+```python
+device_info("Device Information", "IMEI", imei)
+device_info("Device Information", "Serial Number", serial)
+```
+
+Key differences:
+1. No HTML formatting needed - display formatting is handled by the output generator
+2. Information is categorized for better organization
+3. Values are stored in a structured format that's easier to query and display
+4. Source tracking is automatic - the module name is recorded with each value
+5. Duplicate handling is built-in - multiple modules can report the same information
+
+The new structure allows for:
+- Better organization of device information by category
+- Automatic tracking of which modules provided what information
+- Easier querying and filtering of device information
+- Consistent formatting across all outputs
+- De-duplication and conflict resolution
+
+You can view the current categories and labels being used across all modules in the [Device Info Values](device_info_values.md) documentation.
+
 ## Reasoning
 
 This update simplifies module maintenance by:

admin/scripts/device_info_values.py

@@ -0,0 +1,114 @@
+import os
+import re
+import ast
+from pathlib import Path
+
+def find_function_calls(file_path, function_name):
+    """
+    Parse a Python file and find all calls to the specified function
+    Returns a list of tuples containing (category, label) for device_info
+    or (key) for logdevinfo
+    """
+    with open(file_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    calls = []
+    try:
+        tree = ast.parse(content)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Call):
+                if isinstance(node.func, ast.Name) and node.func.id == function_name:
+                    if function_name == 'device_info' and len(node.args) >= 2:
+                        # Get the string values if they're string literals
+                        if isinstance(node.args[0], ast.Constant) and isinstance(node.args[1], ast.Constant):
+                            calls.append((node.args[0].value, node.args[1].value))
+                    elif function_name == 'logdevinfo' and len(node.args) >= 1:
+                        # For logdevinfo, try to extract the message without HTML tags
+                        if isinstance(node.args[0], (ast.Constant, ast.JoinedStr)):
+                            # Convert the argument to string and strip HTML
+                            arg_str = ast.unparse(node.args[0])
+                            # Remove f-string prefix if present
+                            arg_str = arg_str.strip('f').strip('"\'')
+                            # Basic HTML tag removal (can be enhanced if needed)
+                            clean_str = re.sub(r'<[^>]+>', '', arg_str)
+                            calls.append((clean_str,))
+    except:
+        # If parsing fails, try regex as fallback
+        if function_name == 'device_info':
+            pattern = r'device_info\([\'"]([^\'"]+)[\'"],\s*[\'"]([^\'"]+)[\'"]\s*,'
+            calls.extend(re.findall(pattern, content))
+        else:
+            # Updated pattern to handle f-strings and HTML tags
+            pattern = r'logdevinfo\(f?[\'"].*?<b>([^<]+)</b>'
+            matches = re.findall(pattern, content)
+            calls.extend([(m.strip(),) for m in matches])
+
+    return calls
+
+def generate_markdown():
+    script_dir = Path(__file__).parent
+    root_dir = script_dir.parent.parent
+
+    artifacts_dir = Path( root_dir, 'scripts/artifacts')
+    device_info_usage = {}
+    logdevinfo_usage = {}
+
+    # Scan all Python files in artifacts directory
+    for file_path in artifacts_dir.glob('*.py'):
+        module_name = file_path.stem
+
+        # Find device_info calls
+        device_calls = find_function_calls(file_path, 'device_info')
+        if device_calls:
+            for category, label in device_calls:
+                if category not in device_info_usage:
+                    device_info_usage[category] = {}
+                if label not in device_info_usage[category]:
+                    device_info_usage[category][label] = []
+                device_info_usage[category][label].append(module_name)
+
+        # Find logdevinfo calls
+        log_calls = find_function_calls(file_path, 'logdevinfo')
+        if log_calls:
+            for (key,) in log_calls:
+                if key not in logdevinfo_usage:
+                    logdevinfo_usage[key] = []
+                logdevinfo_usage[key].append(module_name)
+
+    # Generate markdown content
+    device_info_md = "| Category | Label | Source Modules |\n|-----------|-------|----------------|\n"
+    for category in sorted(device_info_usage.keys()):
+        for label in sorted(device_info_usage[category].keys()):
+            modules = ", ".join(sorted(set(device_info_usage[category][label])))
+            device_info_md += f"| {category} | {label} | {modules} |\n"
+
+    logdevinfo_md = "| Key | Source Modules |\n|-----|----------------|\n"
+    for key in sorted(logdevinfo_usage.keys()):
+        modules = ", ".join(sorted(set(logdevinfo_usage[key])))
+        logdevinfo_md += f"| {key} | {modules} |\n"
+
+    # Read the existing markdown file
+    doc_path = Path( root_dir, 'admin/docs/device_info_values.md')
+    with open(doc_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    # Replace the placeholders
+    content = re.sub(
+        r'<!-- DEVICE_INFO_START -->.*<!-- DEVICE_INFO_END -->',
+        f'<!-- DEVICE_INFO_START -->\n{device_info_md}<!-- DEVICE_INFO_END -->',
+        content,
+        flags=re.DOTALL
+    )
+    content = re.sub(
+        r'<!-- LOGDEVINFO_START -->.*<!-- LOGDEVINFO_END -->',
+        f'<!-- LOGDEVINFO_START -->\n{logdevinfo_md}<!-- LOGDEVINFO_END -->',
+        content,
+        flags=re.DOTALL
+    )
+
+    # Write the updated content
+    with open(doc_path, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+if __name__ == '__main__':
+    generate_markdown()

admin/test/cases/testdata.advertisingID.json

@@ -0,0 +1,31 @@
+{
+  "mvs_ios_2023": {
+    "description": "",
+    "maker": "",
+    "make_data": {
+      "input_data_path": "/Users/jameshabben/Documents/phone-images/magnet/00008101-0010541A1130001E_files_full-001.zip",
+      "os": "macOS-15.0-x86_64-i386-64bit",
+      "timestamp": "2024-10-29T11:06:55.527987",
+      "last_commit": {
+        "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+        "author_name": "Johann Polewczyk",
+        "author_email": "johannplw@me.com",
+        "date": "2024-10-17T19:06:04+02:00",
+        "message": "resolve kml error issue with 'all' in output_types"
+      }
+    },
+    "artifacts": {
+      "get_adId": {
+        "search_patterns": [
+          "*/containers/Shared/SystemGroup/*/Library/Caches/com.apple.lsdidentifiers.plist"
+        ],
+        "file_count": 1,
+        "expected_output": {
+          "headers": [],
+          "data": []
+        }
+      }
+    },
+    "image_name": "mvs_ios_2023"
+  }
+}

admin/test/results/advertisingID/advertisingID.get_adId.mvs_ios_2023.20241029180900.json

@@ -0,0 +1,32 @@
+{
+  "metadata": {
+    "module_name": "advertisingID",
+    "artifact_name": "Advertising Identifier",
+    "function_name": "get_adId",
+    "case_number": "mvs_ios_2023",
+    "number_of_columns": 2,
+    "number_of_rows": 1,
+    "total_data_size_bytes": 64,
+    "input_zip_path": "admin/test/cases/data/advertisingID/testdata.advertisingID.get_adId.mvs_ios_2023.zip",
+    "start_time": "2024-10-29T18:09:00.255342+00:00",
+    "end_time": "2024-10-29T18:09:00.336795+00:00",
+    "run_time_seconds": 0.02880382537841797,
+    "last_commit": {
+      "hash": "809b08c7dac89b2f2bcc7d6692b9981e3cf39913",
+      "author_name": "Johann Polewczyk",
+      "author_email": "johannplw@me.com",
+      "date": "2024-10-17T19:06:04+02:00",
+      "message": "resolve kml error issue with 'all' in output_types"
+    }
+  },
+  "headers": [
+    "Identifier",
+    "Data Value"
+  ],
+  "data": [
+    [
+      "Apple Advertising Identifier",
+      "A07BF489-9C5A-4915-9999-438B0A93D291"
+    ]
+  ]
+}

scripts/artifacts/advertisingID.py

@@ -28,8 +28,9 @@ def get_adId(files_found, report_folder, seeker, wrap_text, timezone_offset):
         for key, val in pl.items():
             if key == 'LSAdvertiserIdentifier':
                 data_list.append(('Apple Advertising Identifier', val))
-                id_values.append(f"<b>Apple Advertising Identifier: </b>{val}")
+                #id_values.append(f"<b>Apple Advertising Identifier: </b>{val}")
+                device_info("Advertising Identifier", "Apple Advertising Identifier", val)
 
-    device_info("Advertising Identifier", id_values)
+
     data_headers = ('Identifier', 'Data Value')
     return data_headers, data_list, source_path

scripts/artifacts/airdropId.py

@@ -28,8 +28,8 @@ def get_airdropId(files_found, report_folder, seeker, wrap_text, timezone_offset
         for key, val in pl.items():
             if key == 'LSAdvertiserIdentifier':
                 data_list.append('Airdrop ID', val)
-                id_values.append(f"<b>Airdrop ID: </b>{val}")
+                device_info("Airdrop", "Airdrop ID", val)
 
     data_headers = ('Identifer', 'Data Value')
-    device_info("Airdrop", id_values)
+
     return data_headers, data_list, source_path

scripts/artifacts/appleLocationd.py

@@ -28,11 +28,11 @@ def get_applelocationd(files_found, report_folder, seeker, wrap_text, timezone_o
         for key, val in pl.items():
             if key == 'LocationServicesEnabledIn8.0':
                 data_list.append(('Location Services Enabled', val))
-                id_values.append(f"<b>Location Services Enabled: </b>{val}")
+                device_info("Settings", "Location Services Enabled", val)
 
             elif key == 'LastSystemVersion':
                 data_list.append(('Last System Version', val))
-                id_values.append(f"<b>Last System Version: </b>{val}")
+                device_info("Settings", "Last System Version", val)
 
             elif key == 'steadinessClassificationNextClassificationTime' or key == 'VO2MaxCloudKitLastForcedFetch' \
                 or key == 'kP6MWDNextEstimateTime' or key == 'VO2MaxCloudKitManagerNextActivityTime':
@@ -41,6 +41,5 @@ def get_applelocationd(files_found, report_folder, seeker, wrap_text, timezone_o
             else:
                 data_list.append((key, val))
 
-    device_info("Settings", id_values)            
     data_headers = ('Key','Data' )     
     return data_headers, data_list, source_path

scripts/artifacts/appleWifiPlist.py

@@ -55,7 +55,7 @@
 
 import glob
 import plistlib
-from scripts.ilapfuncs import logfunc, logdevinfo, artifact_processor, convert_plist_date_to_timezone_offset
+from scripts.ilapfuncs import device_info, artifact_processor, convert_plist_date_to_timezone_offset
 
 def _bytes_to_mac_address(encoded_bytes):
     return ':'.join(f"{byte:02x}" for byte in encoded_bytes[:6])
@@ -77,7 +77,7 @@ def appleWifiKnownNetworks(files_found, report_folder, seeker, wrap_text, timezo
             deserialized = plistlib.load(f)
             if 'KeepWiFiPoweredAirplaneMode' in deserialized:
                 val = (deserialized['KeepWiFiPoweredAirplaneMode'])
-                logdevinfo(f"<b>Keep Wifi Powered Airplane Mode: </b>{val}")
+                device_info('WiFi', 'Keep Wifi Powered Airplane Mode', val)
 
             if 'List of known networks' in deserialized:
                 for known_network in deserialized['List of known networks']: