v2024.10.16

OneDriveExplorer/OneDriveExplorer.py

@@ -50,7 +50,7 @@
                     )
 
 __author__ = "Brian Maloney"
-__version__ = "2024.09.20"
+__version__ = "2024.10.16"
 __email__ = "bmmaloney97@gmail.com"
 rbin = []
 DATParser = dat_parser.DATParser()
@@ -350,7 +350,9 @@ def output():
                         print(f'\n\nParsing {key} OneDrive logs\n')
                         logs = v
                         odl = parse_odl(logs[0], key)
-                        log_output = f'{key}_logs.csv'
+                        if not args.csv:
+                            args.csv = '.'
+                        log_output = f'{args.csv}/{key}_logs.csv'
                         odl.to_csv(log_output, index=False)
 
     sys.exit()

OneDriveExplorer/ode/helpers/schema

@@ -49,7 +49,7 @@ cstruct = {
     }
 }
 
-# menu_data schema version 1.4
+# menu_data schema version 2.1
 menu_data = {
     'theme': {
         'required': True,
@@ -82,6 +82,10 @@ menu_data = {
     'odl': {
         'required': True,
         'type': 'boolean'
+    },
+	'odl_cor': {
+        'required': True,
+        'type': 'boolean'
     },
     'odl_save': {
         'required': True,
@@ -91,5 +95,21 @@ menu_data = {
         'required': True,
         'nullable': True,
         'type': 'string'
+    },
+	'create': {
+        'required': True,
+        'type': 'boolean'
+    },
+	'access': {
+        'required': True,
+        'type': 'boolean'
+    },
+	'modify': {
+        'required': True,
+        'type': 'boolean'
+    },
+	'size': {
+        'required': True,
+        'type': 'boolean'
     }
 }

OneDriveExplorer/ode/parsers/csv_file.py

@@ -61,7 +61,8 @@ def parse_csv(filename):
               'folderStatus': 'Int64',
               'Path': 'object',
               'shortcutVolumeID': 'Int64',
-              'shortcutItemIndex': 'Int64'
+              'shortcutItemIndex': 'Int64',
+              'hydrationCount': 'Int64'
               }
 
     try:
@@ -79,6 +80,10 @@ def parse_csv(filename):
                            'shortcutVolumeID', 'shortcutItemIndex', 'libraryType']]
         columns_to_fill = df_scope.columns.difference(['libraryType'])
         df_scope[columns_to_fill] = df_scope[columns_to_fill].fillna('')
+
+        if 'remotePath' in df_scope:
+            df_scope.remotePath.fillna('', inplace=True)
+
         scopeID = df_scope['scopeID'].tolist()
 
         if 'inRecycleBin' in df.columns:
@@ -107,9 +112,31 @@ def parse_csv(filename):
             df_GraphMetadata_Records[json_columns] = df_GraphMetadata_Records[json_columns].map(lambda x: ast.literal_eval(x) if pd.notna(x) and x.strip() else '')
             df_GraphMetadata_Records['lastWriteCount'] = df_GraphMetadata_Records['lastWriteCount'].astype('Int64')
 
-        df = df.astype(object)
-        df = df.where(pd.notna(df), None)
+            if 'fileExtension' in df_GraphMetadata_Records:
+                df_GraphMetadata_Records.fileExtension.fillna('', inplace=True)
+
         df.drop(columns=columns_to_drop_2, inplace=True)
+        df.localHashDigest.fillna('', inplace=True)
+        if 'notificationTime' in df:
+            df.HydrationTime.fillna('', inplace=True)
+
+        if 'firstHydrationTime' in df:
+            df.lastHydrationType.fillna('', inplace=True)
+            df.firstHydrationTime.fillna('', inplace=True)
+            df.lastHydrationTime.fillna('', inplace=True)
+
+        if 'diskLastAccessTime' in df:
+            df.diskLastAccessTime.fillna('', inplace=True)
+
+        if 'diskCreationTime' in df:
+            df.diskCreationTime.fillna('', inplace=True)
+
+        if 'lastKnownPinState' in df:
+                    df.lastKnownPinState = df.lastKnownPinState.apply(lambda x: '' if pd.isna(x) else int(x))
+
+        if 'remotePath' in df:
+            df.remotePath.fillna('', inplace=True)
+
     except Exception as e:
         print(e)
         log.error(f'Not a valid csv. {csv_name}')

OneDriveExplorer/ode/parsers/dat.py

@@ -372,8 +372,6 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
                             block = self.datstruct.DAT_FOLDER_v29_v2c(f.read(chunk))
                         else:
                             block = self.datstruct.DAT_FOLDER_v2d_v36(f.read(chunk))
-                        block._values.update([('sharedItem', '')])
-                        print(dict(block._values))
 
                     elif ff == '09':
                         data_type = 'Scope'
@@ -441,10 +439,10 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
                         csvwriter = csv.writer(temp_files, escapechar='\\')
                         if version <= '2b':
                             self.merge_dicts(block._values, self.dict_1)
-                        if version >= '2c' and version <= '34':
+                        elif version >= '2c' and version <= '34':
                             self.merge_dicts(block._values, self.dict_2)
                             block._values['sharedItem'] = [*block.bitMask][29]
-                        if version >= '35':
+                        elif version >= '35':
                             self.merge_dicts(block._values, self.dict_3)
                             block._values['sharedItem'] = [*block.bitMask][29]
                         if not self.files_header:
@@ -453,6 +451,7 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
 
                     if data_type == 'Folder':
                         csvwriter = csv.writer(temp_folders, escapechar='\\')
+                        block._values['sharedItem'] = [*block.bitMask][29]
                         if not self.folders_header:
                             csvwriter.writerow(dict(block._values))
                             self.folders_header = True
@@ -473,7 +472,7 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
 
         except Exception as e:
             # log.error(e)
-            return pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), []
+            return pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), [], 0
 
         if not gui:
             print('\n')

OneDriveExplorer/ode/parsers/onedrive.py

@@ -91,6 +91,8 @@ def find_parent(self, x, id_name_dict, parent_dict):
     # Generate scopeID list instead of passing
     def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_path, rbin_df, account=False, reghive=False, recbin=False, localHashAlgorithm=False, gui=False, pb=False, value_label=False):
 
+        allowed_keys = ['scopeID', 'siteID', 'webID', 'listID', 'tenantID', 'webURL', 'remotePath', 'MountPoint', 'spoPermissions', 'shortcutVolumeID', 'shortcutItemIndex']
+
         df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '')
         df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')
 
@@ -184,7 +186,6 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
         df['volumeID'] = df['volumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')
         df['shortcutVolumeID'] = df['shortcutVolumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '')
         df['shortcutVolumeID'] = df['shortcutVolumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')
-
 
         cache = {}
         final = []
@@ -195,21 +196,26 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
             df_GraphMetadata_Records.set_index('resourceID', inplace=True)
 
         column_len = len(df.columns)
-       
+
         for row in df.sort_values(
             by=['Level', 'parentResourceID', 'Type', 'FileSort', 'FolderSort', 'libraryType'],
                 ascending=[False, False, False, True, False, False]).to_dict('records'):
             if row['Type'] == 'File':
                 try:
-                    if column_len == 32:
-                        file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'size', 'localHashDigest', 'sharedItem', 'Media')}
-
-                    if column_len == 33:
-                        file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'HydrationTime', 'size', 'localHashDigest', 'sharedItem', 'Media')}
-
-                    if column_len == 36:
+                    if column_len == 39:
+                        file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'lastHydrationType', 'lastKnownPinState','spoPermissions', 'volumeID', 'itemIndex', 'diskLastAccessTime', 'diskCreationTime', 'lastChange', 'firstHydrationTime', 'lastHydrationTime', 'hydrationCount', 'size', 'localHashDigest', 'sharedItem', 'Media')}
+                    if column_len == 37:
+                        file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'lastHydrationType', 'spoPermissions', 'volumeID', 'itemIndex', 'diskLastAccessTime','lastChange', 'firstHydrationTime', 'lastHydrationTime', 'hydrationCount', 'size', 'localHashDigest', 'sharedItem', 'Media')}
+
+                    elif column_len == 36:
                         file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'lastHydrationType', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'firstHydrationTime', 'lastHydrationTime', 'hydrationCount', 'size', 'localHashDigest', 'sharedItem', 'Media')}
 
+                    elif column_len == 33:
+                        file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'HydrationTime', 'size', 'localHashDigest', 'sharedItem', 'Media')}
+
+                    elif column_len == 32:
+                        file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'size', 'localHashDigest', 'sharedItem', 'Media')}
+
                 except Exception as e:
                     if gui:
                         log.error(f'Unable to read dataframe. Something went wrong. {e}')
@@ -234,8 +240,7 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
                 if 'Scope' in row['Type']:
                     if row['scopeID'] not in scopeID:
                         continue
-                    scope = {key: row[key] for key in (
-                             'scopeID', 'siteID', 'webID', 'listID', 'tenantID', 'webURL', 'remotePath', 'MountPoint', 'spoPermissions', 'shortcutVolumeID', 'shortcutItemIndex')}
+                    scope = {key: row[key] for key in row if key in allowed_keys}
                     folder = cache.get(row['scopeID'], {})
                     temp = {**scope, **folder}
                     final.insert(0, temp)
@@ -246,8 +251,7 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
                     if row['resourceID'] in scopeID:
                         scopeID.remove(row['resourceID'])
                         for s in df_scope.loc[df_scope['scopeID'] == row['resourceID']].to_dict('records'):
-                            scope = {key: s[key] for key in (
-                                     'scopeID', 'siteID', 'webID', 'listID', 'tenantID', 'webURL', 'remotePath')}
+                            scope = {key: s[key] for key in s if key in allowed_keys}
                             scope['MountPoint'] = row['MountPoint']
                             scope['spoPermissions'] = s['spoPermissions']
                             scope['shortcutVolumeID'] = s['shortcutVolumeID']