Merge pull request #54 from MasterScrat/fix-whatsapp-datetime-am-pm

added am/pm to datetime format parsing in whatsapp

Author: MasterScrat

parsers/whatsapp.py

@@ -13,7 +13,7 @@
 
 log = logging.getLogger(__name__)
 regex_left = r'[\u0000-\u001F\u0100-\uFFFF]?'
-regex_datetime = r'[^\w]?([0-9./\-]{6,10},?[\sT][0-9:]{5,8})[^\w]?\s[\-]?\s?'
+regex_datetime = r'[^\w]?([0-9./\-]{6,10},?[\sT][0-9:]{5,8}\s?[AP]?M?)[^\w]?\s?[\-\–]?\s'
 regex_right = r'(([^:]+):\s)?(.*)'
 regex_message = re.compile(f'^{regex_left}{regex_datetime}{regex_right}$')
 MAX_EXPORTED_MESSAGES = 1000000
@@ -26,7 +26,7 @@ def infer_datetime_regex(f_path, max_messages=100):
         for c, line in enumerate(f):
             if c == max_messages:
                 break;
-            matches = regex_message.search(line)
+            matches = regex_message.search(line.upper())
             if matches:
                 pattern = ""
                 first = True
@@ -50,6 +50,9 @@ def infer_datetime_regex(f_path, max_messages=100):
                         if l in '.*+[]{}()\\|':
                             pattern += '\\'
                         pattern += l
+                        if i > 0 and pattern[-2:] in ['AM','PM']:
+                            pattern = pattern[:-2] + '[APap][Mm]'
+                            last = len(pattern)
                 pattern = pattern[0:last] + ')' + pattern[last:]
                 patterns[pattern] += 1
     if len(patterns) > 0:

tests/test_data/whatsapp/_chat 4.txt

@@ -0,0 +1,3 @@
+09/17/19, 8:30:52 AM – John Doe: US datetime format
+09/17/19, 4:30:10 PM – John Doe: US datetime format
+09/18/19, 4:50:32 pm – John Doe: US datetime format

tests/test_whatsapp.py

@@ -58,6 +58,24 @@
             ]
         }
 
+ground_truth_chat4 = {
+        'datetime': [
+            datetime(2019, 9, 17, 8, 30, 52),
+            datetime(2019, 9, 17, 16, 30, 10),
+            datetime(2019, 9, 18, 16, 50, 32),
+            ],
+        'text': [
+            'US datetime format',
+            'US datetime format',
+            'US datetime format',
+            ],
+        'senderName': [
+            'John Doe',
+            'John Doe',
+            'John Doe',
+            ]
+        }
+
 
 def test_parse_chat_info_chat1():
     data = parse_messages([os.path.join(TEST_DATA_LOCATION, '_chat.txt')], 'John Doe', True)
@@ -83,3 +101,13 @@ def test_parse_us_datetime_chat3():
     assert len(df_truth) == len(df)
     for i, row in df.iloc[:len(df_truth)].iterrows():
         assert row.timestamp == df_truth.iloc[i].datetime.timestamp()
+
+def test_parse_us_datetime_chat4():
+    data = parse_messages([os.path.join(TEST_DATA_LOCATION, '_chat 4.txt')], 'John Doe', True)
+    df = pd.DataFrame(data, columns=config['ALL_COLUMNS'])
+    df_truth = pd.DataFrame(ground_truth_chat4)
+    assert len(df_truth) == len(df)
+    for i, row in df.iloc[:len(df_truth)].iterrows():
+        assert row.timestamp == df_truth.iloc[i].datetime.timestamp()
+        assert row.text == df_truth.iloc[i].text
+        assert row.senderName == df_truth.iloc[i].senderName