Process_Group_Users_2020.py

#============================================================================================================#
#*******************************   HOW TO RUN?  ************************************************************ #
#*********************************************************************************************************** #
# Example:                                                                                                   #
# python3 Process_Group_Users_2020.py /Users/rajattan/venmo/dummy.json ./transactions_date_wise.txt          #
#============================================================================================================#
import re
import os
import sys
import time
import json
import nltk
import pickle
import emoji
import enchant
import datetime
import os.path
import numpy as np
import pandas as pd
import tensorflow as tf
from nltk import tokenize
from langdetect import detect
from transformers import TFBertModel
from transformers import BertTokenizer
from tensorflow.keras.layers import Dense, Flatten

#===============================================================#
BATCH = 1000000
CHUNKSIZE = 10000
#===============================================================#
current = 0
numbatch = 0
transactions = 0
#===============================================================#
cnt = 0
stopwords = set()
date_category_stat = {}
date_personal_stat = {}
unique_s = {}
unique_r = {}

aa=set()
gambling = set()
moreaa=set()
filter_users = {}
unique = {}


#===============================================================#
USERS_FILE = "checkpoint_part1/PARTIAL_OUTPUT.txt"

UNIQUESENDERS = "checkpoint_2020/unique.txt"
CHECKPOINT_FILE = "checkpoint_2020/current.txt"
PATH_TO_STOPWORDS_LIST = "data/STOPWORDS.txt"

PATH_TO_AA_LIST = "data/AA.txt"
PATH_TO_GAMBLING_LIST = "data/GAMBLING.txt"
PATH_TO_MORE_AA_LIST = "data/MOREAA.txt"

#===============================================================#
english_ch = re.compile("[A-Za-z0-9]+")
email = re.compile("[^@]+@[^@]+\.[^@]+")
dat = re.compile("([0-9]{1,2}(/|-)[0-9]{1,2}(/|-)[0-9]{4})|([0-9]{1,2}(/|-)[0-9]{1,2})")
#===============================================================#

if(len(sys.argv) != 3):
    print("==========================================================================")
    print("SORRY!! Please provide the path to the INPUT json file and the OUTPUT file")
    print("==========================================================================")
    print("Example: python3 Process_Group_Users.py ./dummy.json ./output.txt         ")
    print("==========================================================================")
    sys.exit()

f = open(sys.argv[1])

if(os.path.exists(CHECKPOINT_FILE)):
    with open(CHECKPOINT_FILE, "rb") as myFile:
        current = pickle.load(myFile)
    print("RESUMING FROM " + str(current))
    if(current > 0):
        us = UNIQUESENDERS + "." + str(current)
        with open(us, "rb") as myFile:
            unique = pickle.load(myFile)

        if(len(unique) == 0):
            print("===================================================================")
            print("****** COULD NOT SUCCESSFULLY LOAD THE CONTENTS USING PICKLE.******")
            print("***                YOU NEED TO RECOMPUTE THINGS AGAIN.          ***")
            print("***    PLEASE remove the file checkpoint/current.txt and re-run.***")
            print("===================================================================")
            sys.exit()
        else:
            print("=========================================================")
            print(" CHECKPOINT FILES AND DICTIONARIES LOADED SUCCESSFULLY!!!")
            print("=========================================================")


with open(USERS_FILE, "rb") as myFile:
    filter_users = pickle.load(myFile)


if(len(filter_users) == 0):
    print("============================================================================")
    print("****** COULD NOT SUCCESSFULLY LOAD FILTER FILE  CONTENTS USING PICKLE.******")
    print("***                YOU NEED TO RECOMPUTE THINGS AGAIN.                   ***")
    print("***    PLEASE remove the file checkpoint/current.txt and re-run.         ***")
    print("============================================================================")
    sys.exit()
else:
    print("=========================================================")
    print(" CHECKPOINT FILTERED USER     FILE LOADED SUCCESSFULLY!!!")
    print("=========================================================")


#===============================================================#
with open(PATH_TO_STOPWORDS_LIST,'r') as fp:
    for l in fp:
        stopwords.add(l.strip())

#===============================================================#

"""
Convert all letters to lower or upper case (common : lower case)
"""
def convert_letters(tokens, style = "lower"):
    if (style == "lower"):
        return [token.lower() for token in tokens]
    else:
        return [token.upper() for token in tokens]
#===============================================================#
"""
Eliminate all continuous duplicate characters more than twice
"""
def reduce_lengthening(tokens):
    return [pattern.sub(r"\1\1", token) for token in tokens]
#===============================================================#
"""
Eliminate all continuous duplicate characters more than once
"""
def reduce_lengthening_rm1(tokens):
    return [pattern_rm1.sub(r"\1", token) for token in tokens]
#===============================================================#
"""
Remove all digits and special characters
"""
def remove_special(tokens):
  return [re.sub("(\\d|\\W)+", " ", token) for token in tokens]
#===============================================================#
"""
Remove blancs on words
"""
def remove_blanc(tokens):
    return [token.strip() for token in tokens]
#===============================================================#
"""
Email address regex
"""
def contains_email(note):
    if(email.search(note)):
        return True
    return False
#===============================================================#
"""
Stopwords Removal
"""
def remove_stopwords(tokens):
    return [token for token in tokens if token not in stopwords]
#===============================================================#
"""
Preprocessing Work 
"""
def preprocessing(origtokens):
    tokens = convert_letters(origtokens)
    tokens = reduce_lengthening(tokens)
    return tokens
#===============================================================#
"""
More Preprocessing Work 
"""
def preprocessing_cntd(tokens):
    tokens = remove_stopwords(tokens)
    tokens = remove_special(tokens)
    tokens = remove_blanc(tokens)
    tokens = [t for t in tokens if len(t) != 0]
    return tokens
#===============================================================#
with open(PATH_TO_GAMBLING_LIST,'r') as fp:
    for l in fp:
        gambling.add(''.join(convert_letters(l.strip())))


with open(PATH_TO_AA_LIST,'r') as fp:
    for l in fp:
        aa.add(''.join(convert_letters(l.strip())))

with open(PATH_TO_MORE_AA_LIST,'r') as fp:
    for l in fp:
        moreaa.add(''.join(convert_letters(l.strip())))



#===============================================================#
#   MAIN FLOW                                                   #
#===============================================================#
for chunk in pd.read_csv(sys.argv[1], chunksize=CHUNKSIZE, error_bad_lines=False):
    for row in chunk.itertuples():
        transactions = transactions + 1
        try:
            if(transactions < current or len(row) != 11):
                continue

            username = row[5]
            tusername = row[6]


            if(tusername not in filter_users):
                continue

            note = row[1]
            note = note.lower()

            if(tusername not in unique):
                unique[tusername] = {}
    
            recv = tusername
            sen = username
            AAflag = 0
            if("AA-U" in filter_users[tusername]['C'] or "AA-N" in filter_users[tusername]['C']):
                AAflag = 1
    
    
            if(username not in unique[tusername]):
                if(AAflag == 1):
                    unique[tusername][username] = {'TAA':0, 'AA':0,'Tradition':0,'Lunch Bunch':0,'Book Study':0,'Early Bird':0,'Eye Opener':0, 'Attitude':0, 'O' : 0, 'N':0,'T':0, '11 step':0, 'meeting':0, 'dues':0, 'donation':0, 'only emoji':0, 'greeting/gratitude':0, 'date':0}
                else:
                    unique[tusername][username] = {'TG':0,'Poker':0,'Gamble':0, 'Casino':0, 'betting':0, 'play':0, 'Lottery':0, 'date':0, 'greeting/gratitude':0, 'only emoji':0, 'N':0,'T':0, 'Email':0, 'Money':0}
    
            unique[tusername][username]['T'] += 1
            
    
            if(AAflag == 1):
                flag =0
                if("aa" in note or "alcoholics anonymous" in note):
                    unique[recv][sen]['AA'] += 1
                    flag = 1
        
                if("7 th tradition" in note or "7th tradition" in note or "7th" in note or "7" in note or "seventh" in note or "trad" in note):
                    unique[recv][sen]['Tradition'] += 1
                    flag = 1
        
                if("lunch bunch" in note):
                    unique[recv][sen]['Lunch Bunch'] += 1
                    flag = 1
        
                if("book study" in note):
                    unique[recv][sen]['Book Study'] += 1
                    flag = 1
        
                if("early bird" in note):
                    unique[recv][sen]['Early Bird'] += 1
                    flag = 1
        
                if("attitude" in note):
                    unique[recv][sen]['Attitude'] += 1
                    flag = 1
        
                if("eye opener" in note):
                    unique[recv][sen]['Eye Opener'] += 1
                    flag = 1
        
        
                if("11th step" in note or "eleventh step" in note or "11th" in note or "11 th" in note):
                    unique[recv][sen]['11 step'] += 1
                    flag = 1
        
                if("meeting" in note or "meetings" in note or "meet" in note):
                    unique[recv][sen]['meeting'] += 1
                    flag = 1
        
                if("dues" in note or "due" in note or "payment" in note):
                    unique[recv][sen]['dues'] += 1
                    flag = 1
        
                if("donate" in note or "donations" in note or "donation" in note or "contribute" in note or "contribution" in note or "contributing" in note):
                    unique[recv][sen]['donation'] += 1
                    flag = 1
        
                origtokens = nltk.word_tokenize(note)
                if(english_ch.search(note) is None):
                    fl = 0
                    for t in origtokens:
                        if(emoji.emoji_count(t) <= 0):
                            fl = 0
                            break
                        else:
                            fl = 1
                    if(fl == 1):
                        unique[recv][sen]['only emoji'] += 1
                        flag = 1
                if(dat.search(note) is not None):
                    unique[recv][sen]['date'] += 1
                    flag = 1
        
                origtokens = remove_blanc(origtokens)
                origtokens = remove_special(origtokens)
                if(flag == 0):
                    fl = 0
                    for t in origtokens:
                        if(not (t in moreaa)):
                            fl = 0
                            break
                        else:
                            fl = 1
                    if(fl == 1):
                        unique[recv][sen]['greeting/gratitude'] += 1
                        flag = 1
        
                if(flag == 1):
                    unique[recv][sen]['TAA'] += 1
                if(flag == 0):
                    for ll in aa:
                        if(ll in note):
                            unique[recv][sen]['O'] += 1
                            flag = 1
        
                            break
                if(flag == 0):
                    unique[recv][sen]['N'] += 1
    
            else:
                flag =0
                if("pok" in note):
                    unique[recv][sen]['Poker'] += 1
                    flag = 1
        
                if("casino" in note):
                    unique[recv][sen]['Casino'] += 1
                    flag = 1
        
        
                if("bet" in note or "betting" in note):
                    unique[recv][sen]['betting'] += 1
                    flag = 1
        
        
                if("gamble" in note or "gambling" in note):
                    unique[recv][sen]['Gamble'] += 1
                    flag = 1
        
                if("blackjack" in note or "black jack"  in note or "card"  in note or "play"  in note or "game" in note or "fantasy" in note or "ball" in note or "league" in note or "tournament" in note or "pool" in note or "dart" in note):
                    unique[recv][sen]['play'] += 1
                    flag = 1
        
                if("drawing" in note or "raffle" in note or "lottery" in note or "raffle" in note):
                    unique[recv][sen]['Lottery'] += 1
                    flag = 1
        
                if("money" in note or "dollar" in note or "ticket" in note or "package" in note or "vip" in note or "refund" in note):
                    unique[recv][sen]['Money'] += 1
                    flag = 1
        
                if(contains_email(note)):
                    unique[recv][sen]['Email'] += 1
                    flag = 1
        
                origtokens = nltk.word_tokenize(note)
                if(english_ch.search(note) is None):
                    fl = 0
                    for t in origtokens:
                        if(emoji.emoji_count(t) <= 0):
                            fl = 0
                            break
                        else:
                            fl = 1
                    if(fl == 1):
                        unique[recv][sen]['only emoji'] += 1
                        #flag = 1
                if(dat.search(note) is not None):
                    unique[recv][sen]['date'] += 1
                    flag = 1
        
                origtokens = remove_blanc(origtokens)
                origtokens = remove_special(origtokens)
                if(flag == 0):
                    fl = 0
                    for t in origtokens:
                        if(not (t in moreaa)):
                            fl = 0
                            break
                        else:
                            fl = 1
                    if(fl == 1):
                        unique[recv][sen]['greeting/gratitude'] += 1
                        flag = 1
        
                if(flag == 1):
                    unique[recv][sen]['TG'] += 1
                if(flag == 0):
                    unique[recv][sen]['N'] += 1
    
            if cnt == (BATCH-1):
                current = transactions
                numbatch = numbatch + 1
                cnt = -1
                if(numbatch % CHECKPOINT_INTERVAL == 0):
                    strcurrent = "." + str(current)
                    with open(CHECKPOINT_FILE, "wb") as myFile:
                        pickle.dump(current, myFile,protocol=pickle.HIGHEST_PROTOCOL)
                    sen = UNIQUESENDERS + strcurrent
                    with open(sen, "wb") as myFile:
                        pickle.dump(unique, myFile,protocol=pickle.HIGHEST_PROTOCOL)
    
            cnt = cnt + 1
    
    
        except Exception as e:
            continue        
f.close()

# Last batch

strcurrent = "." + str(transactions)
with open(CHECKPOINT_FILE, "wb") as myFile:
    pickle.dump(transactions, myFile,protocol=pickle.HIGHEST_PROTOCOL)
sen = UNIQUESENDERS + strcurrent
with open(sen, "wb") as myFile:
    pickle.dump(unique, myFile,protocol=pickle.HIGHEST_PROTOCOL)


outputfile1 = open(sys.argv[2] + "GAMBLING.txt","w")

scount = -1
for k,v in unique.items():
    tusername = k
    if("AA-U" in filter_users[tusername]['C'] or "AA-N" in filter_users[tusername]['C']):
        continue
    scount = scount + 1

    s = str(scount) + "|" + str(len(v)) + "|"
    #s = str(k) + "|" + str(len(v)) + "|"
    tot = {'TG':0,'Poker':0,'Gamble':0, 'Casino':0, 'betting':0, 'play':0, 'Lottery':0, 'date':0, 'greeting/gratitude':0, 'only emoji':0, 'N':0,'T':0, 'Email':0, 'Money':0}
    for kk,vv in sorted(v.items()):
        for kkk, vvv in sorted(vv.items()):
            tot[kkk] += vvv

    for key, val in sorted(tot.items()):
        s += key + "|" + str(val) + "|"

    per = int((tot['TG']/tot['T'])* 100.0)
    s += str(per) + "|"
    outputfile1.write(s + "\n")

outputfile1.close()

outputfile1 = open(sys.argv[2] + "AA.txt","w")
for k,v in unique.items():

    tusername = k 
    if(not("AA-U" in filter_users[tusername]['C'] or "AA-N" in filter_users[tusername]['C'])):
        continue
    scount = scount + 1
    s = str(scount) + "|" + str(len(v)) + "|"
    #s = str(k) + "|" + str(len(v)) + "|"
    tot = {'TAA':0, 'AA':0,'Tradition':0,'Lunch Bunch':0,'Book Study':0,'Early Bird':0,'Eye Opener':0, 'Attitude':0, 'O' : 0, 'N':0,'T':0, '11 step':0 ,'meeting':0, 'dues':0, 'donation':0, 'only emoji':0, 'greeting/gratitude':0 , 'date':0}
    for kk,vv in sorted(v.items()):
        for kkk, vvv in sorted(vv.items()):
            tot[kkk] += vvv

    for key, val in sorted(tot.items()):
        s += key + "|" + str(val) + "|"

    per = int((tot['TAA']/tot['T'])* 100.0)
    s += str(per) + "|"

    outputfile1.write(s + "\n")
outputfile1.close()