exmsearch

#!/usr/bin/python2.7
# -*- coding: utf-8 -*-

import urwid
import datetime
import io
import os
import subprocess
import sys
import json
import re
import shlex
import gzip
import time
import logging
import collections
import socket
import threading
import getpass
from itertools import islice
from multiprocessing import Pool, Queue, current_process, connection, cpu_count
from multiprocessing.pool import ThreadPool
from datetime import datetime, timedelta
os.nice(10)
FORMAT = '%(lineno)d :: %(funcName)s :: %(message)s'
if getpass.getuser() == 'root':
    logpath = '/root/eximsearch.log'
else:
    logpath = '/home/' + getpass.getuser() + '/eximsearch.log'
logging.basicConfig(
    format=FORMAT,
    filename=logpath,
    filemode='a',
    level=logging.DEBUG)
ACTIVE = 'active'
PREV = 'prev'
BODY = 'body'
HEADER = 'header'
FOOTER = 'footer'

urwid.set_encoding("UTF-8")
info = logging.info
debug = logging.debug
warning = logging.warning

PYTHONIOENCODING = "utf-8"
debug('Encoding Type: %s', PYTHONIOENCODING)

"""
SETTINGS / DEFAULT VALUE CLASSES
"""


class GlobalSettings():
    def __init__(self):
        """This class contains general settings / default variables for the application
        """
        self.dt = DateTimeSettings()
        self.lf = LogFileSettings()
        self.rl = ResultListSettings()
        self.df = DisplayFrameSettings()
        self.menus = Menus()
        self.hostname = socket.gethostname()
        self.menuEnabled = True
        self.divChars = 1
        self.filterGuide = (
            "You can have multiple filters per filter "
            "type(ie. multiple senders, multiple recipients, etc) "
            "and these will be filtered as OR. \nEx: "
            "Two sender filters of user1@domain.com and user2@domain.com, "
            "will show all query results that were sent by user1@domain.com "
            "OR user2@domain.com.\nFilters of different types are filtered "
            "as AND. Ex: Sender filter user1@domain.com and Date filter "
            "of 2019-05-24, will show all query results that were sent by "
            "user1@domain.com on the date 2019-05-24\nAcceptable "
            "Date-time Formats:MM-DD-YYYY or MM-DD-YYYY_HH:MM:SS\n"
            "Date Range formats: Start date,End date with NO SPACES. "
            "Ex: 2019-05-01,2019-05-31\nMessage Type Options: "
            "Incoming, Outgoing, Local"
                )
        self.filterTypes = [
            'sender',
            'recipient',
            'date',
            'type'
        ]

    def dyn_debug(self, *args):
        # print('Current User: ' + getpass.getuser())
        # queryLogProcess2(['wolfindva.com','/var/log/exim_mainlog-20190421'])
        with open('debug.json', 'r') as debug_file:
            x = json.load(debug_file, 'utf-8')
            try:
                eval(x['command'])
            except Exception as e:
                debug('Exception is : %s', e)
            else:
                pass
                return eval(x['command'])

    def unhandled_input(self, key):
        if type(key) == str:
            if key in 'ctrl e':
                views.activate('quit_loop', focus_position=BODY)
            if 'ctrl n' in key:
                views.activate('new_search', focus_position=BODY)
            if 'ctrl f' in key:
                views.activate('add_remove_filters', focus_position=BODY)
            if key in ('B', 'b'):
                if 'single_entry' in state.get_view_name(ACTIVE):
                    views.activate('results_list', focus_position='body')
            if key in 'tab':
                if frame.focus_position == 'footer':
                    frame.focus_position = 'body'
                else:
                    if self.menuEnabled:
                        frame.focus_position = 'footer'
            if 'home' in key:
                    state.go_back()
            if 'end' in key:
                state.go_forward()
            if key in 'f5':
                debug('Current View: %s', state.get_view_name(ACTIVE))
                if hasattr(state, 'prev_view'):
                    if state.prev_view:
                        debug('Previous View: %s', state.get_view_name(PREV))
            if key in 'f6':
                debug('Active Filters: %s', state.get_active_filters())
            if key in 'f7':
                debug('Current Query: %s', state.get_query(ACTIVE))
                if hasattr(state, 'prev_query'):
                    debug('Previous Query: %s', state.get_query(PREV))
            if key in 'f3':
                self.dyn_debug()
                # debug('Body: %s', self.dyn_debug())
            if key in 'f1':
                debug(
                    'Current Result List: %s',
                    state.get_result_list_name(ACTIVE))
                if hasattr(state, 'prev_result_list'):
                    debug(
                        'Previous Result List: %s',
                        state.get_result_list_name(PREV))
            if key in 'f2':
                debug(
                    'Current View Chain Position: %s',
                    state.get_view_chain_pos())
                debug(
                    'Current View Chain Name: %s',
                    state.get_view_from_chain(
                        state.get_view_chain_pos()).view_name)


class DateTimeSettings():
    def __init__(self):
        """Settings for DateTime functions / formatting
        """
        self.logDateFormat = "%Y-%m-%d"
        self.displayDateFormat = "%m-%d-%Y"
        self.stringToDateFormat = "%Y-%m-%d"
        self.logDateTimeFormat = "%Y-%m-%d_%H:%M:%S.%f"
        self.displayDateTimeFormat = "%m-%d-%Y_%H:%M:%S"

    def stringToDate(self, newFilter):
        try:
            datetime.strptime(newFilter, self.displayDateTimeFormat)
        except ValueError:
            try:
                datetime.strptime(newFilter, self.displayDateFormat)
            except:
                return False
            else:
                return datetime.strptime(newFilter, self.displayDateFormat)
        else:
            return datetime.strptime(newFilter, self.displayDateTimeFormat)


class DisplayFrameSettings():
    def __init__(self):
        """Settings & Defaults for the Applications Interface
        """
        self.mainTitle = 'Exim Search Utility'
        self.mainSubTitle = 'If you can do it better, then do it'
        self.palette = [
            # Name  , foreground,  background
            ('header',  'black',    'light gray'),
            ('footer',  'black',    'light gray'),
            ('body',    'white',    'default'),
            ('bold',    'dark green, bold', 'black'),
            ('blue', 'bold', 'dark blue'),
            ('highlight', 'dark green', 'default'),
            ('alert', 'dark red, bold', 'default'),
        ]


class Menus():
        def __init__(self):
            self.choose_logs = [
                ['(Q)uit', 'quit_loop']]

            self.coming_soon = [
                ['         (H)ome         ', 'home']]
            self.stat_summary_results = self.coming_soon

            self.home = [
                ['      New Search      ', 'new_search'],
                [' Add / Remove Filters ', 'add_remove_filters'],
                ['  cPanel User Stats   ', 'stat_summary'],
                [' Change Log Selection ', 'choose_logs'],
                ['          Quit        ', 'quit_loop']]
            self.new_search = self.home
            self.search_progress = self.home
            self.stat_summary = self.home

            self.results_list = [
                ['Filter Current Results', 'add_remove_filters'],
                [' Clear Applied Filters', 'clear_applied_filters'],
                ['         Home         ', 'home'],
                ['      New Search      ', 'new_search'],
                ['          Quit        ', 'quit_loop']]
            self.results_summary = self.results_list
            self.apply_filters = self.results_list
            self.clear_applied_filters = self.results_list

            self.single_entry = [
                ['  Back To Result List ', 'results_list'],
                ['         Home         ', 'home'],
                ['      New Search      ', 'new_search'],
                ['          Quit        ', 'quit_loop']]
            self.add_remove_filters = [
                ['Apply Current Filters ', 'apply_filters'],
                ['  Back To Result List ', 'results_list'],
                ['         Home         ', 'home'],
                ['      New Search      ', 'new_search'],
                ['          Quit        ', 'quit_loop']]
            self.add_remove_filters_no_apply = [
                [' New Search ', 'new_search'],
                ['    Home    ', 'home'],
                ['    Quit    ', 'quit_loop']]
            self.quit_loop = []
            self.legend = [
                ['Prev. Screen', 'Home'],
                ['Next  Screen', 'End'],
                ['Change Fields', '← / ↑ / ↓ / →'],
                ['Move to / from Menu', 'Tab'],
                ['New Search', 'Ctrl + N'],
                ['Add / Remove Filters', 'Ctrl + F'],
                ['Exit', 'Ctrl + E']
            ]


class LogFileSettings():
    def __init__(self):
        """Settings for the LogFile class / objects
        """
        self.dir = '/var/log/'
        self.mainLogName = 'exim_mainlog'
        self.mainLogPath = os.path.join(self.dir, self.mainLogName)
        self.selectedLogs = []


class ResultListSettings():
    def __init__(self):
        """Settings specific to the ResultList view
        """
        self.ButtonColWidth = 7
        self.divChars = 1
        self.resultOverflow = False
s = GlobalSettings()
"""
CUSTOM WIDGET CLASSES
"""


class ButtonLabel(urwid.SelectableIcon):
    def __init__(self, text):
        """Subclassing for urwid.Button's labeling
           This customization removes the cursor from
           the active button
           This should only need to be called by the
           FixedButton class.

        Arguments:
            urwid {class} -- urwid base class
            text {str} -- Button Label
        """
        curs_pos = len(text) + 1
        urwid.SelectableIcon.__init__(
            self,
            text,
            cursor_position=curs_pos)


class FixedButton(urwid.Button):
    """SubClass of the urwid.Button class used
       along with ButtonLabel in order to customize
       the appearance and behavior of buttons.

    Arguments:
        urwid {class} -- urwid base class

    Returns:
        urwid.Button -- a standard urwid Button
    """
    _selectable = True
    signals = ["click"]

    def __init__(self, thisLabel, on_press=None, user_data=None):
        """Creates a new Button

        Arguments:
            thisLabel {text} -- Button Label

        Keyword Arguments:
            on_press {callback} -- function to be executed on click
                (default: {None})
            user_data {tuple} -- tuple (or list) that contains any arguments
                 or data to be passed to on_ress function  (default: {None})
        """
        self._label = ButtonLabel(thisLabel)
        # you could combine the ButtonLabel object with other widgets here
        self.user_data = user_data
        self.on_press = on_press
        display_widget = self._label
        urwid.WidgetWrap.__init__(
            self,
            urwid.AttrMap(
                display_widget,
                None,
                focus_map="header"))
        self.callback = on_press

    def keypress(self, size, key):
        """Overrides default urwid.Button.keypress method

        Arguments:
            size {int} -- size of widget
            key {bytes or unicode} -- [a single keystroke value]

        Returns:
            None or Key -- [None if key was handled by this widget or
                            key (the same value passed) if key was
                            not handled by this widget]
        """
        if key in ('enter', 'space'):
            if self.user_data is not None:
                self.callback(self.user_data)
            else:
                self.callback()
        else:
            return key

    def set_label(self, new_label):
        """Method to allow changing the button's label

        Arguments:
            new_label {[str]} -- [New Button Label]
        """
        self._label.set_text(str(new_label))

    def mouse_event(self, size, event, button, col, row, focus):
        """
        handle any mouse events here
        and emit the click signal along with any data
        """
        pass

    def disable(self):
        """Function to allow the disabling of the button"""
        _selectable = False

    def enable(self):
        """Function to allow the enabling of a disabled button"""
        _selectable = True


class QuestionBox(urwid.Filler):
    def keypress(self, size, key):
        if key != 'enter':
            return super(QuestionBox, self).keypress(size, key)
        user_input = self.original_widget.get_edit_text()
        if len(user_input) >= 255:
            self.original_widget.set_caption(
                'Input Must be 255 Characters or Less\nPlease Try Again\n')
            self.original_widget.set_edit_text('')
        else:
            entry = self.original_widget.get_edit_text()
            self.original_widget.set_edit_text('')
            debug('%s Entry String: %s', self.original_widget, entry)
            state.set_query(entry)
            views.activate(
                'search_progress',
                is_threaded=True,
                on_join=search.new)


class UserBox(urwid.Filler):
    def keypress(self, size, key):
        if key != 'enter':
            return super(UserBox, self).keypress(size, key)
        entry = self.original_widget.get_edit_text()
        self.original_widget.set_edit_text('')
        debug('%s Entry String: %s', self.original_widget, entry)
        state.cpanel_user = entry
        state.list_of_users_emails = uapi.list_email_accounts(entry)
        if not state.cpanel_user:
            self.original_widget.set_caption(
                'That is not a valid cPanel User\n Please Try Again\n')
        else:
            if not state.list_of_users_emails:
                self.original_widget.set_caption(
                    state.cpanel_user +
                    ' user does not have any email addresses\n')
            else:
                views.activate(
                    'search_progress',
                    is_threaded=True,
                    on_join=search.cpanel_summary)


class CustomButton(urwid.Button):
    button_left = urwid.Text('[')
    button_right = urwid.Text(']')


class BoxButton(urwid.WidgetWrap):
    _border_char = u'─'

    def __init__(self, label, on_press=None, user_data=None, enabled=True):
        padding_size = 2
        border = self._border_char * (len(label) + padding_size * 2)
        self.cursor_position = len(border) + padding_size
        self.top = u'┌' + border + u'┐\n'
        self.middle = u'│  ' + label + u'  │\n'
        self.bottom = u'└' + border + u'┘'
        self.on_press_action = on_press
        self.on_press_user_data = user_data
        self.enabled = enabled
        # self.widget = urwid.Text([self.top, self.middle, self.bottom])
        self.widget = urwid.Pile([
            urwid.Text(self.top[:-1], align='center'),
            urwid.Text(self.middle[:-1], align='center'),
            urwid.Text(self.bottom, align='center'),
        ])

        self.widget = urwid.AttrMap(self.widget, '', 'highlight')

        # self.widget = urwid.Padding(self.widget, 'center')
        # self.widget = urwid.Filler(self.widget)

        # here is a lil hack: use a hidden button for evt handling
        # debug('on_press: %s, user_data: %s', )
        self._hidden_btn = urwid.Button(
            'hidden %s' % label,
            on_press,
            user_data)

        super(BoxButton, self).__init__(self.widget)

    def selectable(self):
        if self.enabled:
            return True
        else:
            return False

    def disable(self):
        self.enabled = False

    def enable(self):
        self.enabled = True

    def keypress(self, *args, **kw):
        return self._hidden_btn.keypress(*args, **kw)

    def mouse_event(self, *args, **kw):
        return self._hidden_btn.mouse_event(*args, **kw)


class MyWidgets():
    """A collection of functions to simplify creation of
       frequently used widgets """

    def __init__(self):
        self.div = urwid.Divider(' ', top=0, bottom=0)
        self.blankFlow = self.getText('body', '', 'center')
        self.blankBox = urwid.Filler(self.blankFlow)
        self.searchProgress = urwid.ProgressBar(
            'body',
            'header',
            current=0,
            done=100,
            satt=None)

    def getDiv(self, divider=' '):
        return urwid.Divider(divider, top=0, bottom=0)

    def getButton(self,
                  thisLabel,
                  callingObject,
                  callback,
                  user_data=None,
                  buttonMap='bold',
                  focus_map='header'):
        """Creates and returns a FixedButton object.

        Arguments:
            thisLabel {[str]} -- Label of the Button
            callingObject {obj} -- The name of the object that the
                                    callback belongs to
            callback {function} -- [function to be executed when button
                                    is clicked]

        Keyword Arguments:
            user_data {tuple} -- A tuple or list of arguments or data
                                to be passed to the callback function
                                (default: {None})
        Returns:
            FixedButton -- A FixedButton object
            FLOW WIDGET
        """

        button = FixedButton(
            str(thisLabel),
            on_press=getattr(callingObject, callback),
            user_data=user_data)
        button._label.align = 'center'
        buttonMap = urwid.AttrMap(button, buttonMap, focus_map=focus_map)
        return buttonMap

    def get_custom_button(self, *args, **kwargs):
        b = CustomButton(*args, **kwargs)
        b = urwid.AttrMap(b, '', 'highlight')
        b = urwid.Padding(b, left=4, right=4)
        return b

    def getText(self, format, textString, alignment, **kwargs):
        """Creates a basic urwid.Text widget

        Arguments:
            format {str} -- Name of a format attribute specified in
                            DisplayFrameSettings.pallette
            textString {str} -- The text string contents of text widget
            alignment {str} -- Text alignment (left, right, center)

        Returns:
            urwid.Text -- An urwidText Widget
            FLOW WIDGET
        """
        return urwid.Text(
            (format, textString),
            align=alignment,
            wrap='space',
            **kwargs)

    def getColRow(self, items, dividechars=s.divChars, **kwargs):
        """Creates a single row of columns

        Arguments:
            items {list} -- List of widgets, each item forming one column.
                             Items may be tuples containing width specs

        Returns:
            [urwid.Column] -- An urwid.Columns object
            FLOW / BOX WIDGET
        """
        return urwid.Columns(
            items,
            dividechars=dividechars,
            focus_column=None,
            min_width=1,
            box_columns=None)

    def getLineBox(self,
                   contents,
                   title,
                   tlcorner='┌',
                   tline='─',
                   lline='│',
                   trcorner='┐',
                   blcorner='└',
                   rline='│',
                   bline='─',
                   brcorner='┘',
                   **kwargs):
        """ Creates a SimpleFocusListWalker using contents as the list,
            adds a centered title, and draws a box around it. If the contents
            are not a list of widgets, then set content_list to False.

            The character that is used to draw the border can
            be adjusted with the following keyword arguments:
                tlcorner,tline,trcorner,blcorner,rline,bline,brcorner

        Arguments:
            contents {widget} -- an original_widget, no widget lists -
            title {string} -- Title String

        Keyword Arguments:
            content_list -- If true, the value of contents must be a list
                            of widgets
                            If false, the value must be a single widget to be
                            used as
                            original_widget -- default{False}

        Returns:
            urwid.LineBox -- urwid.LineBox object
            FLOW / BOX WIDGET
        """
        return urwid.LineBox(
            contents,
            title=str(title),
            title_align='center',
            tlcorner=tlcorner,
            tline=tline,
            lline=lline,
            trcorner=trcorner,
            blcorner=blcorner,
            rline=rline,
            bline=bline,
            brcorner=brcorner)

    def getListBox(self, contents):
        """Creates a ListBox using a SimpleFocusListWalker, with the contents
           being a list of widgets

        Arguments:
            contents {list} -- list of widgets

        Returns:
            list -- [0]: urwid.ListBox
                    [1]: urwid.SimpleFocusListWalker - Access this to make
                               changes to the list
                               which the SimpleFocusListWalker will follow.
        BOX WIDGET
        """
        walker = urwid.SimpleFocusListWalker(contents)
        listBox = urwid.ListBox(walker)
        return [listBox, walker]

    def getCheckBox(self, label, on_state_change=None, user_data=None):
        """gets an individual CheckBox item that executes the specified function
            with each change of state.

        Arguments:
            label {str} -- Checkbox item label

        Keyword Arguments:
            on_state_change {list} -- a list of the following [calling object,
            function] (default: {None})
            user_data {list} -- list of values to be bassed to function as
            arguments (default: {None})

        Returns:
            object -- urwid.CheckBox object
            FLOW WIDGET
        """
        return urwid.CheckBox(
            label,
            state=False,
            has_mixed=False,
            on_state_change=getattr(
                on_state_change[0],
                on_state_change[1]),
            user_data=user_data)

    def centeredListLineBox(self, contents, title, listHeight, **kwargs):
        filler = urwid.Filler(contents, height=listHeight)
        insideCol = w.getColRow([
            w.blankBox,
            ('weight', 2, filler),
            w.blankBox])
        lineBox = w.getLineBox(insideCol, title)
        outsidefiller = urwid.Filler(lineBox, height=listHeight)
        outsideCol = w.getColRow([
            w.blankBox,
            ('weight', 2, outsidefiller),
            w.blankBox])
        return urwid.Filler(outsideCol, height=listHeight)

    def getHeaderWidget(self,
                        title=s.df.mainTitle,
                        subtitle=s.df.mainSubTitle):
        """Generates a basic header with a title and optional subtitle
           This is meant to be used exclusively by the Headers.new() method

        Arguments:
            title {str} -- Title String

        Keyword Arguments:
            subtitle {str} -- Optional Sub-Title (default: {''})

        Returns:
            object -- urwid.Pile object to be used as the header's widget
            FLOW WIDGET
        """
        self.title = self.getText('bold', title, 'center')
        self.subtitle = self.getText('bold', subtitle, 'center')
        titleMap = urwid.AttrMap(self.title, 'bold')
        divMap = urwid.AttrMap(self.div, 'body')
        if subtitle:
            subtitleMap = urwid.AttrMap(self.subtitle, 'bold')
            return urwid.Pile((titleMap, subtitleMap, divMap), focus_item=None)
        else:
            return urwid.Pile((titleMap, divMap), focus_item=None)

    def getSearchProgress(self):
        return urwid.ProgressBar(
            'body',
            'header',
            current=0,
            done=100,
            satt=None)

    def getFooterWidget(self, view_instance, menuItems):
        """Generates a footer column row containing a list of buttons for a
            basic menu / navigation. This is meant to be used exclusively by
            the Footers.new() method

        Arguments:
            menuItems {list} -- List of Menu Items (each item is a list)
                                in the following format:
                                [
                                    [Label,callback function]
                                    [Label,callback function]
                                ]
        Returns:
            object -- urwid.menuItems object to be used as the header's widget
            FLOW WIDGET
        """
        debug('Getting Footer')
        menuList = []
        menuGridList = []
        for item in menuItems:
            if len(item) == 3:
                menuGridList.append(
                    BoxButton(
                        item[0],
                        on_press=views.activate,
                        user_data=(item[1], item[2])))
            else:
                menuGridList.append(
                    BoxButton(
                        item[0],
                        on_press=views.activate,
                        user_data=(item[1])))
        itemWidths = []
        for item in menuGridList:
            itemWidths.append(item.cursor_position)
        itemWidths.sort()
        menuColumns = urwid.Columns(
            menuList,
            dividechars=1,
            focus_column=None,
            min_width=1,
            box_columns=None)
        if itemWidths:
            menuGrid = urwid.GridFlow(
                menuGridList,
                itemWidths[-1], 0, 0, 'center')
        else:
            menuGrid = w.div
        debug(
            'menuCol width: %s, menuPadding width:',
            menuColumns.column_widths)
        legendItems = []
        for legend in s.menus.legend:
            legendItems.append(w.getText('bold', legend[0], 'center'))
        legendGrid = urwid.GridFlow(legendItems, 21, 0, 0, 'center')
        legendGridMap = urwid.AttrMap(legendGrid, 'bold')
        legendItems = []
        for legend in s.menus.legend:
            legendItems.append(w.getText('highlight', legend[1], 'center'))
        legendItemsGrid = urwid.GridFlow(legendItems, 21, 0, 0, 'center')
        legendItemsMap = urwid.AttrMap(legendItemsGrid, 'highlight')
        return urwid.Pile([menuGrid, legendGridMap, legendItemsMap])
w = MyWidgets()


class BodyWidgets():
    def get_body_widget(self, view_name, user_args=None, calling_view=None):
        widget_getter = getattr(self, 'get_' + view_name)
        body_widget = widget_getter(
            user_args=user_args,
            calling_view=calling_view)
        return body_widget

    def get_choose_logs(self, **kwargs):
        """Page opened on application start to select the
           logs that will be used in searches / filters
        """
        debug(' kwargs: %s', kwargs)
        logCheckBoxes = [w.div]
        for log in logFiles.availableLogs:
            logCheckBoxes.append(
                w.getCheckBox(
                    log,
                    on_state_change=[logFiles, 'update'],
                    user_data=[log])
                    )
        logCheckBoxes.append(
            BoxButton('Continue', on_press=views.activate, user_data='home'))
        listBox = w.getListBox(logCheckBoxes)[0]
        chooseLogsBox = w.centeredListLineBox(
            listBox,
            'Choose Your Logs to Search',
            len(logCheckBoxes) + 4)
        return chooseLogsBox

    def get_home(self, **kwargs):
        """Page displayed as Home Page for the application
        """
        debug(' kwargs : %s', kwargs)
        homeText = w.getText(
            'body',
            'Welcome to the best Exim Search Utility ever created. ' +
            'Select an option below to begin.',
            'center')
        if not s.lf.selectedLogs:
            debug(' No logs selected, returning to choose_logs ')
            return self.get_choose_logs()
        else:
            debug('Logs Selected: %s', s.lf.selectedLogs)
            logFiles.get_date_range()
            return urwid.Filler(homeText, 'middle')

    def get_new_search(self, **kwargs):
        """Page opened when starting an entirely new
           search. Not used for revising or filtering
           previous searches
        """
        debug(' kwargs : %s', kwargs)
        selectQuery = urwid.Edit('Enter your query below\n', align='center')
        selectFiller = QuestionBox(selectQuery, 'middle')
        return w.centeredListLineBox(selectFiller, 'New Search Query', 5)

    def get_search_progress(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        searchingStatus = urwid.Pile([
            w.getText(
                'body',
                'Processing Your Request. Please wait....',
                'center'),
            w.searchProgress
            ])
        statusFiller = urwid.Filler(searchingStatus, 'middle')
        return w.centeredListLineBox(statusFiller, '', 10)

    def get_results_summary(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        query = state.get_query(ACTIVE)
        result_list = state.get_result_list(ACTIVE)
        debug('resultOverFlow = %s', s.rl.resultOverflow)
        if s.rl.resultOverflow:
            if state.get_query:
                summaryRows = [w.getText('header', ' for ' + query, 'center')]
            else:
                summaryRows = []
            summaryRows.extend([
                w.div,
                w.getText(
                    'bold',
                    'There are too many Results \n Only showing the first ' +
                    str(result_list.count - len(s.lf.selectedLogs)) +
                    ' Results \nConsider applying filters to narrow down ' +
                    'results ',
                    'center'),
                w.div
            ])
            s.rl.resultOverflow = False
        else:
            if query:
                summaryRows = [w.getText('header', ' for ' + query, 'center')]
            else:
                summaryRows = []
            summaryRows = [
                w.div,
                w.getText(
                    'bold',
                    'There are ' + str(result_list.count) + ' results',
                    'center'),
                w.div
            ]
        activeFilters = state.get_active_filters()
        if activeFilters:
            summaryRows.append(w.getText(
                'bold',
                'Currently Active Filters:',
                'center'))
            for activeFilter in activeFilters:
                summaryRows.append(w.getText(
                    'body',
                    activeFilter.filter_field + ' : ' +
                    activeFilter.filter_criteria, 'center'))
        summaryRows.append(BoxButton(
            'Show Results',
            on_press=views.activate,
            user_data='results_list'))
        summary = urwid.SimpleFocusListWalker(summaryRows)
        summaryList = urwid.ListBox(summary)
        return w.centeredListLineBox(
            summaryList,
            'Search Results',
            len(summaryRows) + 6)

    def get_results_list(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        list_of_result_entries = entries.get_list_from_active_result_list()
        x = 1
        self.listDisplayCols = []
        for entry in list_of_result_entries:
            numColWidth = len(str(x)) + 6
            debug('entry.msgType:%s', entry.msgType)
            if len(entry.msgType) > 2:
                msgType = entry.msgType[2]
            else:
                msgType = '-'
            debug('msgType = %s', msgType)
            self.listDisplayCols.append(w.getColRow(
                [
                    (numColWidth, BoxButton(
                        str(x),
                        on_press=views.activate,
                        user_data=['single_entry', entry])),
                    (13, w.getText('body', '\n' + msgType + '\n', 'center')),
                    urwid.Text((
                        'body',
                        '\n' + entry.fullEntryText[2] + '\n'),
                        align='left'),
                    (3, w.getText('body', '', 'left'))
                ]
            ))
            x += 1
        resultListWalker = urwid.SimpleFocusListWalker(self.listDisplayCols)
        resultListBox = urwid.ListBox(resultListWalker)
        return resultListBox

    def get_add_remove_filters(self, **kwargs):
        if not state.get_result_list(ACTIVE):
            footer = w.getFooterWidget(
                state.get_view(ACTIVE),
                Menus().add_remove_filters_no_apply)
            frame.contents.__setitem__('footer', [footer, None])
        else:
            footer = w.getFooterWidget(
                state.get_view(ACTIVE),
                Menus().add_remove_filters)
            frame.contents.__setitem__('footer', [footer, None])
        debug(' kwargs : %s', kwargs)
        list_of_active_filters = []
        current_filters = state.get_active_filters()
        list_of_active_filters.append(w.getColRow([
            (12, urwid.AttrMap(
                w.getText('header', 'Remove', 'center'),
                'header')),
            urwid.AttrMap(
                w.getText('header', 'Filter Type', 'center'),
                'header'),
            urwid.AttrMap(
                w.getText('header', 'Filter Criteria', 'center'),
                'header')
        ]))
        # Populate Columns with filter info
        i = 1
        if len(current_filters) > 0:
            for filter in current_filters:
                list_of_active_filters.append(
                    w.getColRow([
                        BoxButton(
                            'Remove',
                            on_press=state.remove_active_filter,
                            user_data=[filter, i, self]),
                        w.getText(
                            'body',
                            "\n"+filter.filter_field+"\n",
                            'center'),
                        w.getText(
                            'body',
                            "\n"+filter.filter_criteria+"\n",
                            'left')]))
            i + 1
        # Create Filter Walker and listbox / linebox
        self.filter_walker = \
            urwid.SimpleFocusListWalker(list_of_active_filters)
        self.filter_list_box = urwid.BoxAdapter(
            urwid.ListBox(self.filter_walker),
            loop.screen.get_cols_rows()[1]-20)
        filter_line_box = w.getLineBox(self.filter_list_box, 'Active Filters')

        # Get filter Input
        self.filter_input = urwid.Edit(align='center')
        input_box = (w.getLineBox(self.filter_input, 'Add New Filter'))
        debug('edit_text: %s', self.filter_input.get_edit_text())
        # Get Filter Type
        bgroup = []
        filter_type_select = w.getColRow([
            w.getLineBox(
                urwid.RadioButton(
                    bgroup,
                    'Sender',
                    on_state_change=state.filter_input_radio,
                    state=False),
                '',
                brcorner='┴',
                trcorner='┬'),
            w.getLineBox(
                urwid.RadioButton(
                    bgroup,
                    'Recipient',
                    on_state_change=state.filter_input_radio, state=False),
                '',
                blcorner='─',
                brcorner='┴',
                tlcorner='─',
                trcorner='┬',
                lline=''),
            w.getLineBox(
                urwid.RadioButton(
                    bgroup,
                    'Message Type',
                    on_state_change=state.filter_input_radio, state=False),
                '',
                blcorner='─',
                brcorner='┴',
                tlcorner='─',
                trcorner='┬',
                lline=''),
            w.getLineBox(
                urwid.RadioButton(
                    bgroup,
                    'Date',
                    on_state_change=state.filter_input_radio, state=False),
                '',
                blcorner='─',
                tlcorner='─',
                lline='')
        ], dividechars=0)
        # Submit Filter
        filter_type_submit = BoxButton(
            'Add Filter',
            on_press=state.add_active_filter,
            user_data=self)
        # reate Pile

        pile_contents = [
            input_box,
            filter_type_select,
            filter_type_submit,
            ('weight', 4, filter_line_box)]
        filter_input_pile = urwid.Pile(pile_contents)
        # return filter_input_pile
        return urwid.Filler(filter_input_pile, valign='top')

    def get_clear_applied_filters(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        if state.active_filters:
            state.active_filters = []
            original_results_list = \
                state.get_result_list(ACTIVE).get_original_results()
            state.set_result_list(original_results_list)
        return self.get_results_summary()

    def get_quit_loop(self, **kwargs):
        """Page opened upon a request to quit, and
            asks for confirmation of quiting
        """
        debug(' kwargs : %s', kwargs)
        s.menuEnabled = True
        quitList = [
            w.div,
            w.getColRow([
                BoxButton('Yes', on_press=views.exit),
                BoxButton('No', on_press=state.go_back)
                ]),
            w.div]
        quitBox = w.getListBox(quitList)[0]
        return w.centeredListLineBox(
            quitBox,
            'Are You Sure You Want to Quit?',
            len(quitList) + 4)

    def get_coming_soon(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        comingSoonStatus = urwid.Pile([
            w.getText(
                'body',
                'Feature Still under Development. Coming Soon....',
                'center')
            ])
        comingSoonFiller = urwid.Filler(comingSoonStatus, 'middle')
        return w.centeredListLineBox(comingSoonFiller, '', 10)

    def get_stat_summary(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        select_user = urwid.Edit(
            'Enter the cPanel UserName Below\n', align='center')
        select_filler = UserBox(select_user, 'middle')
        return w.centeredListLineBox(
            select_filler,
            'cPanel User Stat Summary',
            5)

    def get_single_entry(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        single_entry = kwargs['user_args'][0]
        state.set_entry_on_screen(single_entry)
        entry_fields = single_entry.get_entry_fields()
        entry_fields.sort()
        entry_field_col_rows = [w.div]
        for field in entry_fields:
            if field[3] == '/':
                entry_field_col_rows.append(
                    w.getColRow([
                        (30, w.getText('body', field[2], 'center')),
                        ('weight', 4, w.getText('body', field[3], 'left'))
                    ]))
            else:
                entry_field_col_rows.append
                (w.getColRow([
                    (30, w.getButton(
                        field[2],
                        search,
                        'related',
                        user_data=field[3],
                        buttonMap='body')),
                    ('weight', 4, w.getText('body', field[3], 'left'))
                ]))
        entry_walker = urwid.SimpleFocusListWalker(entry_field_col_rows)
        return urwid.ListBox(entry_walker)

    def add_filter_to_walker(self, filter_instance):
        debug('filter to add to walker: %s', filter_instance)
        i = len(self.filter_walker)
        self.filter_walker.append(
            w.getColRow([
                (12, BoxButton(
                    'Remove',
                    on_press=state.remove_active_filter,
                    user_data=[filter_instance, i, self])),
                w.getText(
                    'body',
                    "\n"+filter_instance.filter_field+"\n",
                    'center'),
                w.getText(
                    'body',
                    "\n"+filter_instance.filter_criteria+"\n",
                    'left')
            ]))
        self.filter_input.set_edit_text('')

    def get_apply_filters(self, **kwargs):
        debug('kwargs: %s', kwargs)
        if state.get_result_list(ACTIVE):
            if state.active_filters:
                search.filter_active_results()
            return self.get_results_summary()

    def get_stat_summary_results(self, **kwargs):
        debug(' kwargs : %s', kwargs)
        debug('cPanel User: %s', state.cpanel_user)
        user_stats = cpanel_stats.get(state.cpanel_user)
        debug('user_stats instance: %s', user_stats)
        stat_columns = [w.div]
        stat_columns.append(
            w.getColRow([
                w.blankFlow,
                w.getText(
                    'body',
                    'Email Addresses Owned by ' + state.cpanel_user,
                    'center'),
                w.blankFlow
            ])
        )
        stat_columns.append(
            w.getColRow([
                urwid.AttrMap(w.blankFlow, 'header'),
                ('weight', 4, urwid.AttrMap(
                    w.getText('header', 'Email Address', 'center'),
                    'header')),
                (10, urwid.AttrMap(
                    w.getText('header', 'Recipient', 'center'),
                    'header')),
                (10, urwid.AttrMap(
                    w.getText('header', 'Sender', 'center'),
                    'header')),
                (10, urwid.AttrMap(
                    w.getText('header', 'Bounced', 'center'),
                    'header')),
                urwid.AttrMap(w.blankFlow, 'header')
            ], dividechars=0)
        )
        stat_columns.append(w.div)
        for email in user_stats.list_of_email_accounts:
            stat_columns.append(
                w.getColRow([
                    w.blankFlow,
                    ('weight', 4, w.getText('body', email, 'center')),
                    (10, w.getText(
                        'body',
                        str(user_stats.stats[email]['was_recipient']),
                        'center')),
                    (10, w.getText(
                        'body',
                        str(user_stats.stats[email]['was_sender']),
                        'center')),
                    (10, w.getText(
                        'body',
                        str(user_stats.stats[email]['was_bounced']),
                        'center')),
                    w.blankFlow
                ])
            )
        stat_columns.append(w.getDiv(divider='─'))
        stat_columns.append(w.div)
        stat_columns.append(
            w.getColRow([
                w.blankFlow,
                w.getText(
                    'body',
                    'Top Sending Scripts for ' + state.cpanel_user,
                    'center'),
                w.blankFlow
            ])
        )
        stat_columns.append(
            w.getColRow([
                urwid.AttrMap(w.blankFlow, 'header'),
                ('weight', 2, urwid.AttrMap(
                    w.getText(
                        'header',
                        'Script Source Directory',
                        'left'),
                    'header')),
                urwid.AttrMap(
                    w.getText(
                        'header',
                        'Sending Script',
                        'left'),
                    'header'),
                ('weight', 2, urwid.AttrMap(
                    w.getText('header', 'Sending Address', 'center'),
                    'header')),
                (12, urwid.AttrMap(
                    w.getText('header', 'Times Sent', 'center'),
                    'header')),
                urwid.AttrMap(w.blankFlow, 'header')
            ], dividechars=0)
        )
        stat_columns.append(w.div)
        for script in user_stats.sending_scripts.most_common(10):
            debug('Script Count: %s', script[1])
            script_dir = script[0].split()
            debug('Script Dir: %s', script_dir)
            if len(script_dir) > 2:
                script_sender = script_dir[2]
                if script_sender not in state.list_of_users_emails:
                    debug('Address not owned by cPanel User!')
                    script_sender_format = 'alert'
                else:
                    script_sender_format = 'body'
            else:
                script_sender = ''
                script_sender_format = 'body'
            stat_columns.append(
                w.getColRow([
                    w.blankFlow,
                    ('weight', 2, w.getText('body', script_dir[0], 'left')),
                    w.getText('body', script_dir[1], 'left'),
                    ('weight', 2, w.getText(
                        script_sender_format,
                        script_sender,
                        'right')),
                    (12, w.getText(
                        'body',
                        str(script[1]),
                        'center')),
                    w.blankFlow
                ], dividechars=3)
            )
        debug(
            'user_stats["sending_scripts"]: %s',
            user_stats.sending_scripts.most_common(5))
        stat_columns_pile = urwid.Pile(stat_columns)
        stat_line_box = w.getLineBox(
            stat_columns_pile,
            'cPanel User ' + state.cpanel_user + ' Stat Summary')
        stat_filler = urwid.Filler(stat_line_box, valign='middle')
        return stat_filler
"""
STATE MANAGEMENT / TRACKING
"""


class State():
    def __init__(self):
        self.active_view = None
        self.previous_view = None
        self.cpanel_user = None
        self.active_result_list = None
        self.prev_result_list = None
        self.active_query = None
        self.previous_query = None
        self.active_filters = []
        self.active_entry_on_screen = None
        self.prev_entry_on_screen = None
        self.searchCounter = 1
        self.view_count = -1
        self.view_chain = []
        self.view_chain_pos = -1
        self.filterCounter = 1
        self.current_filter_type = None
        self.current_filter_type_state = None
        self.current_edit_text = None
        self.list_of_users_emails = []

    def increment_counter(self):
        self.searchCounter += 1

    def filter_input_radio(self, *args, **kwargs):
        debug('args: %s, kwargs: %sa', args[0].state, kwargs)
        if args[0].state is False:
            self.current_filter_type = args[0].label
            self.current_filter_type_state = args[1]
        widget_contents = \
            self.get_view(ACTIVE).body.original_widget.contents[0][0]
        original_widget = widget_contents.original_widget
        self.current_edit_text = original_widget.get_edit_text()
        debug('edit text: %s', self.current_edit_text)

    def get_new_search_number(self):
        self.searchCounterStr = 'search-' + str(self.searchCounter).zfill(3)
        self.searchCounter += 1
        return self.searchCounterStr

    def get_new_filter_number(self):
        self.filterCounter += 1
        return self.filterCounter

    def set_view(self, view):
        debug('State.set_view: %s', view.view_name)
        # assign current view to previous view and store view as active_view
        if self.active_view:
            self.prev_view = self.active_view
        else:
            self.prev_view = None
        self.active_view = view

        # store view names in easily accessible attributes

        self.active_view_name = self.active_view.view_name
        if self.prev_view:
            self.prev_view_name = self.prev_view.view_name
        else:
            self.prev_view_name = None
        if len(self.view_chain) > self.view_chain_pos:
            self.view_chain.insert(self.view_chain_pos, view)
        else:
            self.view_chain.append(view)
        self.view_chain = self.view_chain[0:self.view_chain_pos + 1]
        # store status of active and prev view
        self.is_active_view_result_list = self.active_view.is_view_result_list
        self.is_active_view_single_entry = \
            self.active_view.is_view_single_entry
        self.is_active_view_add_filters = self.active_view.is_view_add_filters
        if self.prev_view:
            self.is_prev_view_result_list = self.prev_view.is_view_result_list
            self.is_prev_view__single_entry = \
                self.prev_view.is_view_single_entry
            self.is_prev_view__add_filters = self.prev_view.is_view_add_filters

    def get_view(self, active_prev):
        if active_prev == 'active':
            return self.active_view
        if active_prev == 'prev':
            return self.prev_view
        else:
            warning('State.get_view() active_prev parameter is invalid.')
            sys.exit('State.get_view() active_prev parameter is invalid.')

    def get_view_name(self, active_prev):
        if active_prev == 'active':
            return self.active_view.view_name
        if active_prev == 'prev':
            return self.prev_view.view_name
        else:
            warning('State.get_view_name() active_prev parameter is invalid.')
            sys.exit('State.get_view_name() active_prev parameter is invalid.')

    def get_view_from_chain(self, chain_pos):
        return self.view_chain[chain_pos]

    def set_result_list(self, result_list):
        """ assign current result_list to previous result_list and store
        result_list as active_result_list """

        debug('State.set_result_list: %s', result_list)
        if self.active_result_list:
            self.prev_result_list = self.active_result_list
        else:
            self.prev_result_list = None
        self.active_result_list = result_list

        # store result_list names in easily accessible attributes
        self.active_result_list_name = self.active_result_list.list_name
        if self.prev_result_list:
            self.prev_result_list_name = self.prev_result_list.list_name
        else:
            self.prev_result_list_name = None
        # store status of result_list as filtered or not.
        self.is_active_result_list_filtered = \
            self.active_result_list.is_filtered
        if self.prev_result_list:
            self.is_prev_result_list_filtered = \
                self.prev_result_list.is_filtered
        else:
            self.prev_result_list_filtered = None

    def get_result_list(self, active_prev):
        if active_prev == 'active':
            return self.active_result_list
        if active_prev == 'prev':
            return self.prev_result_list
        else:
            warning(
                'State.get_result_list() active_prev parameter is invalid.')
            sys.exit(
                'State.get_result_list() active_prev parameter is invalid.')

    def get_result_list_name(self, active_prev):
        if active_prev == 'active':
            if self.active_result_list:
                return self.active_result_list.list_name
            return False
        if active_prev == 'prev':
            if self.prev_result_list:
                return self.prev_result_list.list_name
            return False
        else:
            warning(
                'State.get_result_list_name() \
                    active_prev parameter is invalid.')
            sys.exit(
                'State.get_result_list_name() \
                    active_prev parameter is invalid.')

    def set_query(self, query):
        debug('State.set_query: %s', query)
        self.prev_query = self.active_query
        self.active_query = query

    def get_query(self, active_prev):
        if active_prev == 'active':
            return self.active_query
        if active_prev == 'prev':
            return self.prev_query
        else:
            warning('State.get_query() active_prev parameter is invalid.')
            sys.exit('State.get_query() active_prev parameter is invalid.')

    def add_active_filter(self, *args):
        debug('args: %s', args)
        body_instance = args[1]
        filter_number = self.get_new_filter_number()
        name = 'filter-' + str(filter_number)
        setattr(self, name, Filter(
            name,
            filter_number,
            self.current_filter_type,
            self.current_edit_text.lower()))
        self.active_filters.append(
            getattr(self, name))
        debug('active_filters: %s', self.active_filters)
        debug('Added New Filter: %s', name)
        body_instance.add_filter_to_walker(self.active_filters[-1])

    def remove_active_filter(self, *args):
        debug(
            'filter: %s, walker_index: %s, body_instance: %s',
            args[1][0], args[1][1], args[1][2])
        filter_instance = args[1][0]
        body_instance = args[1][2]
        self.active_filters.remove(filter_instance)
        debug('filter_walker length: %s', len(body_instance.filter_walker))
        for row in body_instance.filter_walker:
            if body_instance.filter_walker.index(row) > 0:
                filter_criteria = filter_instance.filter_criteria
                row_text = row.contents[2][0].get_text()[0]
                if filter_criteria in row_text:
                    if filter_instance.filter_type in row.contents[1][0]:
                        debug(
                            'Current filter walker rows: %s, \
                                current filter criteria: %s',
                            row.contents[2][0].get_text()[0],
                            filter_instance.filter_criteria)
                        body_instance.filter_walker.remove(row)

    def get_active_filters(self):
        debug('State.get_active_filters: %s', self.active_filters)
        return self.active_filters

    def set_entry_on_screen(self, entry_on_screen):
        debug('State.set_entry_on_screen: %s', entry_on_screen)
        self.prev_entry_on_screen = self.active_entry_on_screen
        self.active_entry_on_screen = entry_on_screen

    def get_entry_on_screen(self, active_prev):
        debug('State.get_entry_on_scareen: %s', active_prev)
        if active_prev == 'active':
            return self.active_entry_on_screen
        if active_prev == 'prev':
            return self.prev_entry_on_screen
        else:
            warning(
                'State.entry_on_screen() active_prev parameter is invalid.')
            sys.exit(
                'State.entry_on_screen() active_prev parameter is invalid.')

    def get_entry_on_screen_name(self, active_prev):
        debug('State.get_entry_on_screen_name: %s', active_prev)
        if active_prev == 'active':
            return self.active_entry_on_screen.entry_name
        if active_prev == 'prev':
            return self.prev_entry_on_screen.entry_name
        else:
            warning(
                'State.entry_on_screen() active_prev parameter is invalid.')
            sys.exit(
                'State.entry_on_screen() active_prev parameter is invalid.')

    def set_view_chain_pos(self, adjustment):
        viewchain_pos = self.get_view_chain_pos()
        chain_length = len(self.view_chain)
        if adjustment > 0 and viewchain_pos < chain_length - 1:
            self.view_chain_pos += adjustment
            return True
        if adjustment < 0 and self.get_view_chain_pos() > 0:
            self.view_chain_pos += adjustment
            return True
        else:
            return False

    def get_view_chain_pos(self):
        return self.view_chain_pos

    def go_back(self, *args):
        if self.set_view_chain_pos(-1):
            x = state.get_view_from_chain(state.get_view_chain_pos())
            x.reload()

    def go_forward(self):
        if self.set_view_chain_pos(1):
            x = state.get_view_from_chain(state.get_view_chain_pos())
            x.reload()


class View():
    def __init__(self, view_name, default_view_focus,
                 header_title=s.df.mainTitle,
                 header_subtitle='',
                 is_view_result_list=False,
                 is_view_single_entry=False,
                 is_view_add_filters=False):
        self.default_view_focus = default_view_focus
        self.view_name = view_name
        self.header_title = header_title
        self.header_subtitle = header_subtitle
        self.is_view_result_list = is_view_result_list
        self.is_view_single_entry = is_view_single_entry
        self.is_view_add_filters = is_view_add_filters

    def start(self, previous_view,
              focus_position=None,
              user_args=None, add_to_view_chain=True):
        debug(
            'view_name: %s, focus_position: %s',
            self.view_name, focus_position)
        self.previous_view = previous_view
        self.menuItems = getattr(s.menus, self.view_name)
        loop.screen.clear()
        self.footer = w.getFooterWidget(self, self.menuItems)
        self.show_footer()
        self.body = body.get_body_widget(
            self.view_name,
            user_args=user_args,
            calling_view=self)
        self.show_body()
        self.show_header()
        if focus_position:
            frame.set_focus(focus_position)
        else:
            frame.set_focus(self.default_view_focus)
        if add_to_view_chain:
            state.view_chain_pos += 1
            state.set_view(self)

    def reload(self):
        self.show_header()
        self.show_body()
        self.show_footer()

    def show_header(self):
        if not self.header_subtitle:
            self.header = w.getHeaderWidget(
                self.header_title,
                subtitle=s.df.mainSubTitle)
        else:
            self.header = w.getHeaderWidget(
                self.header_title,
                subtitle=self.header_subtitle)
        frame.contents.__setitem__('header', [self.header, None])

    def show_body(self):
        frame.contents.__setitem__('body', [self.body, None])

    def show_footer(self):
        frame.contents.__setitem__('footer', [self.footer, None])

    def draw_screen(self, loop):
        loop.draw_screen()

    def set_focus(self, target_frame, focus_position):
        target_frame.set_focus(focus_position)


class ViewSets():
    def __init__(self):
        self.quit_loop = View('quit_loop', BODY)
        self.choose_logs = View('choose_logs', BODY)
        self.home = View('home', FOOTER)
        self.new_search = View('new_search', BODY)
        self.search_progress = View('search_progress', BODY)
        self.results_summary = View('results_summary', BODY)
        self.results_list = View(
            'results_list', BODY, is_view_result_list=True)
        self.add_remove_filters = View(
            'add_remove_filters', BODY, is_view_add_filters=True)
        self.apply_filters = View('apply_filters', BODY)
        self.clear_applied_filters = View('clear_applied_filters', BODY)
        self.coming_soon = View('coming_soon', FOOTER)
        self.single_entry = View(
            'single_entry', BODY, is_view_single_entry=True)
        self.stat_summary = View('stat_summary', BODY)
        self.stat_summary_results = View('stat_summary_results', FOOTER)

    def get_view(self, view_name):
        return getattr(self, view_name)

    def activate(self, *args, **kwargs):
        debug('views.activate args: %s', args)
        focus_position = None
        is_threaded = False
        on_join = None
        user_data = None
        current_view = state.get_view(ACTIVE)
        if len(args) == 2:
            if type(args[0]) == urwid.Button:
                if type(args[1]) == list and len(args[1]) > 1:
                    view_name = args[1][0]
                    user_data = args[1][1:]
                else:
                    view_name = args[1]
            else:
                debug('type of urwid.Button: %s', type(args[0]))
                view_name = args[0]
                focus_position = args[1]
        elif len(args) == 1:
            view_name = args[0]
        else:
            view_name = args[1][0]
        activating_view = getattr(views, view_name)
        if 'is_threaded' in kwargs.keys():
            is_threaded = True
            passed_args = args
            on_join = kwargs['on_join']
        if is_threaded:
            debug('view.activate is threaded')
            updateThread = threading.Thread(
                target=activating_view.start(
                    current_view,
                    focus_position=focus_position,
                    user_args=passed_args,
                    add_to_view_chain=False))
            updateThread.start()
            updateThread.join()
            on_join()
        else:
            activating_view.start(
                current_view,
                focus_position=focus_position,
                user_args=user_data)

    def exit(self, *args):
        debug
        raise urwid.ExitMainLoop()
state = State()

"""
LOG CLASSES
"""


class LogFiles():
    def __init__(self):
        """This Class handles obtaining and updating
           lists of log files including the currently
           available logs, and the logss currently
           selected for searching.
        """
        self.availableLogs = self.getListofAvailableLogs()
        self.sortLogs(self.availableLogs)

    def sortLogs(self, logsToBeSorted):
        """Sorts list of log files with the primary file first
            and the gzipped files next, sorted by date in decending
            order.

        Arguments:
            logsToBeSorted {list} -- the logfile list that will be sorted
        """
        logsToBeSorted.sort(reverse=True)
        for log in logsToBeSorted:
            if log == s.lf.mainLogPath:
                logsToBeSorted.insert(
                    0,
                    logsToBeSorted.pop(logsToBeSorted.index(s.lf.mainLogPath)))

    def update(self, *args):
        """updates an item in the selectedLogs list by either adding
             or removing the item

        Arguments:
            itemsToUpdate {list} -- list of items to update. if only updating
            one item, it must still be contained in a list

        Raises:
            TypeError: raised if itemsToUpdate is not a list

        Returns:
            dict -- Dictionary of items added or removed in this call.
        """

        updates = {'removed': [], 'added': []}
        if type(args[0]) == urwid.wimp.CheckBox:
            itemsToUpdate = args[2]
            isLogSelected = args[1]

            if type(itemsToUpdate) != list:
                raise TypeError(
                    '{} provided where list is required',
                    type(itemsToUpdate))
            if isLogSelected:
                for x in itemsToUpdate:
                    if x not in s.lf.selectedLogs:
                        updates['added'].append(x)
                        s.lf.selectedLogs.append(x)
                        self.sortLogs(s.lf.selectedLogs)
                    else:
                        warning(
                            'LogFiles.update: Failed to add log %s : \
                                Log already on selectedLogs list',
                            x)
            else:
                for x in itemsToUpdate:
                    if x in s.lf.selectedLogs:
                        updates['removed'].append(x)
                        s.lf.selectedLogs.remove(x)
                        self.sortLogs(s.lf.selectedLogs)
                    else:
                        warning(
                            'LogFiles.update: Failed to remove log %s : \
                                Log is not on selectedLogs list',
                            x)
        return updates

    def getListofAvailableLogs(self):
        """Obtains list of log files available in the
           directory set in LogFileSettings.dir
        Returns:
            list -- list of available log files
        """
        logdir = s.lf.dir
        loglist = []
        for file in os.listdir(logdir):
            if file.startswith("exim_mainlog"):
                loglist.append(os.path.join(logdir, file))
        return loglist

    def get_date_range(self):
        date_ranges = []
        for log in s.lf.selectedLogs:
            if log[-2:] != 'gz':
                popen_args = "sudo head -1 " + log
                reqOutput, _ = subprocess.Popen(
                    popen_args,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True
                    ).communicate()
                first_line = reqOutput[:10]
                debug('first_line: %s', first_line)
                popen_args = "sudo tail -1 " + log
                reqOutput, _ = subprocess.Popen(
                    popen_args,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True
                    ).communicate()
                last_line = reqOutput[:10]
                debug('last_line: %s', last_line)
                date_ranges.append(
                    [first_line, last_line])
            else:
                popen_args = "sudo zcat " + log + " | head -1"
                reqOutput, _ = subprocess.Popen(
                    popen_args,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True
                    ).communicate()
                first_line = reqOutput[:10]
                debug('first_line: %s', first_line)
                popen_args = "sudo zcat " + log + " | tail -1"
                reqOutput, _ = subprocess.Popen(
                    popen_args,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True
                    ).communicate()
                last_line = reqOutput[:10]
                date_ranges.append([first_line, last_line])
                debug('last_line: %s', last_line)
        date_ranges.sort()
        debug('Date Ranges: %s', date_ranges)
        fixed_dates = []
        i = 0
        while i < len(date_ranges):
            if len(date_ranges) > i + 1:
                if date_ranges[i][1] != date_ranges[i+1][0]:
                    fixed_dates.append(date_ranges[i])
                if date_ranges[i][1] == date_ranges[i+1][0]:
                    date_ranges[i+1][0] = date_ranges[i][0]
            else:
                fixed_dates.append(date_ranges[i])
            i += 1
        formatted_date_ranges = []
        for date_range in fixed_dates:
            x = datetime.strptime(date_range[0], s.dt.logDateFormat)
            y = datetime.strptime(date_range[1], s.dt.logDateFormat)
            formatted_date_ranges.append(
                '( ' + x.strftime('%b %d %Y') +
                ' - ' + y.strftime('%b %d %Y') + ' )'
            )
        debug('Formatted Date Ranges: %s', formatted_date_ranges)
        s.df.mainSubTitle = 'Log Dates: ' + ' '.join(formatted_date_ranges)
"""
DATA PROCESSING / SEARCH AND RESULT CLASSES
"""


class Search():
    def new(self, *args):
        # if hasattr(self, 'process_pool'):
        #    self.process_pool.terminate()
        #    del(self.process_pool)
        #    self.process_pool = Pool(maxtasksperchild=1)
        # else:
        #    self.process_pool = Pool(maxtasksperchild=1)
        if args:
            debug('Search Has special arguments: %s', args)
            state.set_query(args[0])
        debug('New Search Object with query: %s', state.get_query(ACTIVE))
        debug("CPU Count = %s", cpu_count())
        searchNumber = state.get_new_search_number()
        results.new(
            searchNumber,
            'logResults',
            self.filter_logs_2(),
            original_results=searchNumber
            )
        result_list = results.get_result_list(searchNumber)
        if hasattr(self, 'process_pool'):
            self.process_pool.terminate()
            del(self.process_pool)
        if state.active_filters:
            views.activate('apply_filters')
        views.activate(
            'results_summary',
            focus_position=BODY,
            user_args=result_list)

    def related(self, related_query):
        debug('related_query: %s', related_query)
        state.set_query(related_query)
        views.activate(
            'search_progress',
            is_threaded=True,
            on_join=search.new)

    def cpanel_summary(self):
        logPoolArgs = []
        for log in s.lf.selectedLogs:
            logPoolArgs.append(log)

        if hasattr(self, 'process_pool'):
            self.process_pool.terminate()
            del(self.process_pool)
            self.process_pool = Pool(maxtasksperchild=1)
        else:
            self.process_pool = Pool(maxtasksperchild=1)
        debug('logPoolArgs: %s', logPoolArgs)
        results = self.process_pool.map(query_for_cpanel_summary, logPoolArgs)
        if state.cpanel_user:
            cpanel_stats.new(
                state.cpanel_user,
                state.list_of_users_emails,
                results)

        if hasattr(self, 'process_pool'):
            self.process_pool.terminate()
            del(self.process_pool)
        views.activate('stat_summary_results')

    def filter_active_results(self):
        results_to_filter = state.get_result_list(ACTIVE)
        list_of_active_filters = state.get_active_filters()
        list_of_entries_to_filter = entries.get_list_from_active_result_list()
        debug(
            'results_to_filter: %s, list_of_active_filters: %s',
            results_to_filter,
            list_of_active_filters)
        search_no = state.get_new_search_number()
        i = 0
        list_of_matching_entries = []
        for entry in list_of_entries_to_filter:
            for result_filter in list_of_active_filters:
                if hasattr(entry, result_filter.filter_field):
                    filter_criteria = result_filter.filter_criteria
                    filter_field = getattr(entry, result_filter.filter_field)
                    if filter_criteria in filter_field:
                        debug('Matching Entry: %s', entry.msgType)
                        name = 'entry-' + search_no + '-' + str(i)
                        i += 1
                        setattr(entries, name, entry)
                        debug(
                            'New entries.%s, %s',
                            name,
                            getattr(entries, name))
                        list_of_matching_entries.append(getattr(entries, name))
        debug('list_of_matching_entries: %s', list_of_matching_entries)
        results.new(
            search_no,
            'filtered_results',
            list_of_matching_entries,
            original_results=results_to_filter,
            isFiltered=True,
            filters_applied=list_of_active_filters)

    def filter_logs(self, related_query=False):
        starttime = datetime.now()
        logPoolArgs = []
        query = state.get_query(ACTIVE)
        for log in s.lf.selectedLogs:
            logPoolArgs.append([query, log])
        if hasattr(self, 'process_pool'):
            self.process_pool.terminate()
            del(self.process_pool)
            self.process_pool = Pool(maxtasksperchild=1)
        else:
            self.process_pool = Pool(maxtasksperchild=1)
        searchedLogs = self.process_pool.map(queryLogProcess, logPoolArgs)
        results = []
        for resultList in searchedLogs:
            results.extend(resultList)
        logging.info(
            'QT = %s : filteredLog Pool Result Count: %s',
            datetime.now() - starttime,
            len(results))
        if hasattr(self, 'process_pool'):
            self.process_pool.terminate()
            del(self.process_pool)
        return results

    def filter_logs_2(self, related_query=False):
        starttime = datetime.now()
        logPoolArgs = []
        query = state.get_query(ACTIVE)
        results = []
        for log in s.lf.selectedLogs:
            x = True
            total_lines, error = subprocess.Popen(
                ["wc -l " + log + " | awk '{print $1}'"],
                stderr=subprocess.PIPE,
                stdout=subprocess.PIPE,
                shell=True).communicate()
            debug("Total Lines: %s", total_lines)
            debug("Error: %s", error)
            segment_line_size = int(total_lines) / (cpu_count() - 1)
            # segment_line_size = 100000
            start_line = 1
            end_line = segment_line_size
            # with open(log) as f:
            #    while x:
            #        try:
            #            line = next(islice(f, start_line - 1, start_line))
            #        except StopIteration:
            #            debug('Last Start Line: %s', start_line)
            #            x = False
            #        else:
            #            logPoolArgs.append(
            #                [query, log, start_line, end_line])
            #        start_line = start_line + segment_line_size
            #        end_line = end_line + segment_line_size
            while start_line < int(total_lines):
                logPoolArgs.append(
                    [query, log, start_line, end_line])
                start_line = start_line + segment_line_size
                end_line = end_line + segment_line_size
            pool_size_time = datetime.now() - starttime
            debug("Time to Determine Pool Sizes: %s", pool_size_time)
            debug("ProcessPool Args: %s", len(logPoolArgs))
            if hasattr(self, 'process_pool'):
                self.process_pool.terminate()
                del(self.process_pool)
                self.process_pool = Pool()
            else:
                self.process_pool = Pool(cpu_count())
            searchedLogs = self.process_pool.map(
                queryLogProcess_2, logPoolArgs)
            for resultList in searchedLogs:
                results.extend(resultList)
            logging.info(
                'QT = %s : filteredLog Pool Result Count: %s',
                datetime.now() - starttime, len(results))
            if hasattr(self, 'process_pool'):
                self.process_pool.terminate()
                del(self.process_pool)
        return results

    def filter_logs_3(self, related_query=False):
        starttime = datetime.now()
        jobs = []
        pool = Pool()
        results = []
        query = state.get_query(ACTIVE)
        for log in s.lf.selectedLogs:
            if log[-2:] != 'gz':
                with open(log, mode='r') as f:
                    nextLineByte = f.tell()
                    for line in f:
                        jobs.append(
                            pool.apply_async(
                                process_wrapper,
                                (nextLineByte, query, log)))
                        nextLineByte = f.tell()
            else:
                with gzip.open(log, mode='r') as gz:
                    f = io.BufferedReader(gz)
                    # nextLineByte = f.tell()
                    for line in f:
                        if len(jobs) < 10000:
                            jobs.append(
                                pool.apply_async(
                                    process_wrapper,
                                    (None, query, log)))
                        # nextLineByte = f.tell()
        for job in jobs:
            if job:
                results.append(job.get())
        logging.info(
            'QT = %s : ',
            datetime.now() - starttime)
        debug('Results Count: %s', len(results))
        pool.close()


class CpanelStats():
    def new(self, cpanel_user_name, list_of_email_accounts, stat_results):
        setattr(self, cpanel_user_name, CpanelUserStats(
            cpanel_user_name,
            list_of_email_accounts,
            stat_results))

    def get(self, cpanel_user_name):
        debug('cpanel_user_name: %s', cpanel_user_name)
        return getattr(self, cpanel_user_name)


class CpanelUserStats():
    def __init__(self, cpanel_user_name, list_of_email_accounts, stat_results):
        debug('Init cPanelUserStats: %s', cpanel_user_name)
        self.cpanel_user_name = cpanel_user_name
        self.list_of_email_accounts = list_of_email_accounts
        self.stats = {}
        self.sending_scripts = collections.Counter()
        for item in stat_results:
            for line in item['sending_scripts']:
                x = line.split()
                for y in x:
                    if 'cwd=' in y:
                        debug('sending_script directory: %s', y[4:])
                        debug('sending_script: %s', x[x.index(y) + 3])
                        debug('Sending_script PID: %s', x[x.index(y) - 1])
                        pid = x[x.index(y) - 1]
                        sending_email = ''
                        for line in item['script_related_lines']:
                            if pid in line:
                                i = line.split()
                                for j in i:
                                    if 'from' in j:
                                        debug(
                                            'script_sending_email: %s',
                                            i[i.index(j) + 1][1:-1])
                                        sending_email = i[i.index(j) + 1][1:-1]
                        z = y[4:] + ' ' + x[x.index(y) + 3]
                        z = z + ' ' + sending_email
                        self.sending_scripts[z] += 1
        for email in self.list_of_email_accounts:
            self.stats[email] = {
                'was_recipient': 0,
                'was_sender': 0,
                'was_bounced': 0
            }
            for item in stat_results:
                if email in item:
                    for key in self.stats[email].keys():
                        if key in item[email]:
                            self.stats[email][key] += item[email][key]
        debug('Stat Results: %s', stat_results)
        debug(
            'Most Common Sending Script Directories: %s',
            self.sending_scripts.most_common(5))
        debug('self.stats: %s', self.stats)


class Results():
    def __init__(self):
        self.currentFilters = {
            'Type': [],
            'Sender': [],
            'Recipient': [],
            'Date': [],
        }
        self.entries = {}
        self.filterEntryEditText = ''

    def new(self, name, resultType,
            resultContents, original_results='',
            isFiltered=False, filters_applied=[]):
        """Class of Result Lists

        Arguments:
            name {str} -- name of this result instance
            resultType {str} -- the origin source of this result list
            (logResults,FilteredResults, etc)
            resultContents {list} -- a list of results. each list item must be
            str

        Raises:
            TypeError: raised if resultContents is not a list
            TypeError: raised if the first item in resultContents is not a str
        """
        debug('New Result List created: %s', name)
        if resultType == 'logResults':
            if type(resultContents) != list:
                raise TypeError(
                    '{} provided :: resultContents must be a list of\
                         strings'.format(type(resultContents)))
            try:
                type(resultContents[0])
            except:
                pass
            else:
                if type(resultContents[0]) != str:
                    raise TypeError(
                        '{} provided :: resultContents must be a list of\
                             strings'.format(type(resultContents[0])))
            count = len(resultContents)
            if hasattr(self, name):
                raise Exception(
                    'A result list by the name of {} \
                        already exists'.format(name))
            else:
                setattr(self, name, ResultLists(
                    name, resultType,
                    resultContents, count,
                    original_results,
                    isFiltered, filters_applied))
                state.set_result_list(getattr(self, name))
                debug('Finished Creating New Result List')
        else:
            count = len(resultContents)
            setattr(self, name, ResultLists(
                name, resultType,
                resultContents, count,
                original_results,
                isFiltered, filters_applied))
            debug('Finished Creating New Result List')
            state.set_result_list(getattr(self, name))

    def get_single_entry(self, entryNo, searchNo):
        debug('Started Results.getSingleEntry: %s, %s', entryNo, searchNo)
        entryId = 'entry-' + str(entryNo)
        resultList = getattr(self, searchNo)
        debug('resultList : %s', resultList)
        entry = getattr(resultList, entryId)
        return entry.getEntryFields()

    def get_active_filter_strings(self):
        activeFilterStringList = []
        for filterType, activeFilter in self.currentFilters.items():
            if activeFilter:
                activeFilterStringList.append(
                    'Message ' + filterType + ' = ' + ', '.join(activeFilter))
        return activeFilterStringList

    def get_count(self, searchNo):
        resultList = getattr(self, searchNo)
        return resultList.count

    def get_result_list(self, searchNo):
        return getattr(self, searchNo)

    def check_for_add_filter_entry(self, *args):
        debug('Results.checkForAddFilterEntry args: %s', args)

    def add_filters(self, *args):
        debug('Results.addFilters args: %s', args)


class ResultLists():
    def __init__(self, list_name, result_type, result_contents,
                 count, original_results='', is_filtered=False,
                 filters_applied=[]):
        debug('Init ResultLists')
        self.list_name = list_name
        self.is_filtered = is_filtered
        self.query = state.get_query(ACTIVE)
        debug('query: %s', self.query)
        self.original_results = original_results
        self.filteredApplied = filters_applied
        self.count = count
        self.previous_results = None
        self.contents = []
        if result_type == 'logResults':
            debug('result_type == logResults')
            i = 0
            search_number = state.searchCounterStr
            debug('search_number: %s', search_number)
            for result in result_contents:
                if result == 'RESULTSOVERFLOW':
                    s.rl.resultOverflow = True
                else:
                    name = 'entry-' + search_number + '-' + str(i)
                    entries.new(name, result)
                    i += 1
            self.contents = entries.get_list_from_search_no(search_number)
        else:
            self.contents = result_contents
        debug('Init ResultLists Complete')

    def parseEntries(self):
        i = 0
        searchNumber = state.searchCounterStr
        for result in self.contents:
            name = 'entry-' + searchNumber + '-' + str(i)
            entries.new(name, result)
            i += 1

    def getListOfEntries(self):
        listOfEntries = []
        for attr in dir(self):
            if 'entry-' in attr:
                listOfEntries.append(getattr(self, attr))
        return listOfEntries

    def get_original_results(self):
        return self.original_results

    def set_previous_list(self):
        if self.previous_results:
            state.prev_result_list = self.previous_results
        else:
            self.previous_results = state.prev_result_list


class Entries():
    def new(self, name, new_entry):
        if hasattr(self, name):
            raise Exception(
                'Entry number {} has already been parsed'.format(name))
        else:
            setattr(self, name, Entry(new_entry, name))

    def get_list_from_active_result_list(self):
        listOfEntries = []
        active_result_list_name = state.get_result_list_name(ACTIVE)
        for attr in dir(self):
            if active_result_list_name in attr:
                listOfEntries.append(getattr(self, attr))
        return listOfEntries

    def get_list_from_search_no(self, search_no):
        listOfEntries = []
        active_result_list_name = search_no
        for attr in dir(self):
            if active_result_list_name in attr:
                listOfEntries.append(getattr(self, attr))
        return listOfEntries

    def get_entry(self, entry_name):
        if hasattr(self, entry_name):
            return getattr(self, entry_name)
        else:
            return False


class Entry():
    def __init__(self, fullEntryText, name):
        """This class is used to create single-view Entry Objects
        """
        self.name = name
        self.id = []
        self.msgType = []
        self.date = []
        self.time = []
        self.sendAddr = []
        self.recipient = []
        self.fullEntryText = fullEntryText
        self.entryType = []
        if 'Warning: "SpamAssassin' in self.fullEntryText:
            self.msgType = [15, 'Message Type: ', 'Spam Status']
        try:
            shlex.split(self.fullEntryText)
        except:
            try:
                self.fullEntryText.split()
            except:
                warning('Could Not Parse line: %s', self.fullEntryText)
            else:
                m = self.fullEntryText.split()
                if self.msgType:

                    if self.msgType[2] == 'spam status':
                            if 'detected' in m:
                                self.spamStatus = [
                                    15,
                                    'spam status',
                                    ' '.join(m[m.index('detected') + 3: -2])]
                                self.spamScore = [15, 'spam score', m[-1]]
                x = 0
                while x <= len(m):
                    if x == 0:
                        self.date = [10, 'Date: ', m[x]]
                    if x == 1:
                        self.time = [11, 'Time: ', m[x]]
                    if x == 2:
                        self.pid = [12, 'Process ID: ', m[x]]
                    if x == 3:
                        self.id = [13, 'Message ID: ', m[x]]
                    x += 1
                self.fullEntryText = [14, 'Full Entry: ', self.fullEntryText]
        else:
            m = shlex.split(self.fullEntryText)
            x = 0
            while x < len(m):
                if x == 0:
                    self.date = [10, 'Date: ', m[x]]
                if x == 1:
                    self.time = [11, 'Time: ', m[x]]
                if x == 2:
                    self.pid = [12, 'Process ID: ', m[x]]
                if x == 3:
                    if m[x].startswith('cwd'):
                        self.cwd = [
                            13, 'Source Directory: ', m[x].split('=')[1]]
                        if self.cwd[2] == '/var/spool/exim':
                            self.msgType = [15, 'Message Type: ', 'exim queue']
                            self.id = [13, 'Message Id: ', m[-1]]
                        else:
                            self.msgType = [
                                15, 'Message Type: ', 'script mailer']
                        if 'args:' in m:
                            args = m.index('args:')
                            self.script = [13, 'Sending Script: ', m[args + 1]]
                            if len(m) > args + 2:
                                self.scriptArgs = [
                                    13,
                                    'Script Arguments: ',
                                    ' '.join(m[args + 2:])]
                    else:
                        self.id = [13, 'Message ID: ', m[x]]
                    if 'Sender' in m[x]:
                        self.msgType = [15, 'Message Type: ', 'sender id']
                if x == 4:
                    if len(m[x]) == 2:
                        self.entryType = [22, 'Entry Type Symbol: ', m[x]]
                    elif 'Completed' in m[x]:
                        self.msgType = [15, 'Message Type: ', 'queue status']
                if self.msgType:
                    if self.msgType[2] == 'Spam Status':
                        if 'detected' in m[x]:
                            n = m[x].split()
                            if 'detected' in n:
                                self.spamStatus = [
                                    15, 'Spam Status',
                                    ' '.join(n[n.index('detected')+3:-1])]
                                self.spamScore = [15, 'Spam Score', n[-1]]
                if 'H=' in m[x]:
                    if len(m) > x + 1:
                        if m[x+1][0] == '(':
                            self.host = [
                                16, 'Host: ', m[x][2:] + ' ' + m[x+1]]
                            self.hostIp = [
                                17, 'Host IP: ', m[x+2].split(':')[0]]
                            if s.hostname in self.host:
                                self.msgType = [15, 'Type: ', 'relay']
                        else:
                            self.host = [16, 'Host: ', m[x][2:]]
                            self.hostIp = [
                                17, 'Host IP: ', m[x+1].split(':')[0]]
                            if s.hostname in self.host:
                                self.msgType = [15, 'Type: ', 'relay']
                if m[x] == 'SMTP':
                    self.smtpError = [22, 'SMTP Message: ', " ".join(m[x:])]
                if 'S=' in m[x] and m[x][0] != 'M':
                    self.size = [22, 'Size: ', m[x][2:]]
                if 'I=' in m[x] and m[x][0] != 'S':
                    self.interface = [
                        22, 'Receiving Interface: ', m[x].split(':')[0][2:]]
                if 'R=' in m[x]:
                    self.bounceId = [22, 'Bounce ID: ', m[x][2:]]
                if 'U=' in m[x]:
                    self.mta = [22, 'MTA / User: ', m[x][2:]]
                if 'id=' in m[x]:
                    self.remoteId = [
                        22, 'Sending Server Message ID: ', m[x][3:]]
                if 'F=<' in m[x]:
                    self.sendAddr = [18, 'Sender: ', m[x][2:]]
                    if not self.sendAddr[1] == '<>':
                        self.sendAddr = [18, 'Sender: ', m[x][3:-1]]
                    self.fr = self.sendAddr
                if 'C=' in m[x]:
                    self.delStatus = [22, 'Delivery Status: ', m[x][2:]]
                if 'QT=' in m[x]:
                    self.timeInQueue = [22, 'Time Spent in Queue: ', m[x][3:]]
                if 'DT=' in m[x]:
                    self.deliveryTime = [
                        22, 'Time Spent being Delivered: ', m[x][3:]]
                if 'RT=' in m[x]:
                    self.deliveryTime = [
                        22, 'Time Spent being Delivered: ', m[x][3:]]
                if '<=' in fullEntryText:
                    if 'A=' in m[x]:
                        self.smtpAuth = [22, 'Auth. Method: ', m[x][2:]]
                    if x == 5:
                        self.sendAddr = [18, 'Sender: ', m[x]]
                    if 'P=' in m[x]:
                        self.protocol = [22, 'Protocol: ', m[x][2:]]
                    if 'T=' in m[x] and m[x][0] != 'R':
                        self.topic = [21, 'Subject: ', m[x][2:]]
                    if m[x] == 'from':
                        if len(m) > x + 1:
                            if m[x+1] == '<>':
                                self.fr = [18, 'Sender: ', m[x+1]]
                                self.msgType = [15, 'Type: ', 'bounce']
                            else:
                                self.fr = [18, 'Sender: ', m[x+1][1:-1]]
                    if m[x] == 'for':
                        if '(' in m[x+1] and ')' in m[x+1]:
                            y = m[x+1]
                            stripped_recip = y[y.index('(') + 1:y.index(')')]
                            self.recipient = [
                                19, 'Recipient: ', stripped_recip]
                        if '<' in m[x+1] and '>' in m[x+1]:
                            y = m[x+1]
                            stripped_recip = y[y.index('<') + 1:y.index('>')]
                            self.recipient = [
                                19, 'Recipient: ', stripped_recip]
                        else:
                            self.recipient = [19, 'Recipient: ', m[x+1]]
                    if 'P=local' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'local']
                    elif 'A=dovecot' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'relay']
                    else:
                        self.msgType = [15, 'Message Type: ', 'incoming']
                else:
                    if x == 5:
                        if '@' in m[x]:
                            if '(' in m[x] and ')' in m[x]:
                                y = m[x]
                                stripped_recip = y[
                                    y.index('(') + 1:y.index(')')]
                                self.recipient = [
                                    19, 'Recipient: ', stripped_recip]
                            if '<' in m[x] and '>' in m[x]:
                                y = m[x]
                                stripped_recip = y[
                                    y.index('<') + 1:y.index('>')]
                                self.recipient = [
                                    19, 'Recipient: ', stripped_recip]
                            self.recipient = [19, 'Recipient: ', m[x]]
                        else:
                            if len(m) > x + 1:
                                if '@' in m[x + 1]:
                                    if '(' in m[x+1] and ')' in m[x+1]:
                                        y = m[x+1]
                                        stripped_recip = y[
                                            y.index('(') + 1:y.index(')')]
                                        self.recipient = [
                                            19, 'Recipient: ', stripped_recip]
                                    if '<' in m[x+1] and '>' in m[x+1]:
                                        y = m[x+1]
                                        stripped_recip = y[
                                            y.index('<') + 1:y.index('>')]
                                        self.recipient = [
                                            19, 'Recipient: ', stripped_recip]
                                    else:
                                        self.recipient = [
                                            19, 'Recipient: ', m[x] + m[x+1]]
                    if 'P=' in m[x]:
                        self.returnPath = [20, 'Return Path: ', m[x][3:-1]]
                    if 'T=' in m[x] and m[x][0] != 'D' and m[x][0] != 'Q':
                        self.mta = [22, 'MTA: ', m[x][2:]]
                        if 'dovecot' in self.mta[1]:
                            self.msgType = [15, 'Message Type: ', 'local']
                    if ' => ' in fullEntryText:
                        if 'T=dovecot' in fullEntryText:
                            self.msgType = [15, 'Message Type: ', 'dovecot']
                        else:
                            self.msgType = [15, 'Message Type: ', 'outgoing']
                    if ' -> ' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'forwarder']
                    if not self.entryType and ' rejected ' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'bounce']
                        self.id = []
                    if 'SMTP connection' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'SMTP Connect']
                    if ' ** ' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'bounce']
                    if ' no host name found ' in fullEntryText or \
                            ' failed to find host name ' in fullEntryText:
                        self.msgType = [15, 'Message Type: ', 'Hostname Error']
                x += 1
            self.fullEntryText = [14, 'Full Entry: ', self.fullEntryText]

    def getTimeOrd(self):
        x = str(self.date[2]) + '_' + str(self.time[2])
        x = datetime.strptime(x, s.dt.logDateTimeFormat)
        return x.toordinal()

    def get_entry_fields(self):
        fieldList = [
            a for a in dir(self)
            if not a.startswith('__') and not callable(getattr(self, a))]
        fields = []
        for field in fieldList:
            x = getattr(self, field)
            if not x == self.name:
                if x:
                    fields.append([
                        x[0],
                        field,
                        x[1],
                        x[2]
                    ])
        return fields

    def get_field_text(self, field_name):
        field = getattr(self, field_name)
        if len(field) >= 3:
            return field[2]
        else:
            return field


class Filters():
    def add(self, filter_field, filter_criteria, body_instance):
        filter_number = state.get_new_filter_number()
        name = 'filter-' + str(filter_number)
        if hasattr(self, name):
            raise Exception(
                'Filter number {} has already been created'.format(name))
        else:
            setattr(self, name, Filter(
                name, filter_number, filter_field, filter_criteria))

    def rem(self, filter_to_remove):
        filter_name = filter_to_remove.name
        if hasattr(self, filter_name):
            delattr(self, filter_name)
        else:
            raise Exception(
                'Filter number {} does not exist'.format(filter_name))

    def get_filters_list(self, list_by, option):
        list_of_filters = []
        all_filters = [
            a for a in dir(self)
            if not a.startswith('__') and not callable(getattr(self, a))]
        if list_by == 'all':
            debug('all_filters: %s', all_filters)
            for filters in all_filters:
                x = getattr(self, filters)
                if not type(x) == int or not type(x) == str:
                    list_of_filters.append(x)
                    debug('type of attr: %s', type(x))
        else:
            for filters in all_filters:
                if hasattr(filters, list_by):
                    x = getattr(filters, list_by)
                    if option in x:
                        list_of_filters.append(filters)
        debug('list_of_filters: %s', list_of_filters)
        return list_of_filters


class Filter():
    def __init__(self, name, filter_number, filter_field, filter_criteria):
        debug('New Filter: %s', self)
        self.name = name
        self.filter_number = filter_number
        if filter_field == 'Message Type':
            self.filter_field = 'msgType'
        if filter_field == 'Sender':
            self.filter_field = 'sendAddr'
        if filter_field == 'Recipient':
            self.filter_field = 'recipient'
        if filter_field == 'Date':
            self.filter_field = 'date'
        self.filter_criteria = filter_criteria


class UAPI():
    """This is the class used to make calls to WHM

    Arguments:
        APIClient {Class} -- [https://github.com/0x6c/whmapi.git]

    Returns:
        [NONE] -- [NONE]
    """
    def call(self, user, module, function, req_params=None):
        """Method to run WHMAPI function via subprocess module

        Arguments:
            function {string} -- [provides the function to be called]
            req_params {dict} -- [dictionary of key + values]
        """

        debug(self)

        popen_args = [
            "/usr/local/cpanel/bin/uapi",
            '--user='+user,
            module,
            function,
            '--output=json']
        if req_params:
            for key, value in req_params.items():
                x = str(key) + "=" + str(value)
                popen_args.append(x)

        reqOutput, reqError = subprocess.Popen(
            popen_args,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
            ).communicate()
        debug('reqOutput: %s', reqOutput)
        if reqError:
            warning(
                'Error Returned when Attempting to call cpanel \
                    for email list: %s',
                reqError)
            if 'UserNotFound' in reqError:
                warning('Cpanel User Selected does not exist!')
                state.cpanel_user = None
            return False
        data = json.loads(reqOutput)
        debug("uapi call Results: %s", data)
        debug("uapi data metadata: %s", data['result']['data'])
        resultCode = data['result']['status']
        resultReason = data['result']['errors']
        if resultCode == 1:
            resultData = data['result']['data']
            return resultData
        else:
            debug("""The data you entered is invalid!\n
            Take a deep hard look at yourself\n
            And reconsider whether or not you
            know what you are doing...""")
            debug('Result Code: %s --- %s', resultCode, resultReason)
        return (resultCode, resultReason)

    def list_email_accounts(self, user):
        data = self.call(user, 'Email', 'list_pops')
        email_list = []
        if data:
            for item in data:
                email = item['email']
                if email == user:
                    email = email + '@' + s.hostname
                email_list.append(email)
        debug('Email_List: %s', email_list)
        return email_list


def process_wrapper(lineByte, query, log):
    if log[-2:] != 'gz':
        with open(log) as f:
            f.seek(lineByte)
            line = f.readline()
            if query in line:
                debug('match: %s', line)
                return line
            else:
                return False
    else:
        with gzip.open(log) as gz:
            f = io.BufferedReader(gz)
            line = f.readline()
            if query in line:
                return line
            else:
                return False


def queryLogProcess(poolArgs):
    process_start_time = datetime.now()
    os.nice(20)
    query, log = poolArgs
    rawEntries = []
    if log[-2:] != 'gz':
        with open(log, mode='r') as f:
            for i, line in enumerate(f):
                pass
            totalLines = i
        with open(log, mode='r') as p:
            for i, line in enumerate(p):
                if len(rawEntries) == 1000:
                    s.rl.resultOverflow = True
                    debug('ResultOverflow: %s', s.rl.resultOverflow)
                    rawEntries.append('RESULTSOVERFLOW')
                    break
                if query in line:
                    if line not in rawEntries:
                        rawEntries.append(line)
                if i % 1000000 == 0:
                    percent = float(i) / float(totalLines)
                    completion = int(percent * 100)
                    w.searchProgress.set_completion(completion)
                    loop.draw_screen()
    else:
        with gzip.open(log, mode='r') as f:
            for i, line in enumerate(f):
                pass
            totalLines = i
        with gzip.open(log, mode='r') as f:
            for i, line in enumerate(f):
                if len(rawEntries) == 1000:
                    s.rl.resultOverflow = True
                    debug('ResultOverflow: %s', s.rl.resultOverflow)
                    rawEntries.append('RESULTSOVERFLOW')
                    break
                if query in line:
                    if line not in rawEntries:
                        rawEntries.append(line)
                if i % 1000000 == 0:
                    percent = float(i) / float(totalLines)
                    completion = int(percent * 100)
                    w.searchProgress.set_completion(completion)
                    loop.draw_screen()
    process_run_time = datetime.now() - process_start_time
    return rawEntries


def queryLogProcess_2(poolArgs):
    process_start_time = datetime.now()
    os.nice(10)
    query, log, start_line, end_line = poolArgs
    rawEntries = []
    loop.draw_screen()
    if log[-2:] != 'gz':
        with open(log, mode='r') as p:
            current_line = 1
            for line in p:
                if len(rawEntries) == 1000:
                    s.rl.resultOverflow = True
                    debug('ResultOverflow: %s', s.rl.resultOverflow)
                    rawEntries.append('RESULTSOVERFLOW')
                    break
                if current_line >= start_line and current_line <= end_line:
                    if query in line:
                        if line not in rawEntries:
                            rawEntries.append(line)

    # base_widget = frame.contents['body'][0].base_widget.contents[1][0]
    # original_widget = base_widget.original_widget.original_widget
    # search_progress_box = original_widget.original_widget
    # progress_bar = search_progress_box.original_widget.contents[1][0]
    # progress_bar_text = progress_bar.get_text()[:-1].strip()
    # debug('searchProgress.get_text: %s', progress_bar_text)
    # completion = int(progress_bar_text) + 1
    # debug("completion: %s", completion)
    # progress_bar.set_completion(completion)
    # loop.draw_screen()
    process_run_time = datetime.now() - process_start_time
    debug('Process Run Time: %s', process_run_time)
    return rawEntries


def queryLogProcess2(poolArgs):
    os.nice(20)
    query, log = poolArgs
    debug('Log = %s, filters = %s', log, query)
    rawEntries = []
    starttime = datetime.now()
    if log[-2:] != 'gz':
        tail_args = ["sudo", "tail", "-1", log]
        last_line, _ = subprocess.Popen(
            tail_args,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
            ).communicate()
        debug('Last Line = %s', last_line)
        x = True
        i = 1
        log_file_contents = []
        process_pool = Pool(maxtasksperchild=1)

        results = []
        for resultList in searchedLogs:
            results.extend(resultList)
        while x:
            sed_string = "'" + str(i) + "," + str(i + 999) + "p'"
            searchedLogs = self.process_pool.map(queryLogProcess, sed_string)
            debug('sed_string: %s', sed_string)
            popen_args = ["sudo sed -n " + sed_string + " " + log]
            reqStdOut, _ = subprocess.Popen(
                popen_args,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                shell=True
                ).communicate()
            reqStdOut = reqStdOut.splitlines()
            for line in reqStdOut:
                if query in line:
                    if line not in rawEntries:
                        rawEntries.append(line)
                if line == last_line:
                    break
            i = i + 1000
        debug("log_file_contents: %s", log_file_contents)
    runtime = datetime.now() - starttime
    debug("Runtime: %s", runtime)


def query_for_cpanel_summary(poolArgs):
    os.nice(20)
    result_counts = {}
    log = poolArgs
    email_list = state.list_of_users_emails
    pid = ''
    for email in email_list:
        result_counts[email] = {
            'was_sender': 0,
            'was_recipient': 0,
            'was_bounced': 0
        }
    result_counts['sending_scripts'] = []
    result_counts['script_related_lines'] = []
    if log[-2:] != 'gz':
        with open(log, mode='rb') as f:
            for i, line in enumerate(f):
                pass
            totalLines = i
        with open(log, mode='rb') as p:
            for i, line in enumerate(p):
                try:
                    line.encode('utf-8').strip()
                except UnicodeDecodeError as e:
                    debug(
                        'Log: %s, Line: %s, Offset: %s, %s',
                        log, i, e.start, e.reason)
                else:
                    line.encode('utf-8').strip()
                    for email in email_list:
                        if email in line:
                            x = Entry(line, 'temp')
                            if len(x.recipient) > 2:
                                if email in x.recipient[2]:
                                    result_counts[email]['was_recipient'] += 1
                            if len(x.sendAddr) > 2:
                                if email in x.sendAddr[2]:
                                    result_counts[email]['was_sender'] += 1
                            if len(x.msgType) > 2:
                                if 'bounce' in x.msgType[2]:
                                    result_counts[email]['was_bounced'] += 1
                    if state.cpanel_user in line:
                        if 'cwd=' in line:
                            result_counts['sending_scripts'].append(line)
                            pid = line[line.index('['):line.index(']')]
                    if pid and pid in line:
                        result_counts['script_related_lines'].append(line)
                if i % 1000000 == 0:
                    percent = float(i) / float(totalLines)
                    completion = int(percent * 100)
                    w.searchProgress.set_completion(completion)
                    loop.draw_screen()
    else:
        with gzip.open(log, mode='r') as f:
            for i, line in enumerate(f):
                pass
            totalLines = i
        with gzip.open(log, mode='r') as f:
            for i, line in enumerate(f):
                try:
                    line.encode('utf-8').strip()
                except UnicodeDecodeError as e:
                    debug(
                        'Log: %s, Line: %s, Offset: %s, %s',
                        log, i, e.start, e.reason)
                else:
                    line.encode('utf-8').strip()
                    for email in email_list:
                        if email in line:
                            x = Entry(line, 'temp')
                            if len(x.recipient) > 2:
                                if email in x.recipient[2]:
                                    result_counts[email]['was_recipient'] += 1
                            if len(x.sendAddr) > 2:
                                if email in x.sendAddr[2]:
                                    result_counts[email]['was_sender'] += 1
                    if state.cpanel_user in line:
                        if 'cwd=' in line:
                            result_counts['sending_scripts'].append(line)
                            pid = line[line.index('['):line.index(']')]
                    if pid and pid in line:
                        result_counts['script_related_lines'].append(line)
                if i % 1000000 == 0:
                    percent = float(i) / float(totalLines)
                    completion = int(percent * 100)
                    w.searchProgress.set_completion(completion)
                    loop.draw_screen()
    debug('Result_counts: %s', result_counts)
    return result_counts
s = GlobalSettings()
if __name__ == '__main__':
    debug("\n****\nApplication Start!\n****\n")
    # Initialize Frame Widget
    frame = urwid.Frame(urwid.Filler(w.getText(
        'body', 'Loading...Please Wait', 'center')))
    # Initialize Loop object
    loop = urwid.MainLoop(
        frame,
        s.df.palette,
        unhandled_input=s.unhandled_input,
        handle_mouse=False)
    # Initialize LogFiles
    logFiles = LogFiles()
    uapi = UAPI()
    cpanel_stats = CpanelStats()
    # Initiatlize Views and BodyWidgets
    views = ViewSets()
    body = BodyWidgets()

    # Initialize Entries
    entries = Entries()

    # Initialize Filters
    filters = Filters()

    # Initialize Results and Search Container Objects
    results = Results()
    search = Search()

    # Activate initial screen 'choose_logs'
    views.activate('choose_logs', BODY)

    # Run Forest Run.
    loop.run()
    debug("\n****\nApplication Ended Normally!\n****\n")