FetchData.py

"""This module handles all operations involving fetching reports.

The process of fetching reports is made up of these steps:

1. Each vendor is queried for its supported reports using the SUSHI API
2. The vendor is then queried for each supported report, also using the SUSHI API
3. The raw JSON response is converted to JsonModel objects that make it easier to work with the JSON data.
4. The model objects are then used to create create row objects that will be in the final TSV file.
5. The row objects are then sorted by their primary columns, for example, item reports are sorted by the item column.
6. The sorted rows are then used to create and save a final TSV report file that adheres to the COUNTER 5 standards.
7. After all reports are processed, the database is updated with the new data

.. NOTE::
    All fetch operations are multi-threaded. Each vendor has it's own thread, each report for that vendor
    also has it's own thread. The maximum concurrent vendors and reports (per vendor) can be changed in the settings tab
    on the GUI
"""

from os import path, makedirs
import csv
import json
import requests
import platform
import copy
import ctypes

from PyQt5.QtCore import QObject, QThread, pyqtSignal, QDate, Qt
from PyQt5.QtGui import QStandardItemModel, QStandardItem
from PyQt5.QtWidgets import QPushButton, QDialog, QWidget, QProgressBar, QLabel, QVBoxLayout, QDialogButtonBox, \
    QCheckBox, QDateEdit, QFrame, QHBoxLayout, QSizePolicy, QLineEdit, QListView, QRadioButton, QButtonGroup

import GeneralUtils
from ui import FetchReportsTab, FetchSpecialReportsTab, FetchProgressDialog, ReportResultWidget, VendorResultsWidget
from GeneralUtils import JsonModel
from ManageVendors import Vendor
from Settings import SettingsModel
from ManageDB import UpdateDatabaseWorker
from Constants import *


# region Models
class SupportedReportModel(JsonModel):
    """Models a SUSHI Supported Report"""
    def __init__(self, report_id: str):
        self.report_id = report_id

    @classmethod
    def from_json(cls, json_dict: dict):
        report_id = str(json_dict["Report_ID"]).upper() if "Report_ID" in json_dict else ""

        return cls(report_id)


class PeriodModel(JsonModel):
    """Models a SUSHI Period"""
    def __init__(self, begin_date: str, end_date: str):
        self.begin_date = begin_date
        self.end_date = end_date

    @classmethod
    def from_json(cls, json_dict: dict):
        begin_date = json_dict["Begin_Date"] if "Begin_Date" in json_dict else ""
        end_date = json_dict["End_Date"] if "End_Date" in json_dict else ""

        return cls(begin_date, end_date)


class InstanceModel(JsonModel):
    """Models a SUSHI Instance"""
    def __init__(self, metric_type: str, count: int):
        self.metric_type = metric_type
        self.count = count

    @classmethod
    def from_json(cls, json_dict: dict):
        metric_type = json_dict["Metric_Type"] if "Metric_Type" in json_dict else ""
        count = int(json_dict["Count"]) if "Count" in json_dict else 0

        return cls(metric_type, count)


class PerformanceModel(JsonModel):
    """Models a SUSHI Performance"""
    def __init__(self, period: PeriodModel, instances: list):
        self.period = period
        self.instances = instances

    @classmethod
    def from_json(cls, json_dict: dict):
        period = PeriodModel.from_json(json_dict["Period"]) if "Period" in json_dict else None

        instances = get_models("Instance", InstanceModel, json_dict)

        return cls(period, instances)


class TypeValueModel(JsonModel):
    """Models SUSHI models that are made up of Type and value"""
    def __init__(self, item_type: str, value: str):
        self.item_type = item_type
        self.value = value

    @classmethod
    def from_json(cls, json_dict: dict):
        item_type = str(json_dict["Type"]) if "Type" in json_dict else ""
        value = str(json_dict["Value"]) if "Value" in json_dict else ""

        return cls(item_type, value)


class NameValueModel(JsonModel):
    """Models SUSHI models that are made up of Name and value"""
    def __init__(self, name: str, value: str):
        self.name = name
        self.value = value

    @classmethod
    def from_json(cls, json_dict: dict):
        name = str(json_dict["Name"]) if "Name" in json_dict else ""
        value = str(json_dict["Value"]) if "Value" in json_dict else ""

        return cls(name, value)


class ExceptionModel(JsonModel):
    """Models a SUSHI Exception"""
    def __init__(self, code: int, message: str, severity: str, data: str):
        self.code = code
        self.message = message
        self.severity = severity
        self.data = data

    @classmethod
    def from_json(cls, json_dict: dict):
        code = int(json_dict["Code"]) if "Code" in json_dict else 0
        message = json_dict["Message"] if "Message" in json_dict else ""
        severity = json_dict["Severity"] if "Severity" in json_dict else ""
        data = json_dict["Data"] if "Data" in json_dict else ""

        return cls(code, message, severity, data)


class ReportHeaderModel(JsonModel):
    """Models a SUSHI Report Header"""
    def __init__(self, report_name: str, report_id: str, release: str, institution_name: str, institution_ids: list,
                 report_filters: list, report_attributes: list, exceptions: list, created: str, created_by: str):
        self.report_name = report_name
        self.report_id = report_id
        self.release = release
        self.institution_name = institution_name
        self.institution_ids = institution_ids
        self.report_filters = report_filters
        self.report_attributes = report_attributes
        self.exceptions = exceptions
        self.created = created
        self.created_by = created_by

        # Not part of JSON
        self.major_report_type = None

    @classmethod
    def from_json(cls, json_dict: dict):
        report_name = json_dict["Report_Name"] if "Report_Name" in json_dict else ""
        report_id = str(json_dict["Report_ID"]).upper() if "Report_ID" in json_dict else ""
        release = json_dict["Release"] if "Release" in json_dict else ""
        institution_name = json_dict["Institution_Name"] if "Institution_Name" in json_dict else ""
        created = json_dict["Created"] if "Created" in json_dict else ""
        created_by = json_dict["Created_By"] if "Created_By" in json_dict else ""

        institution_ids = get_models("Institution_ID", TypeValueModel, json_dict)
        report_filters = get_models("Report_Filters", NameValueModel, json_dict)
        report_attributes = get_models("Report_Attributes", NameValueModel, json_dict)
        exceptions = get_models("Exceptions", ExceptionModel, json_dict)

        return cls(report_name, report_id, release, institution_name, institution_ids, report_filters,
                   report_attributes, exceptions, created, created_by)


class ReportModel(JsonModel):
    """Models a SUSHI Report"""
    def __init__(self, report_header: ReportHeaderModel, report_items: list):
        self.report_header = report_header
        self.report_items = report_items

        # Not part of JSON
        self.exceptions = []

    @classmethod
    def from_json(cls, json_dict: dict):
        exceptions = ReportModel.process_exceptions(json_dict)

        report_header = ReportHeaderModel.from_json(json_dict["Report_Header"])
        report_type = report_header.report_id
        major_report_type = GeneralUtils.get_major_report_type(report_type)
        report_header.major_report_type = major_report_type

        report_items = []
        if "Report_Items" in json_dict:
            report_item_dicts = json_dict["Report_Items"]
            if len(report_item_dicts) > 0:
                if major_report_type == MajorReportType.PLATFORM:
                    for report_item_dict in report_item_dicts:
                        report_items.append(PlatformReportItemModel.from_json(report_item_dict))

                elif major_report_type == MajorReportType.DATABASE:
                    for report_item_dict in report_item_dicts:
                        report_items.append(DatabaseReportItemModel.from_json(report_item_dict))

                elif major_report_type == MajorReportType.TITLE:
                    for report_item_dict in report_item_dicts:
                        report_items.append(TitleReportItemModel.from_json(report_item_dict))

                elif major_report_type == MajorReportType.ITEM:
                    for report_item_dict in report_item_dicts:
                        report_items.append(ItemReportItemModel.from_json(report_item_dict))

        report_model = cls(report_header, report_items)
        report_model.exceptions = exceptions
        return report_model

    @classmethod
    def process_exceptions(cls, json_dict: dict) -> list:
        """Gets all exception models in a JSON dict, returns them as a list

        :param json_dict: A JSON dict
        :raises ReportHeaderMissingException: When the report header is missing
        :raises RetryLaterException: When a retry later exception model is received
        :raises UnacceptableCodeException: When the report cannot be processed based on the exception code
        """
        exceptions = []

        if "Exception" in json_dict:
            exceptions.append(ExceptionModel.from_json(json_dict["Exception"]))

        code = int(json_dict["Code"]) if "Code" in json_dict else ""
        message = json_dict["Message"] if "Message" in json_dict else ""
        data = json_dict["Data"] if "Data" in json_dict else ""
        severity = json_dict["Severity"] if "Severity" in json_dict else ""
        if code:
            exceptions.append(ExceptionModel(code, message, severity, data))

        if "Report_Header" in json_dict:
            report_header = ReportHeaderModel.from_json(json_dict["Report_Header"])
            if len(report_header.exceptions) > 0:
                for exception in report_header.exceptions:
                    exceptions.append(exception)
        else:
            raise ReportHeaderMissingException(exceptions)

        for exception in exceptions:
            if exception.code in RETRY_LATER_CODES:
                raise RetryLaterException(exceptions)
            elif exception.code not in ACCEPTABLE_CODES:
                raise UnacceptableCodeException(exceptions)

        return exceptions


class PlatformReportItemModel(JsonModel):
    """Models a SUSHI Platform Report Item"""
    def __init__(self, platform: str, data_type: str, access_method: str, performances: list):
        self.platform = platform
        self.data_type = data_type
        self.access_method = access_method
        self.performances = performances

    @classmethod
    def from_json(cls, json_dict: dict):
        platform = json_dict["Platform"] if "Platform" in json_dict else ""
        data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else ""
        access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else ""

        performances = get_models("Performance", PerformanceModel, json_dict)

        return cls(platform, data_type, access_method, performances)


class DatabaseReportItemModel(JsonModel):
    """Models a SUSHI Database Report Item"""
    def __init__(self, database: str, publisher: str, item_ids: list, publisher_ids: list, platform: str,
                 data_type: str, access_method: str,
                 performances: list):
        self.database = database
        self.publisher = publisher
        self.item_ids = item_ids
        self.publisher_ids = publisher_ids
        self.platform = platform
        self.data_type = data_type
        self.access_method = access_method
        self.performances = performances

    @classmethod
    def from_json(cls, json_dict: dict):
        database = json_dict["Database"] if "Database" in json_dict else ""
        publisher = json_dict["Publisher"] if "Publisher" in json_dict else ""
        platform = json_dict["Platform"] if "Platform" in json_dict else ""
        data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else ""
        access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else ""

        item_ids = get_models("Item_ID", TypeValueModel, json_dict)
        publisher_ids = get_models("Publisher_ID", TypeValueModel, json_dict)
        performances = get_models("Performance", PerformanceModel, json_dict)

        return cls(database, publisher, item_ids, publisher_ids, platform, data_type, access_method, performances)


class TitleReportItemModel(JsonModel):
    """Models a SUSHI Title Report Item"""
    def __init__(self, title: str, item_ids: list, platform: str, publisher: str, publisher_ids: list, data_type: str,
                 section_type: str, yop: str, access_type: str, access_method: str, performances: list):
        self.title = title
        self.item_ids = item_ids
        self.platform = platform
        self.publisher = publisher
        self.publisher_ids = publisher_ids
        self.data_type = data_type
        self.section_type = section_type
        self.yop = yop  # Year of publication
        self.access_type = access_type
        self.access_method = access_method
        self.performances = performances

    @classmethod
    def from_json(cls, json_dict: dict):
        title = json_dict["Title"] if "Title" in json_dict else ""
        platform = json_dict["Platform"] if "Platform" in json_dict else ""
        publisher = json_dict["Publisher"] if "Publisher" in json_dict else ""
        data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else ""
        section_type = json_dict["Section_Type"] if "Section_Type" in json_dict else ""
        yop = json_dict["YOP"] if "YOP" in json_dict else ""
        access_type = json_dict["Access_Type"] if "Access_Type" in json_dict else ""
        access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else ""

        item_ids = get_models("Item_ID", TypeValueModel, json_dict)
        publisher_ids = get_models("Publisher_ID", TypeValueModel, json_dict)
        performances = get_models("Performance", PerformanceModel, json_dict)

        return cls(title, item_ids, platform, publisher, publisher_ids, data_type, section_type, yop, access_type,
                   access_method, performances)


class ItemContributorModel(JsonModel):
    """Models a SUSHI Item Contributor"""
    def __init__(self, item_type: str, name: str, identifier: str):
        self.item_type = item_type
        self.name = name
        self.identifier = identifier

    @classmethod
    def from_json(cls, json_dict: dict):
        item_type = json_dict["Type"] if "Type" in json_dict else ""
        name = json_dict["Name"] if "Name" in json_dict else ""
        identifier = json_dict["Identifier"] if "Identifier" in json_dict else ""

        return cls(item_type, name, identifier)


class ItemParentModel(JsonModel):
    """Models a SUSHI Item Parent"""
    def __init__(self, item_name: str, item_ids: list, item_contributors: list, item_dates: list, item_attributes: list,
                 data_type: str):
        self.item_name = item_name
        self.item_ids = item_ids
        self.item_contributors = item_contributors
        self.item_dates = item_dates
        self.item_attributes = item_attributes
        self.data_type = data_type

    @classmethod
    def from_json(cls, json_dict: dict):
        item_name = json_dict["Item_Name"] if "Item_Name" in json_dict else ""
        data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else ""

        item_ids = get_models("Item_ID", TypeValueModel, json_dict)
        item_contributors = get_models("Item_Contributors", ItemContributorModel, json_dict)
        item_dates = get_models("Item_Dates", TypeValueModel, json_dict)
        item_attributes = get_models("Item_Attributes", TypeValueModel, json_dict)

        return cls(item_name, item_ids, item_contributors, item_dates, item_attributes, data_type)


class ItemComponentModel(JsonModel):
    """Models a SUSHI Item Component"""
    def __init__(self, item_name: str, item_ids: list, item_contributors: list, item_dates: list, item_attributes: list,
                 data_type: str, performances: list):
        self.item_name = item_name
        self.item_ids = item_ids
        self.item_contributors = item_contributors
        self.item_dates = item_dates
        self.item_attributes = item_attributes
        self.data_type = data_type
        self.performances = performances

    @classmethod
    def from_json(cls, json_dict: dict):
        item_name = json_dict["Item_Name"] if "Item_Name" in json_dict else ""
        data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else ""

        item_ids = get_models("Item_ID", TypeValueModel, json_dict)
        item_contributors = get_models("Item_Contributors", ItemContributorModel, json_dict)
        item_dates = get_models("Item_Dates", TypeValueModel, json_dict)
        item_attributes = get_models("Item_Attributes", TypeValueModel, json_dict)
        performances = get_models("Performance", PerformanceModel, json_dict)

        return cls(item_name, item_ids, item_contributors, item_dates, item_attributes, data_type, performances)


class ItemReportItemModel(JsonModel):
    """Models a SUSHI Item Report model"""
    def __init__(self, item: str, item_ids: list, item_contributors: list, item_dates: list, item_attributes: list,
                 platform: str, publisher: str, publisher_ids: list, item_parent: ItemParentModel,
                 item_components: list, data_type: str, yop: str, access_type: str,
                 access_method: str, performances: list):
        self.item = item
        self.item_ids = item_ids
        self.item_contributors = item_contributors
        self.item_dates = item_dates
        self.item_attributes = item_attributes
        self.platform = platform
        self.publisher = publisher
        self.publisher_ids = publisher_ids
        self.item_parent = item_parent
        self.item_components = item_components
        self.data_type = data_type
        self.yop = yop  # Year of publication
        self.access_type = access_type
        self.access_method = access_method
        self.performances = performances

    @classmethod
    def from_json(cls, json_dict: dict):
        item = json_dict["Item"] if "Item" in json_dict else ""
        platform = json_dict["Platform"] if "Platform" in json_dict else ""
        publisher = json_dict["Publisher"] if "Publisher" in json_dict else ""
        data_type = json_dict["Data_Type"] if "Data_Type" in json_dict else ""
        yop = json_dict["YOP"] if "YOP" in json_dict else ""
        access_type = json_dict["Access_Type"] if "Access_Type" in json_dict else ""
        access_method = json_dict["Access_Method"] if "Access_Method" in json_dict else ""

        item_parent = ItemParentModel.from_json(json_dict["Item_Parent"]) if "Item_Parent" in json_dict else None

        item_ids = get_models("Item_ID", TypeValueModel, json_dict)
        item_contributors = get_models("Item_Contributors", ItemContributorModel, json_dict)
        item_dates = get_models("Item_Dates", TypeValueModel, json_dict)
        item_attributes = get_models("Item_Attributes", TypeValueModel, json_dict)
        publisher_ids = get_models("Publisher_ID", TypeValueModel, json_dict)
        item_components = get_models("Item_Component", ItemComponentModel, json_dict)
        performances = get_models("Performance", PerformanceModel, json_dict)

        return cls(item, item_ids, item_contributors, item_dates, item_attributes, platform, publisher, publisher_ids,
                   item_parent, item_components, data_type, yop, access_type, access_method, performances)


# endregion


# region Custom Exceptions
class RetryLaterException(Exception):
    """An exception raised when a retry later exception code is received in an exception model"""
    def __init__(self, exceptions: list):
        self.exceptions = exceptions


class ReportHeaderMissingException(Exception):
    """An exception raised when a report header is missing from a report"""
    def __init__(self, exceptions: list):
        self.exceptions = exceptions


class UnacceptableCodeException(Exception):
    """An exception raised when a report cannot be processed based on an exception code"""
    def __init__(self, exceptions: list):
        self.exceptions = exceptions
# endregion


def exception_models_to_message(exceptions: list) -> str:
    """Formats a list of exception models into a single string """
    message = ""
    for exception in exceptions:
        if message: message += "\n\n"
        message += f"Code: {exception.code}" \
                   f"\nMessage: {exception.message}" \
                   f"\nSeverity: {exception.severity}" \
                   f"\nData: {exception.data}"

    return message


def get_models(model_key: str, model_type, json_dict: dict) -> list:
    """This converts json lists into a list of the specified SUSHI model type

    :param model_key: The target key to get the list of JSONObjects
    :param model_type: The target model type, e.g PerformanceModel
    :param json_dict: The JSON dict to get the list from
    """
    # Some vendors sometimes return a single dict even when the standard specifies a list,
    # we need to check for that
    models = []
    if model_key in json_dict and json_dict[model_key] is not None:
        if type(json_dict[model_key]) is list:
            model_dicts = json_dict[model_key]
            for model_dict in model_dicts:
                models.append(model_type.from_json(model_dict))

        elif type(json_dict[model_key]) is dict:
            models.append(model_type.from_json(json_dict[model_key]))

    return models


def get_month_years(begin_date: QDate, end_date: QDate) -> list:
    """Returns a list of month-year (MMM-yyyy) strings within a date range"""
    month_years = []
    num_months = (end_date.year() - begin_date.year()) * 12 + \
                 (end_date.month() - begin_date.month()) + 1

    for i in range(num_months):
        month_years.append(begin_date.addMonths(i).toString("MMM-yyyy"))

    return month_years


class SpecialReportOptions:
    """This holds all the parameters that are used to process a special report

    The options are stored as tuples, (option has non-default value, option type, option name, list of option values)
    """
    def __init__(self):
        # PR, DR, TR, IR
        self.data_type = False, SpecialOptionType.AP, "Data_Type", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.access_method = False, SpecialOptionType.AP, "Access_Method", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.metric_type = False, SpecialOptionType.POS, "Metric_Type", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.exclude_monthly_details = False, SpecialOptionType.TO, None, None
        # TR, IR
        current_date = QDate.currentDate()
        self.yop = False, SpecialOptionType.ADP, "YOP", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.access_type = False, SpecialOptionType.AP, "Access_Type", [DEFAULT_SPECIAL_OPTION_VALUE]
        # TR
        self.section_type = False, SpecialOptionType.AP, "Section_Type", [DEFAULT_SPECIAL_OPTION_VALUE]
        # IR
        self.authors = False, SpecialOptionType.AO, "Authors", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.publication_date = False, SpecialOptionType.AO, "Publication_Date", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.article_version = False, SpecialOptionType.AO, "Article_Version", [DEFAULT_SPECIAL_OPTION_VALUE]
        self.include_component_details = False, SpecialOptionType.POB, None, None
        self.include_parent_details = False, SpecialOptionType.POB, None, None


class ReportRow:
    """This models a row in the generated report, it contains every possible column

    :param begin_date: The begin date of the request, used to populate the month columns in the report
    :param end_date: The end date of the request, used to populate the month columns in the report
    """
    def __init__(self, begin_date: QDate, end_date: QDate):
        self.database = ""
        self.title = ""
        self.item = ""
        self.publisher = ""
        self.publisher_id = ""
        self.platform = ""
        self.authors = ""
        self.publication_date = ""
        self.article_version = ""
        self.doi = ""
        self.proprietary_id = ""
        self.online_issn = ""
        self.print_issn = ""
        self.linking_issn = ""
        self.isbn = ""
        self.uri = ""
        self.parent_title = ""
        self.parent_authors = ""
        self.parent_publication_date = ""
        self.parent_article_version = ""
        self.parent_data_type = ""
        self.parent_doi = ""
        self.parent_proprietary_id = ""
        self.parent_online_issn = ""
        self.parent_print_issn = ""
        self.parent_linking_issn = ""
        self.parent_isbn = ""
        self.parent_uri = ""
        self.component_title = ""
        self.component_authors = ""
        self.component_publication_date = ""
        self.component_data_type = ""
        self.component_doi = ""
        self.component_proprietary_id = ""
        self.component_online_issn = ""
        self.component_print_issn = ""
        self.component_linking_issn = ""
        self.component_isbn = ""
        self.component_uri = ""
        self.data_type = ""
        self.section_type = ""
        self.yop = ""
        self.access_type = ""
        self.access_method = ""
        self.metric_type = ""
        self.total_count = 0

        self.month_counts = {}

        for month_year_str in get_month_years(begin_date, end_date):
            self.month_counts[month_year_str] = 0


class RequestData:
    """This holds the data about a report request

    :param vendor: The vendor being processed
    :param target_report_types: The report types to attempt to fetch
    :param begin_date: The begin date to specify in the request
    :param end_date: The end date to specify in the request
    :param save_location: Where the generated report should be saved
    :param settings: The system's settings object
    :param special_options: Special options if fetching a special report
    """
    def __init__(self, vendor: Vendor, target_report_types: list, begin_date: QDate, end_date: QDate,
                 save_location: str, settings: SettingsModel, special_options: SpecialReportOptions = None):
        self.vendor = vendor
        self.target_report_types = target_report_types
        self.begin_date = begin_date
        self.end_date = end_date
        self.save_location = save_location
        self.settings = settings
        self.special_options = special_options


class ProcessResult:
    """This holds the results of an fetch process

    :param vendor: The target vendor
    :param report_type: The target report type
    """
    def __init__(self, vendor: Vendor, report_type: str = None):
        self.vendor = vendor
        self.report_type = report_type
        self.completion_status = CompletionStatus.SUCCESSFUL
        self.message = ""
        self.retry = False
        self.file_name = ""
        self.file_dir = ""
        self.file_path = ""
        self.protected_file_path = ""
        self.year = ""


class FetchReportsAbstract:
    def __init__(self, vendors: list, settings: SettingsModel, widget: QWidget):
        """This contains common functionality shared between classes that fetch reports

        :param vendors: The list of vendors in the system
        :param settings: The system's user settings
        :param widget: The widget of the tab that this class controls
        """

        # region General
        self.widget = widget
        self.vendors = []
        self.update_vendors(vendors)
        self.selected_data = []  # List of ReportData Objects
        self.retry_data = []  # List of (Vendor, list[report_types])>
        self.vendor_workers = {}  # <k = worker_id, v = (VendorWorker, Thread)>
        self.started_processes = 0
        self.completed_processes = 0
        self.total_processes = 0
        self.begin_date = QDate()
        self.end_date = QDate()
        self.selected_options = None
        self.save_dir = ""
        self.is_cancelling = False
        self.is_yearly_fetch = False
        self.settings = settings
        self.database_report_data = []
        # endregion

        # region Fetch Progress Dialog
        self.fetch_progress_dialog: QDialog = None
        self.progress_bar: QProgressBar = None
        self.status_label: QLabel = None
        self.scroll_contents: QWidget = None
        self.scroll_layout: QVBoxLayout = None
        self.ok_button: QPushButton = None
        self.retry_button: QPushButton = None
        self.cancel_button: QPushButton = None

        self.vendor_result_widgets = {}  # <k = vendor name, v = (VendorResultsWidget, VendorResultsUI)>
        # endregion

        # region Update Database Dialog
        self.is_updating_database = False
        self.add_to_database = True
        self.database_thread = None
        self.database_worker = None

        # endregion

    def on_vendors_changed(self, vendors: list):
        """Handles the signal emitted when the system's vendor list is updated

        :param vendors: An updated list of the system's vendors
        """
        self.update_vendors(vendors)
        self.update_vendors_ui()

    def update_vendors(self, vendors: list):
        """ Updates the local copy of vendors that support report fetching (SUSHI)

        :param vendors: A list of vendors
        """
        self.vendors = []
        for vendor in vendors:
            if vendor.is_non_sushi: continue
            self.vendors.append(vendor)

    def update_vendors_ui(self):
        """Updates the UI to show vendors that support report fetching (SUSHI)"""
        raise NotImplementedError()

    def fetch_vendor_data(self, request_data: RequestData):
        """Initiates the process to fetch reports from a vendor

        This creates a new thread to work on this vendor

        :param request_data: The request data for this vendor request
        """
        worker_id = request_data.vendor.name
        if worker_id in self.vendor_workers: return  # Avoid processing a vendor twice

        vendor_worker = VendorWorker(worker_id, request_data)
        vendor_worker.worker_finished_signal.connect(self.on_vendor_worker_finished)
        vendor_thread = QThread()
        self.vendor_workers[worker_id] = vendor_worker, vendor_thread
        vendor_worker.moveToThread(vendor_thread)
        vendor_thread.started.connect(vendor_worker.work)
        vendor_thread.finished.connect(vendor_thread.deleteLater)

        vendor_thread.start()

        if self.settings.show_debug_messages: print(f"{worker_id}: Added a process, total processes: {self.total_processes}")
        self.update_results_ui(request_data.vendor)

    def update_results_ui(self, vendor: Vendor, vendor_result: ProcessResult = None, report_results: list = None):
        """Updates the fetch progress dialog to show results

        :param vendor: The vendor being updated
        :param vendor_result: The result of the vendor
        :param report_results: The results of the vendor's reports
        """
        self.progress_bar.setValue(int((self.completed_processes / self.total_processes) * 100))
        if not self.is_cancelling and self.completed_processes != self.total_processes:
            self.status_label.setText(f"Vendor progress: {self.completed_processes}/{self.total_processes}")

        if vendor.name in self.vendor_result_widgets:
            vendor_results_widget, vendor_results_ui = self.vendor_result_widgets[vendor.name]
            vertical_layout = vendor_results_ui.results_frame.layout()
            status_label = vendor_results_ui.status_label
        else:
            vendor_results_widget = QWidget(self.scroll_contents)
            vendor_results_ui = VendorResultsWidget.Ui_VendorResultsWidget()
            vendor_results_ui.setupUi(vendor_results_widget)
            vendor_results_ui.vendor_label.setText(vendor.name)
            vertical_layout = vendor_results_ui.results_frame.layout()

            status_label = vendor_results_ui.status_label
            frame = vendor_results_ui.results_frame
            expand_button = vendor_results_ui.expand_button
            collapse_button = vendor_results_ui.collapse_button

            status_label.setText("Working...")
            frame.hide()
            expand_button.clicked.connect(lambda: frame.show())
            collapse_button.clicked.connect(lambda: frame.hide())

            self.vendor_result_widgets[vendor.name] = vendor_results_widget, vendor_results_ui
            self.scroll_layout.addWidget(vendor_results_widget)

        if vendor_result is None: return

        status_label.setText("Done")
        result_widget = self.get_result_widget(vendor, vendor_results_widget, vendor_result)
        vertical_layout.addWidget(result_widget)

        for report_result in report_results:
            result_widget = self.get_result_widget(vendor, vendor_results_widget, report_result)
            vertical_layout.addWidget(result_widget)

    def get_result_widget(self, vendor: Vendor, vendor_widget: QWidget, process_result: ProcessResult) -> QWidget:
        """This creates a result widget for either a vendor or a vendor's report

        :param vendor: The target vendor
        :param vendor_widget: The vendor's widget in the fetch progress dialog
        :param process_result: The result to show
        """
        completion_status = process_result.completion_status
        report_result_widget = QWidget(vendor_widget)
        report_result_ui = ReportResultWidget.Ui_ReportResultWidget()
        report_result_ui.setupUi(report_result_widget)

        if process_result.message:
            report_result_ui.message_label.setText(process_result.message)
        else:
            report_result_ui.message_label.hide()

        if process_result.report_type is not None:  # If this is a report result, not vendor
            report_result_ui.report_type_label.setText(process_result.report_type)
            if completion_status == CompletionStatus.SUCCESSFUL or completion_status == CompletionStatus.WARNING:
                report_result_ui.file_frame.show()

                report_result_ui.folder_button.clicked.connect(
                    lambda: GeneralUtils.open_file_or_dir(process_result.file_dir))

                report_result_ui.file_label.setText(f"Saved as: {process_result.file_name}")
                report_result_ui.file_label.mousePressEvent = \
                    lambda event: GeneralUtils.open_file_or_dir(process_result.file_path)
            else:
                report_result_ui.file_frame.hide()
        else:
            report_result_ui.report_type_label.setText("Target Reports")
            report_result_ui.file_frame.hide()
            report_result_ui.retry_frame.hide()

        report_result_ui.success_label.setText(process_result.completion_status.value)
        if completion_status == CompletionStatus.FAILED:
            report_result_ui.retry_check_box.stateChanged.connect(
                lambda checked_state: self.on_report_to_retry_toggled(checked_state, vendor, process_result.report_type))
        else:
            report_result_ui.retry_frame.hide()

        return report_result_widget

    def on_vendor_worker_finished(self, worker_id: str):
        """Handles the signal emmited when a vendor worker has finished

        :param worker_id: The worker ID of the vendor
        """
        self.completed_processes += 1

        thread: QThread
        worker: VendorWorker
        worker, thread = self.vendor_workers[worker_id]
        self.update_results_ui(worker.vendor, worker.process_result, worker.report_process_results)

        if self.is_yearly_fetch:
            process_result: ProcessResult
            for process_result in worker.report_process_results:
                if process_result.completion_status != CompletionStatus.SUCCESSFUL:
                    continue

                self.database_report_data.append({'file': process_result.protected_file_path,
                                                  'vendor': process_result.vendor.name,
                                                  'year': process_result.year})

        worker.deleteLater()
        thread.quit()
        thread.wait()
        self.vendor_workers.pop(worker_id, None)

        if self.started_processes < self.total_processes and not self.is_cancelling:
            request_data = self.selected_data[self.started_processes]
            self.fetch_vendor_data(request_data)
            self.started_processes += 1

        elif len(self.vendor_workers) == 0: self.finish_fetching_reports()

    def start_progress_dialog(self, window_title: str):
        """Sets up and shows the fetch progress dialog

        :param window_title: The title of the fetch progress dialog
        """
        self.vendor_result_widgets = {}

        if self.fetch_progress_dialog: self.fetch_progress_dialog.close()
        self.fetch_progress_dialog = QDialog(self.widget, flags=Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint)
        fetch_progress_ui = FetchProgressDialog.Ui_FetchProgressDialog()
        fetch_progress_ui.setupUi(self.fetch_progress_dialog)
        self.fetch_progress_dialog.setWindowTitle(window_title)

        self.progress_bar = fetch_progress_ui.progress_bar
        self.status_label = fetch_progress_ui.status_label
        self.scroll_contents = fetch_progress_ui.scroll_area_widget_contents
        self.scroll_layout = fetch_progress_ui.scroll_area_vertical_layout

        self.ok_button = fetch_progress_ui.buttonBox.button(QDialogButtonBox.Ok)
        self.retry_button = fetch_progress_ui.buttonBox.button(QDialogButtonBox.Retry)
        self.cancel_button = fetch_progress_ui.buttonBox.button(QDialogButtonBox.Cancel)

        self.ok_button.setEnabled(False)
        self.retry_button.setEnabled(False)
        self.retry_button.setText("Retry Selected")
        self.ok_button.clicked.connect(lambda: self.fetch_progress_dialog.close())
        self.retry_button.clicked.connect(lambda: self.retry_selected_reports(window_title))
        self.cancel_button.clicked.connect(self.cancel_workers)

        self.status_label.setText("Starting...")
        self.fetch_progress_dialog.show()

    def on_report_to_retry_toggled(self, checked_state: int, vendor: Vendor, report_type):
        """Handles the signal emmited when a report result's retry checkbox is toggled

        The report is added the the list of reports to be retried when the 'retry selected reports' button is clicked

        :param checked_state: The new checked state
        :param vendor: The vendor that this report belongs to
        :param report_type: The report type of the report
        """
        if checked_state == Qt.Checked:
            found = False
            for i in range(len(self.retry_data)):
                v, report_types = self.retry_data[i]
                if v == vendor:
                    report_types.append(report_type)
                    found = True

            if not found:
                self.retry_data.append((vendor, [report_type]))

        elif checked_state == Qt.Unchecked:
            for i in range(len(self.retry_data)):
                v, report_types = self.retry_data[i]
                if v == vendor:
                    report_types.remove(report_type)
                    if len(report_types) == 0: self.retry_data.pop(i)

    def retry_selected_reports(self, progress_window_title: str):
        """Retries the selected reports to retry

        :param progress_window_title: The title of the fetch progress dialog
        """
        if len(self.retry_data) == 0:
            GeneralUtils.show_message("No report selected")
            return

        self.selected_data = []
        for vendor, report_types in self.retry_data:
            request_data = RequestData(vendor, report_types, self.begin_date, self.end_date, self.save_dir,
                                       self.settings, self.selected_options)
            self.selected_data.append(request_data)

        self.start_progress_dialog(progress_window_title)
        self.retry_data = []

        self.total_processes = len(self.selected_data)
        self.started_processes = 0
        concurrent_vendors = self.settings.concurrent_vendors
        while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors:
            request_data = self.selected_data[self.started_processes]
            self.fetch_vendor_data(request_data)
            self.started_processes += 1

    def finish_fetching_reports(self):
        """Finishes up the fetch process"""
        self.started_processes = 0
        self.completed_processes = 0
        self.total_processes = 0
        self.is_cancelling = False
        self.cancel_button.setEnabled(False)

        # Start updating database...
        if self.is_yearly_fetch and len(self.database_report_data) > 0:
            if not self.start_updating_database(): self.finish_updating_database()
        else:
            self.finish_updating_database()

    def finish_updating_database(self):
        """Finishes up the database update process"""
        self.is_updating_database = False
        self.database_report_data = []

        self.ok_button.setEnabled(True)
        self.retry_button.setEnabled(True)
        self.status_label.setText("Done!")
        if self.settings.show_debug_messages: print("Fin!")

    def cancel_workers(self):
        """Sends a cancel signal to all vendor workers, updates the UI accordingly"""
        self.is_cancelling = True
        self.total_processes = self.started_processes
        self.status_label.setText(f"Cancelling... (Waiting for started requests to finish)")
        for worker, thread in self.vendor_workers.values():
            worker.set_cancelling()

    def is_yearly_range(self, begin_date: QDate, end_date: QDate) -> bool:
        """Checks if a date range will retrieve all available reports for one year

        :param begin_date: The begin date
        :param end_date: The end date
        """
        current_date = QDate.currentDate()

        if begin_date.year() != end_date.year() or begin_date.year() > current_date.year():
            return False

        if begin_date.year() == current_date.year():
            if begin_date.month() == 1 and end_date.month() == max(current_date.month() - 1, 1):
                return True
        else:
            if begin_date.month() == 1 and end_date.month() == 12:
                return True

        return False

    def start_updating_database(self) -> bool:
        """Starts a thread to update the database. Returns True if successfully started"""
        if self.is_updating_database:
            if self.settings.show_debug_messages: print("Database is already updating")
            return False

        self.is_updating_database = True
        self.status_label.setText("Updating database...")

        self.database_thread = QThread()
        self.database_worker = UpdateDatabaseWorker(self.database_report_data, False)
        self.database_worker.moveToThread(self.database_thread)

        def on_progress_changed(progress: int):
            self.progress_bar.setValue(int((progress / len(self.database_report_data)) * 100))

        def on_worker_finished(code):
            self.database_thread.quit()
            self.database_thread.wait()
            self.finish_updating_database()

        self.database_worker.progress_changed_signal.connect(on_progress_changed)
        self.database_worker.worker_finished_signal.connect(on_worker_finished)

        self.database_thread.started.connect(self.database_worker.work)
        self.database_thread.start()

        return True


class FetchReportsController(FetchReportsAbstract):
    """Controls the Fetch Reports tab

    This class fetches master and standard reports with default parameters. The generated TSV files only include
    the mandatory columns for each report type

    :param vendors: The list of vendors in the system
    :param settings: The system's user settings
    :param widget: The fetch reports widget
    :param fetch_reports_ui: The UI for the fetch reports widget
    """
    def __init__(self, vendors: list, settings: SettingsModel, widget: QWidget,
                 fetch_reports_ui: FetchReportsTab.Ui_fetch_reports_tab):
        super().__init__(vendors, settings, widget)

        # region General
        current_date = QDate.currentDate()
        begin_date = QDate(current_date.year(), 1, 1)
        end_date = QDate(current_date.year(), max(current_date.month() - 1, 1), 1)
        self.fetch_all_begin_date = QDate(begin_date)
        self.adv_begin_date = QDate(begin_date)
        self.fetch_all_end_date = QDate(end_date)
        self.adv_end_date = QDate(end_date)
        self.is_last_fetch_advanced = False
        # endregion

        # region Start Fetch Buttons
        self.fetch_all_btn = fetch_reports_ui.fetch_all_data_button
        self.fetch_all_btn.clicked.connect(self.fetch_all_basic_data)

        self.fetch_adv_btn = fetch_reports_ui.fetch_advanced_button
        self.fetch_adv_btn.clicked.connect(self.fetch_advanced_data)
        # endregion

        # region Vendors
        self.vendor_list_view = fetch_reports_ui.vendors_list_view_fetch
        self.vendor_list_model = QStandardItemModel(self.vendor_list_view)
        self.vendor_list_view.setModel(self.vendor_list_model)
        self.update_vendors_ui()

        self.select_vendors_btn = fetch_reports_ui.select_vendors_button_fetch
        self.select_vendors_btn.clicked.connect(self.select_all_vendors)
        self.deselect_vendors_btn = fetch_reports_ui.deselect_vendors_button_fetch
        self.deselect_vendors_btn.clicked.connect(self.deselect_all_vendors)
        # endregion

        # region Report Types
        self.report_type_list_view = fetch_reports_ui.report_types_list_view
        self.report_type_list_model = QStandardItemModel(self.report_type_list_view)
        self.report_type_list_view.setModel(self.report_type_list_model)
        for report_type in ALL_REPORTS:
            item = QStandardItem(report_type)
            item.setCheckable(True)
            item.setEditable(False)
            self.report_type_list_model.appendRow(item)

        self.select_report_types_btn = fetch_reports_ui.select_report_types_button_fetch
        self.select_report_types_btn.clicked.connect(self.select_all_report_types)
        self.deselect_report_types_btn = fetch_reports_ui.deselect_report_types_button_fetch
        self.deselect_report_types_btn.clicked.connect(self.deselect_all_report_types)

        self.report_types_help_btn = fetch_reports_ui.report_types_help_button
        self.report_types_help_btn.clicked.connect(
            lambda: GeneralUtils.show_message("Only reports supported by each respective vendor will be retrieved, "
                                              "and unsupported reports will be listed in \"Expand\" results"))
        # endregion

        # region Custom Directory
        self.custom_dir_frame = fetch_reports_ui.custom_dir_frame
        self.custom_dir_frame.hide()
        self.custom_dir_message_frame = fetch_reports_ui.frame
        self.custom_dir_message_frame.hide()
        self.custom_dir_message_frame2 = fetch_reports_ui.frame_2
        self.custom_dir_message_frame2.hide()
        self.custom_dir_frame_message1 = fetch_reports_ui.label_38
        self.custom_dir_frame_message2 = fetch_reports_ui.label
        self.custom_dir_edit = fetch_reports_ui.custom_dir_edit
        self.custom_dir_edit.setText(self.settings.other_directory)
        self.custom_dir_button = fetch_reports_ui.custom_dir_button
        self.custom_dir_button.clicked.connect(self.on_custom_dir_clicked)
        self.date_range_help_btn = fetch_reports_ui.date_range_help_button
        self.date_range_help_btn.clicked.connect(
            lambda: GeneralUtils.show_message("Reports run for date ranges that represent a normal Jan-Dec calendar" 
                                              " year, or Jan-last month for calendar years in progress, will be added" 
                                              " to (or updated in) the search database. All other date ranges will be " 
                                              "saved in the specified folder but not added to the search database."))
        self.date_range_help_btn2 = fetch_reports_ui.date_range_help_button2
        self.date_range_help_btn2.clicked.connect(
            lambda: GeneralUtils.show_message("Reports run for date ranges that represent a normal Jan-Dec calendar"
                                              " year, or Jan-last month for calendar years in progress, will be added"
                                              " to (or updated in) the search database. All other date ranges will be "
                                              "saved in the specified folder but not added to the search database."))
        self.date_range_help_btn3 = fetch_reports_ui.date_range_help_button3
        self.date_range_help_btn3.clicked.connect(
            lambda: GeneralUtils.show_message("See Other Reports Directory setting in Settings for default"))

        # endregion

        # region Date Edits
        self.all_date_edit = fetch_reports_ui.All_reports_edit_fetch
        self.all_date_edit.setDate(self.fetch_all_begin_date)
        self.all_date_edit.dateChanged.connect(self.on_fetch_all_date_changed)

        self.begin_date_edit_year = fetch_reports_ui.begin_date_edit_fetch_year
        self.begin_date_edit_year.setDate(self.adv_begin_date)
        self.begin_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "adv_begin"))

        self.end_date_edit_year = fetch_reports_ui.end_date_edit_fetch_year
        self.end_date_edit_year.setDate(self.adv_end_date)
        self.end_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "adv_end"))

        self.begin_month_combo_box = fetch_reports_ui.begin_month_combo_box
        for month in MONTH_NAMES:
            self.begin_month_combo_box.addItem(month)
        self.begin_month_combo_box.currentIndexChanged.connect(
            lambda index: self.on_date_month_changed(index + 1, "adv_begin"))
        self.begin_month_combo_box.setCurrentIndex(self.adv_begin_date.month() - 1)

        self.end_month_combo_box = fetch_reports_ui.end_month_combo_box
        for month in MONTH_NAMES:
            self.end_month_combo_box.addItem(month)
        self.end_month_combo_box.currentIndexChanged.connect(
            lambda index: self.on_date_month_changed(index + 1, "adv_end"))
        self.end_month_combo_box.setCurrentIndex(self.adv_end_date.month() - 1)

        # endregion

    def update_vendors_ui(self):
        """Updates the UI to show vendors that support report fetching (SUSHI)"""
        self.vendor_list_model.clear()
        for vendor in self.vendors:
            item = QStandardItem(vendor.name)
            item.setCheckable(True)
            item.setEditable(False)
            self.vendor_list_model.appendRow(item)

    def on_fetch_all_date_changed(self, date: QDate):
        """Handles the signal emitted when the 'fetch all' date is changed

        :param date: The new date
        """
        current_date = QDate.currentDate()
        if date.year() == current_date.year():
            self.fetch_all_begin_date = QDate(current_date.year(), 1, 1)
            self.fetch_all_end_date = QDate(current_date.year(), max(current_date.month() - 1, 1), 1)
        elif date.year() < current_date.year():
            self.fetch_all_begin_date = QDate(date.year(), 1, 1)
            self.fetch_all_end_date = QDate(date.year(), 12, 1)
        else:
            self.all_date_edit.setDate(current_date)

    def on_date_year_changed(self, date: QDate, date_type: str):
        """Handles the signal emitted when a date's year is changed

        :param date: The new date
        :param date_type: The date to be updated
        """
        if date_type == "adv_begin":
            self.adv_begin_date = QDate(date.year(), self.adv_begin_date.month(), self.adv_begin_date.day())

        elif date_type == "adv_end":
            self.adv_end_date = QDate(date.year(), self.adv_end_date.month(), self.adv_end_date.day())

        if self.is_yearly_range(self.adv_begin_date, self.adv_end_date):
            self.custom_dir_frame.hide()
            self.custom_dir_message_frame.hide()
            self.custom_dir_message_frame2.hide()
        else:
            self.custom_dir_frame.show()
            if self.custom_dir_frame_message_show(self.adv_begin_date, self.adv_end_date):
                self.custom_dir_message_frame2.show()
                self.custom_dir_message_frame.hide()
            else:
                self.custom_dir_message_frame.show()
                self.custom_dir_message_frame2.hide()

    def on_date_month_changed(self, month: int, date_type: str):
        """Handles the signal emitted when a date's month is changed

        :param month: The new month
        :param date_type: The date to be updated
        """
        if date_type == "adv_begin":
            self.adv_begin_date = QDate(self.adv_begin_date.year(), month, self.adv_begin_date.day())

        elif date_type == "adv_end":
            self.adv_end_date = QDate(self.adv_end_date.year(), month, self.adv_end_date.day())

        if self.is_yearly_range(self.adv_begin_date, self.adv_end_date):
            self.custom_dir_frame.hide()
            self.custom_dir_message_frame.hide()
            self.custom_dir_message_frame2.hide()
        else:
            self.custom_dir_frame.show()
            if self.custom_dir_frame_message_show(self.adv_begin_date, self.adv_end_date):
                self.custom_dir_message_frame2.show()
                self.custom_dir_message_frame.hide()
            else:
                self.custom_dir_message_frame.show()
                self.custom_dir_message_frame2.hide()

    def custom_dir_frame_message_show(self, begin_date: QDate, end_date: QDate) -> bool:
        """Checks which message will show on the custom dir frame

        :param begin_date: The begin date
        :param end_date: The end date
        """
        current_date = QDate.currentDate()

        if begin_date.year() == end_date.year() == current_date.year():
            if begin_date.month() == 1 and end_date.month() == 12:
                return True

        return False

    def on_custom_dir_clicked(self):
        """Handles the signal emitted when the choose custom directory button is clicked"""
        dir_path = GeneralUtils.choose_directory()
        if dir_path: self.custom_dir_edit.setText(dir_path)

    def select_all_vendors(self):
        """Checks all vendors in the vendors list view"""
        for i in range(self.vendor_list_model.rowCount()):
            self.vendor_list_model.item(i).setCheckState(Qt.Checked)

    def deselect_all_vendors(self):
        """Un-checks all vendors in the vendors list view"""
        for i in range(self.vendor_list_model.rowCount()):
            self.vendor_list_model.item(i).setCheckState(Qt.Unchecked)

    def select_all_report_types(self):
        """Checks all report types in the report types list view"""
        for i in range(self.report_type_list_model.rowCount()):
            self.report_type_list_model.item(i).setCheckState(Qt.Checked)

    def deselect_all_report_types(self):
        """Un-checks all report types in the report types list view"""
        for i in range(self.report_type_list_model.rowCount()):
            self.report_type_list_model.item(i).setCheckState(Qt.Unchecked)

    def fetch_all_basic_data(self):
        """Fetches all reports for the selected year"""
        if self.total_processes > 0 or self.is_updating_database:
            GeneralUtils.show_message(f"Waiting for pending processes to complete...")
            if self.settings.show_debug_messages: print(f"Waiting for pending processes to complete...")
            return

        if len(self.vendors) == 0:
            GeneralUtils.show_message("Vendor list is empty")
            return

        self.begin_date = self.fetch_all_begin_date
        self.end_date = self.fetch_all_end_date
        if self.begin_date > self.end_date:
            GeneralUtils.show_message("\'Begin Date\' is higher than \'End Date\'")
            return

        self.is_yearly_fetch = True
        self.save_dir = self.settings.yearly_directory
        self.selected_data = []
        for i in range(len(self.vendors)):
            if self.vendors[i].is_non_sushi: continue

            request_data = RequestData(self.vendors[i], ALL_REPORTS, self.begin_date, self.end_date,
                                       self.save_dir, self.settings)
            self.selected_data.append(request_data)

        self.is_last_fetch_advanced = False
        self.start_progress_dialog("Fetch Reports Progress")
        self.retry_data = []

        self.total_processes = len(self.selected_data)
        self.started_processes = 0
        concurrent_vendors = self.settings.concurrent_vendors
        while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors:
            request_data = self.selected_data[self.started_processes]
            self.fetch_vendor_data(request_data)
            self.started_processes += 1

    def fetch_advanced_data(self):
        """Fetches reports based on the selected options in the advanced view of the UI"""
        if self.total_processes > 0 or self.is_updating_database:
            GeneralUtils.show_message(f"Waiting for pending processes to complete...")
            if self.settings.show_debug_messages: print(f"Waiting for pending processes to complete...")
            return

        if len(self.vendors) == 0:
            GeneralUtils.show_message("Vendor list is empty")
            return

        self.begin_date = self.adv_begin_date
        self.end_date = self.adv_end_date
        if self.begin_date > self.end_date:
            GeneralUtils.show_message("\'Begin Date\' is higher than \'End Date\'")
            return

        self.selected_data = []
        selected_report_types = []
        for i in range(len(ALL_REPORTS)):
            if self.report_type_list_model.item(i).checkState() == Qt.Checked:
                selected_report_types.append(ALL_REPORTS[i])
        if len(selected_report_types) == 0:
            GeneralUtils.show_message("No report type selected")
            return

        self.is_yearly_fetch = self.is_yearly_range(self.adv_begin_date, self.adv_end_date)
        custom_dir = self.custom_dir_edit.text()
        if not custom_dir: custom_dir = self.settings.other_directory
        self.save_dir = custom_dir if not self.is_yearly_fetch else self.settings.yearly_directory
        for i in range(self.vendor_list_model.rowCount()):
            if self.vendor_list_model.item(i).checkState() == Qt.Checked:
                request_data = RequestData(self.vendors[i], selected_report_types, self.begin_date, self.end_date,
                                           self.save_dir, self.settings)
                self.selected_data.append(request_data)
        if len(self.selected_data) == 0:
            GeneralUtils.show_message("No vendor selected")
            return

        self.start_progress_dialog("Fetch Reports Progress")
        self.is_last_fetch_advanced = False
        self.retry_data = []

        self.total_processes = len(self.selected_data)
        self.started_processes = 0
        concurrent_vendors = self.settings.concurrent_vendors
        while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors:
            request_data = self.selected_data[self.started_processes]
            self.fetch_vendor_data(request_data)
            self.started_processes += 1


class FetchSpecialReportsController(FetchReportsAbstract):
    def __init__(self, vendors: list, settings: SettingsModel, widget: QWidget,
                 fetch_special_reports_ui: FetchSpecialReportsTab.Ui_fetch_special_reports_tab):
        super().__init__(vendors, settings, widget)

        # region General
        self.selected_report_type = None
        self.selected_options = SpecialReportOptions()
        current_date = QDate.currentDate()
        self.begin_date = QDate(current_date.year(), 1, 1)
        self.end_date = QDate(current_date.year(), max(current_date.month() - 1, 1), 1)
        # endregion

        # region Start Fetch Button
        self.fetch_special_btn = fetch_special_reports_ui.fetch_special_data_button
        self.fetch_special_btn.clicked.connect(self.fetch_special_data)
        # endregion

        # region Vendors
        self.vendor_list_view = fetch_special_reports_ui.vendors_list_view_special
        self.vendor_list_model = QStandardItemModel(self.vendor_list_view)
        self.vendor_list_view.setModel(self.vendor_list_model)
        self.update_vendors_ui()

        self.select_vendors_btn = fetch_special_reports_ui.select_vendors_button_special
        self.select_vendors_btn.clicked.connect(self.select_all_vendors)
        self.deselect_vendors_btn = fetch_special_reports_ui.deselect_vendors_button_special
        self.deselect_vendors_btn.clicked.connect(self.deselect_all_vendors)
        # endregion

        # region Options
        self.options_frame = fetch_special_reports_ui.options_frame
        self.options_layout = self.options_frame.layout()
        help_message = "The 'show' checkboxes will break down your data by displaying extra columns with the " \
                       "selected attributes (usually results in more lines per item)\n" \
                       "Filters will limit your data to just the selected options (implies also showing that " \
                       "attribute).\n" \
                       "You can show the attribute breakdown without selecting filtering to specific values."
        fetch_special_reports_ui.options_help_button.clicked.connect(
            lambda: GeneralUtils.show_message(help_message)
        )
        # endregion

        # region Report Types
        self.pr_radio_button = fetch_special_reports_ui.pr_radio_button
        self.dr_radio_button = fetch_special_reports_ui.dr_radio_button
        self.tr_radio_button = fetch_special_reports_ui.tr_radio_button
        self.ir_radio_button = fetch_special_reports_ui.ir_radio_button

        self.pr_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.PLATFORM))
        self.dr_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.DATABASE))
        self.tr_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.TITLE))
        self.ir_radio_button.clicked.connect(lambda checked: self.on_report_type_selected(MajorReportType.ITEM))

        self.pr_radio_button.setChecked(True)
        self.on_report_type_selected(MajorReportType.PLATFORM)

        # endregion

        # region Date Edits
        self.date_range_help_btn = fetch_special_reports_ui.date_range_help_button
        self.date_range_help_btn.clicked.connect(
            lambda: GeneralUtils.show_message("Many vendors do not support a date range longer than 12 consecutive months."))

        self.begin_date_edit_year = fetch_special_reports_ui.begin_date_edit_special_year
        self.begin_date_edit_year.setDate(self.begin_date)
        self.begin_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "begin_date"))

        self.end_date_edit_year = fetch_special_reports_ui.end_date_edit_special_year
        self.end_date_edit_year.setDate(self.end_date)
        self.end_date_edit_year.dateChanged.connect(lambda date: self.on_date_year_changed(date, "end_date"))

        self.begin_month_combo_box = fetch_special_reports_ui.begin_month_combo_box
        for month in MONTH_NAMES:
            self.begin_month_combo_box.addItem(month)
        self.begin_month_combo_box.currentIndexChanged.connect(
            lambda index: self.on_date_month_changed(index + 1, "begin_date"))
        self.begin_month_combo_box.setCurrentIndex(self.begin_date.month() - 1)

        self.end_month_combo_box = fetch_special_reports_ui.end_month_combo_box
        for month in MONTH_NAMES:
            self.end_month_combo_box.addItem(month)
        self.end_month_combo_box.currentIndexChanged.connect(
            lambda index: self.on_date_month_changed(index + 1, "end_date"))
        self.end_month_combo_box.setCurrentIndex(self.end_date.month() - 1)

        # endregion

        # region Custom Directory
        self.custom_dir_frame = fetch_special_reports_ui.custom_dir_frame
        self.custom_dir_edit = fetch_special_reports_ui.custom_dir_edit
        self.custom_dir_edit.setText(self.settings.other_directory)
        self.custom_dir_button = fetch_special_reports_ui.custom_dir_button
        self.custom_dir_button.clicked.connect(self.on_custom_dir_clicked)
        self.custom_dir_help_btn = fetch_special_reports_ui.custom_dir_help_button
        self.custom_dir_help_btn.clicked.connect(
            lambda: GeneralUtils.show_message("See Other Reports Directory setting in Settings for default."))
        # endregion

    def update_vendors_ui(self):
        """Updates the UI to show vendors that support report fetching (SUSHI)"""
        self.vendor_list_model.clear()
        for vendor in self.vendors:
            item = QStandardItem(vendor.name)
            item.setCheckable(True)
            item.setEditable(False)
            self.vendor_list_model.appendRow(item)

    def on_date_year_changed(self, date: QDate, date_type: str):
        """Handles the signal emitted when a date's year is changed

        :param date: The new date
        :param date_type: The date to be updated
        """
        if date_type == "begin_date":
            self.begin_date = QDate(date.year(),self.begin_date.month(),self.begin_date.day())
        elif date_type == "end_date":
            self.end_date = QDate(date.year(),self.end_date.month(),self.end_date.day())

    def on_date_month_changed(self, month: int, date_type: str):
        """Handles the signal emitted when a date's month is changed

        :param month: The new month
        :param date_type: The date to be updated
        """
        if date_type == "begin_date":
            self.begin_date = QDate(self.begin_date.year(), month, self.begin_date.day())
        elif date_type == "end_date":
            self.end_date = QDate(self.end_date.year(), month, self.end_date.day())

    def on_report_type_selected(self, major_report_type: MajorReportType):
        """Handles the signal emitted when a report type is selected

        :param major_report_type: The major report type (Only master reports are supported)
        """
        if major_report_type == self.selected_report_type: return

        self.selected_report_type = major_report_type
        self.selected_options = SpecialReportOptions()

        # Remove existing options from ui
        for i in reversed(range(self.options_layout.count())):
            widget = self.options_layout.itemAt(i).widget()
            # remove it from the layout list
            self.options_layout.removeWidget(widget)
            # remove it from the gui
            widget.deleteLater()

        # Add new options
        self.options_layout.addWidget(QLabel("Show", self.options_frame), 0, 0)
        self.options_layout.addWidget(QLabel("Filters", self.options_frame), 0, 1)

        special_options = SPECIAL_REPORT_OPTIONS[major_report_type]
        for i in range(len(special_options)):
            option_name = special_options[i][1]
            checkbox = QCheckBox(option_name, self.options_frame)
            checkbox.toggled.connect(
                lambda is_checked, option=option_name: self.on_special_option_toggled(option, is_checked))
            self.options_layout.addWidget(checkbox, i + 1, 0)

            option_type: SpecialOptionType = special_options[i][0]
            if option_type == SpecialOptionType.AP or option_type == SpecialOptionType.POS\
                    or option_type == SpecialOptionType.ADP:
                line_edit = QLineEdit(DEFAULT_SPECIAL_OPTION_VALUE, self.options_frame)
                line_edit.setReadOnly(True)
                button = QPushButton("Choose", self.options_frame)
                if option_type == SpecialOptionType.ADP:
                    button.clicked.connect(
                        lambda c, so=special_options[i], edit=line_edit:
                        self.on_special_date_parameter_option_button_clicked(so, edit))
                else:
                    button.clicked.connect(
                        lambda c, so=special_options[i], edit=line_edit:
                        self.on_special_parameter_option_button_clicked(so, edit))

                self.options_layout.addWidget(line_edit, i + 1, 1)
                self.options_layout.addWidget(button, i + 1, 2)

    def on_special_option_toggled(self, option: str, is_checked: bool):
        """Handles the signal emitted when a special option is checked or un-checked

        :param option: The special option
        :param is_checked: Checked or un-checked
        """
        option = option.lower()
        __, option_type, option_name, curr_options = self.selected_options.__getattribute__(option)
        self.selected_options.__setattr__(option, (is_checked, option_type, option_name, curr_options))

    def on_special_parameter_option_button_clicked(self, special_option, line_edit):
        """Handles the signal emitted when a special option with parameters clicked

        :param special_option: The special option
        :param line_edit: The line edit to show the selected parameters for this option
        """
        option_type, option_name, option_list = special_option
        is_selected, option_type, option_name, curr_options = self.selected_options.__getattribute__(option_name.lower())

        dialog = QDialog(self.options_frame, flags=Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint)
        dialog.setWindowTitle(option_name + " options")
        layout = QVBoxLayout(dialog)

        list_view = QListView(dialog)
        list_view.setAlternatingRowColors(True)
        model = QStandardItemModel(list_view)

        for option in option_list:
            item = QStandardItem(option)
            item.setCheckable(True)
            item.setEditable(False)
            if option in curr_options:
                item.setCheckState(Qt.Checked)
            model.appendRow(item)

        list_view.setModel(model)

        layout.addWidget(list_view)

        def on_ok_button_clicked():
            checked_list = []
            for i in range(model.rowCount()):
                if model.item(i).checkState() == Qt.Checked:
                    checked_list.append(model.item(i).text())

            if len(checked_list) == 0:
                checked_list = [DEFAULT_SPECIAL_OPTION_VALUE]

            line_edit.setText("|".join(checked_list))
            self.selected_options.__setattr__(option_name.lower(), (is_selected, option_type, option_name, checked_list))
            dialog.close()

        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel, dialog)
        button_box.accepted.connect(on_ok_button_clicked)
        button_box.rejected.connect(lambda: dialog.close())
        button_box.setCenterButtons(True)
        layout.addWidget(button_box)

        dialog.exec_()

    def on_special_date_parameter_option_button_clicked(self, special_option, line_edit):
        """Handles the signal emitted when a date special option is clicked

        :param special_option: The special option
        :param line_edit: The line edit to show the selected date parameters for this option
        """
        option_type, option_name = special_option
        is_selected, option_type, option_name, selected_options = self.selected_options.__getattribute__(option_name.lower())

        dialog = QDialog(self.options_frame, flags=Qt.Window | Qt.WindowTitleHint | Qt.CustomizeWindowHint)
        dialog.setWindowTitle(option_name + " options")
        layout = QVBoxLayout(dialog)

        radio_button_group = QButtonGroup(dialog)

        default_frame = QFrame(dialog)
        default_frame.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
        default_layout = QHBoxLayout(default_frame)
        default_layout.setContentsMargins(0, 0, 0, 0)

        default_radio_btn = QRadioButton(default_frame)
        default_radio_btn.setChecked(True)
        radio_button_group.addButton(default_radio_btn)
        default_label = QLabel(DEFAULT_SPECIAL_OPTION_VALUE, dialog)

        default_layout.addWidget(default_radio_btn)
        default_layout.addWidget(default_label)

        single_date_frame = QFrame(dialog)
        single_date_frame.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
        single_date_layout = QHBoxLayout(single_date_frame)
        single_date_layout.setContentsMargins(0, 0, 0, 0)

        single_date_radio_btn = QRadioButton(single_date_frame)
        radio_button_group.addButton(single_date_radio_btn)
        single_date_edit = QDateEdit(QDate.currentDate(), single_date_frame)
        single_date_edit.setDisplayFormat("yyyy")

        single_date_layout.addWidget(single_date_radio_btn)
        single_date_layout.addWidget(single_date_edit)

        range_date_frame = QFrame(dialog)
        range_date_frame.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
        range_date_layout = QHBoxLayout(range_date_frame)
        range_date_layout.setContentsMargins(0, 0, 0, 0)

        range_date_radio_btn = QRadioButton(range_date_frame)
        radio_button_group.addButton(range_date_radio_btn)
        begin_date_edit = QDateEdit(QDate.currentDate(), range_date_frame)
        end_date_edit = QDateEdit(QDate.currentDate(), range_date_frame)
        begin_date_edit.setDisplayFormat("yyyy")
        end_date_edit.setDisplayFormat("yyyy")
        to_label = QLabel(" to ", range_date_frame)
        to_label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)

        range_date_layout.addWidget(range_date_radio_btn)
        range_date_layout.addWidget(begin_date_edit)
        range_date_layout.addWidget(to_label)
        range_date_layout.addWidget(end_date_edit)

        def on_ok_button_clicked():
            new_selection = [DEFAULT_SPECIAL_OPTION_VALUE]
            checked_button = radio_button_group.checkedButton()
            if checked_button == single_date_radio_btn:
                new_selection = [single_date_edit.date().toString("yyyy")]
            elif checked_button == range_date_radio_btn:
                new_selection = [begin_date_edit.date().toString("yyyy") + "-" + end_date_edit.date().toString("yyyy")]

            line_edit.setText(new_selection[0])
            self.selected_options.__setattr__(option_name.lower(), (is_selected, option_type, option_name, new_selection))
            dialog.close()

        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel, dialog)
        button_box.accepted.connect(on_ok_button_clicked)
        button_box.rejected.connect(lambda: dialog.close())
        button_box.setCenterButtons(True)

        layout.addWidget(default_frame)
        layout.addWidget(single_date_frame)
        layout.addWidget(range_date_frame)
        layout.addWidget(button_box)

        dialog.exec_()

    def on_custom_dir_clicked(self):
        """Handles the signal emitted when the choose custom directory button is clicked"""
        dir_path = GeneralUtils.choose_directory()
        if dir_path: self.custom_dir_edit.setText(dir_path)

    def select_all_vendors(self):
        """Checks all vendors in the vendors list view"""
        for i in range(self.vendor_list_model.rowCount()):
            self.vendor_list_model.item(i).setCheckState(Qt.Checked)

    def deselect_all_vendors(self):
        """Un-checks all vendors in the vendors list view"""
        for i in range(self.vendor_list_model.rowCount()):
            self.vendor_list_model.item(i).setCheckState(Qt.Unchecked)

    def fetch_special_data(self):
        """Fetches reports based on the selected options in the UI"""
        if self.total_processes > 0 or self.is_updating_database:
            GeneralUtils.show_message(f"Waiting for pending processes to complete...")
            if self.settings.show_debug_messages: print(f"Waiting for pending processes to complete...")
            return

        if len(self.vendors) == 0:
            GeneralUtils.show_message("Vendor list is empty")
            return

        if self.begin_date > self.end_date:
            GeneralUtils.show_message("\'Begin Date\' is higher than \'End Date\'")
            return

        self.selected_data = []
        selected_report_types = [self.selected_report_type.value]

        self.is_yearly_fetch = False
        custom_dir = self.custom_dir_edit.text()
        self.save_dir = custom_dir if custom_dir else self.settings.other_directory
        for i in range(self.vendor_list_model.rowCount()):
            if self.vendor_list_model.item(i).checkState() == Qt.Checked:
                request_data = RequestData(self.vendors[i], selected_report_types, self.begin_date, self.end_date,
                                           self.save_dir, self.settings, self.selected_options)
                self.selected_data.append(request_data)
        if len(self.selected_data) == 0:
            GeneralUtils.show_message("No vendor selected")
            return

        self.start_progress_dialog("Fetch Special Reports Progress")
        self.retry_data = []

        self.total_processes = len(self.selected_data)
        self.started_processes = 0
        concurrent_vendors = self.settings.concurrent_vendors
        while self.started_processes < len(self.selected_data) and self.started_processes < concurrent_vendors:
            request_data = self.selected_data[self.started_processes]
            self.fetch_vendor_data(request_data)
            self.started_processes += 1


class VendorWorker(QObject):
    """This does all the work for a vendor when fetching reports

    :param worker_id: The ID to identify this worker (vendor_name)
    :param request_data: The request data for this request
    """
    worker_finished_signal = pyqtSignal(str)

    def __init__(self, worker_id: str, request_data: RequestData):
        super().__init__()
        self.worker_id = worker_id
        self.request_data = request_data
        self.vendor = request_data.vendor
        self.target_report_types = request_data.target_report_types
        self.show_debug = request_data.settings.show_debug_messages
        self.concurrent_reports = request_data.settings.concurrent_reports
        self.request_interval = request_data.settings.request_interval
        self.request_timeout = request_data.settings.request_timeout
        self.user_agent = request_data.settings.user_agent
        self.reports_to_process = []
        self.started_processes = 0
        self.completed_processes = 0
        self.total_processes = 0

        self.process_result = ProcessResult(self.vendor)
        self.report_process_results = []
        self.report_workers = {}  # <k = worker_id, v = (ReportWorker, Thread)>
        self.is_cancelling = False

    def work(self):
        """Processes the vendor's requests

        Requests the vendor's supported reports before requesting only the supported reports
        """
        if self.show_debug: print(f"{self.vendor.name}: Fetching supported reports")
        request_query = {}
        if self.vendor.customer_id.strip(): request_query["customer_id"] = self.vendor.customer_id
        if self.vendor.requestor_id.strip(): request_query["requestor_id"] = self.vendor.requestor_id
        if self.vendor.api_key.strip(): request_query["api_key"] = self.vendor.api_key
        if self.vendor.platform.strip(): request_query["platform"] = self.vendor.platform

        request_url = self.vendor.base_url

        try:
            # Some vendors only work if they think a web browser is making the request...
            response = requests.get(request_url, request_query, headers={'User-Agent': self.user_agent},
                                    timeout=self.request_timeout)
            if self.show_debug: print(response.url)
            if response.status_code == 200:
                self.process_response(response)
            else:
                self.process_result.completion_status = CompletionStatus.FAILED
                self.process_result.message = f"Unexpected HTTP status code received: {response.status_code}"
        except requests.exceptions.Timeout as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = f"Request timed out after {self.request_timeout} second(s)"
            if self.show_debug: print(f"{self.vendor.name}: Request timed out")
        except requests.exceptions.RequestException as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = f"Request Exception: {e}"
            if self.show_debug: print(f"{self.vendor.name}: Request Exception: {e}")

        if len(self.report_workers) == 0: self.notify_worker_finished()

    def process_response(self, response: requests.Response):
        """Processes the response from a REST request

        Requests the target reports that are supported by the vendor
        """
        if self.is_cancelling:
            self.process_result.message = "Target reports not processed"
            self.process_result.completion_status = CompletionStatus.CANCELLED
            return

        try:
            json_response = response.json()
            exceptions = self.check_for_exception(json_response)
            if len(exceptions) > 0:
                self.process_result.message = exception_models_to_message(exceptions)
                self.process_result.completion_status = CompletionStatus.FAILED
                return

            json_dicts = []
            if type(json_response) is dict:  # This should never be a dict by the standard, but some vendors......
                json_dicts = json_response["Report_Items"] if "Report_Items" in json_response else []  # Report_Items???
                # Some other vendor might implement it in a different way..........................
            elif type(json_response) is list:
                json_dicts = json_response

            if len(json_dicts) == 0:
                raise Exception("JSON is empty")

            supported_report_types = []
            self.reports_to_process = []
            for json_dict in json_dicts:
                supported_report = SupportedReportModel.from_json(json_dict)
                supported_report_types.append(supported_report.report_id)

                if supported_report.report_id in self.target_report_types:
                    self.reports_to_process.append(supported_report.report_id)

            unsupported_report_types = list(set(self.target_report_types) - set(supported_report_types))

            self.process_result.message = "Supported by vendor: "
            self.process_result.message += ", ".join(self.reports_to_process)
            self.process_result.message += "\nUnsupported: "
            self.process_result.message += ", ".join(unsupported_report_types)

            if len(self.reports_to_process) == 0: return

            self.total_processes = len(self.reports_to_process)
            self.started_processes = 0
            while self.started_processes < self.total_processes and self.started_processes < self.concurrent_reports:
                QThread.currentThread().sleep(self.request_interval)  # Avoid spamming vendor's server
                if self.is_cancelling:
                    self.process_result.completion_status = CompletionStatus.CANCELLED
                    return

                self.fetch_report(self.reports_to_process[self.started_processes])
                self.started_processes += 1

        except json.JSONDecodeError as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = f"JSON Exception: {e}"
            if self.show_debug: print(f"{self.vendor.name}: JSON Exception: {e.msg}")
        except Exception as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = str(e)
            if self.show_debug: print(f"{self.vendor.name}: Exception: {e}")

    def fetch_report(self, report_type: str):
        """Initiates the process to fetch a report

        :param report_type: The target report type
        """
        worker_id = report_type
        if worker_id in self.report_workers: return  # Avoid fetching a report twice, app will crash!!

        report_worker = ReportWorker(worker_id, report_type, self.request_data)
        report_worker.worker_finished_signal.connect(self.on_report_worker_finished)
        report_thread = QThread()
        self.report_workers[worker_id] = report_worker, report_thread
        report_worker.moveToThread(report_thread)
        report_thread.started.connect(report_worker.work)
        report_thread.finished.connect(report_thread.deleteLater)

        report_thread.start()

    def check_for_exception(self, json_response) -> list:
        """Checks a JSON response for exception models

        :param json_response: The JSON response to be processed
        """
        exceptions = []

        if type(json_response) is dict:
            if "Exception" in json_response:
                exceptions.append(ExceptionModel.from_json(json_response["Exception"]))
                # raise Exception(f"Code: {exception.code}, Message: {exception.message}")

            code = int(json_response["Code"]) if "Code" in json_response else ""
            message = json_response["Message"] if "Message" in json_response else ""
            data = json_response["Data"] if "Data" in json_response else ""
            severity = json_response["Severity"] if "Severity" in json_response else ""
            if code:
                exceptions.append(ExceptionModel(code, message, severity, data))

        elif type(json_response) is list:
            for json_dict in json_response:
                exception = ExceptionModel.from_json(json_dict)
                if exception.code:
                    exceptions.append(exception)

        return exceptions

    def notify_worker_finished(self):
        """Notifies any listeners that this worker has finished"""
        self.worker_finished_signal.emit(self.vendor.name)

    def on_report_worker_finished(self, worker_id: str):
        """Handles the signal emmited when a report worker has finished

        :param worker_id: The report worker's worker id
        """
        self.completed_processes += 1

        thread: QThread
        worker: ReportWorker
        worker, thread = self.report_workers[worker_id]

        self.report_process_results.append(worker.process_result)
        worker.deleteLater()
        thread.quit()
        thread.wait()
        self.report_workers.pop(worker_id, None)

        if self.started_processes < self.total_processes and not self.is_cancelling:
            QThread.currentThread().sleep(self.request_interval)  # Avoid spamming vendor's server
            self.fetch_report(self.reports_to_process[self.started_processes])
            self.started_processes += 1

        if len(self.report_workers) == 0: self.notify_worker_finished()

    def set_cancelling(self):
        """Sets the worker to a cancelling state"""
        self.is_cancelling = True


class ReportWorker(QObject):
    """This does all the work for a report

    :param worker_id: The ID to identify this worker (vendor_name-report_type)
    :param report_type: The report type to be processed
    :param request_data: The request data for this request
    """
    worker_finished_signal = pyqtSignal(str)

    def __init__(self, worker_id: str, report_type: str, request_data: RequestData):
        super().__init__()
        self.worker_id = worker_id
        self.report_type = report_type
        self.vendor = request_data.vendor
        self.begin_date = request_data.begin_date
        self.end_date = request_data.end_date
        self.show_debug = request_data.settings.show_debug_messages
        self.request_timeout = request_data.settings.request_timeout
        self.user_agent = request_data.settings.user_agent
        self.save_dir = request_data.save_location
        self.special_options = request_data.special_options

        self.is_yearly = self.save_dir == request_data.settings.yearly_directory
        self.is_special = self.special_options is not None
        self.is_master = self.report_type in MASTER_REPORTS

        self.process_result = ProcessResult(self.vendor, self.report_type)
        self.retried_request = False

    def work(self):
        """Processes the report request"""
        if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Fetching Report")

        self.make_request()

        if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Done")
        self.notify_worker_finished()

    def make_request(self):
        """Sends the request fetch a report"""
        request_query = {}
        if self.vendor.customer_id.strip(): request_query["customer_id"] = self.vendor.customer_id
        if self.vendor.requestor_id.strip(): request_query["requestor_id"] = self.vendor.requestor_id
        if self.vendor.api_key.strip(): request_query["api_key"] = self.vendor.api_key
        if self.vendor.platform.strip(): request_query["platform"] = self.vendor.platform
        request_query["begin_date"] = self.begin_date.toString("yyyy-MM")
        request_query["end_date"] = self.end_date.toString("yyyy-MM")

        attributes_to_show = ""
        if self.is_special:
            attr_count = 0
            special_options_dict = self.special_options.__dict__
            for option in special_options_dict:
                value = special_options_dict[option]
                is_selected, option_type, option_name, option_parameters = value

                if is_selected:
                    if option_type == SpecialOptionType.AP or option_type == SpecialOptionType.ADP\
                            or option_type == SpecialOptionType.POS:
                        if option_parameters[0] != DEFAULT_SPECIAL_OPTION_VALUE:
                            request_query[option] = "|".join(option_parameters)

                    elif option_type == SpecialOptionType.POB:
                        request_query[option] = "True"

                    if option_type == SpecialOptionType.AP or option_type == SpecialOptionType.ADP\
                            or option_type == SpecialOptionType.AO:
                        if attr_count == 0:
                            attributes_to_show += option_name
                            attr_count += 1
                        elif attr_count > 0:
                            attributes_to_show += f"|{option_name}"
                            attr_count += 1
        elif self.is_yearly and self.is_master:
            major_report_type = GeneralUtils.get_major_report_type(self.report_type)
            if major_report_type == MajorReportType.PLATFORM:
                attributes_to_show = "|".join(PLATFORM_REPORTS_ATTRIBUTES)
            elif major_report_type == MajorReportType.DATABASE:
                attributes_to_show = "|".join(DATABASE_REPORTS_ATTRIBUTES)
            elif major_report_type == MajorReportType.TITLE:
                attributes_to_show = "|".join(TITLE_REPORTS_ATTRIBUTES)
            elif major_report_type == MajorReportType.ITEM:
                attributes_to_show = "|".join(ITEM_REPORTS_ATTRIBUTES)
                request_query["include_parent_details"] = True
                request_query["include_component_details"] = True

        if attributes_to_show: request_query["attributes_to_show"] = attributes_to_show

        request_url = f"{self.vendor.base_url}/{self.report_type.lower()}"

        try:
            # Some vendors only work if they think a web browser is making the request...
            response = requests.get(request_url, request_query, headers={'User-Agent': self.user_agent},
                                    timeout=self.request_timeout)
            if self.show_debug: print(response.url)
            if response.status_code == 200:
                self.process_response(response)
            else:
                self.process_result.completion_status = CompletionStatus.FAILED
                self.process_result.message = f"Unexpected HTTP status code received: {response.status_code}"
        except requests.exceptions.Timeout as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = f"Request timed out after {self.request_timeout} second(s)"
            if self.show_debug: print(f"{self.vendor.name}: Request timed out")
        except requests.exceptions.RequestException as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = f"Request Exception: {e}"
            if self.show_debug: print(
                f"{self.vendor.name}-{self.report_type}: Request Exception: {e}")

    def process_response(self, response: requests.Response):
        """Processes the response from a request

        Converts the receoved JSON to a SUSHI Report Model

        :param response: The received response
        """
        try:
            json_string = response.text
            if self.is_yearly: self.save_json_file(json_string)

            json_dict = json.loads(json_string)
            report_model = ReportModel.from_json(json_dict)
            self.process_report_model(report_model)
            if len(report_model.report_items) > 0 or len(report_model.exceptions) > 0:
                self.process_result.message = exception_models_to_message(report_model.exceptions)
            else:
                self.process_result.message = "Report items not received. No exception received."
        except json.JSONDecodeError as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            if e.msg == "Expecting value":
                self.process_result.message = f"Vendor did not return any data"
            else:
                self.process_result.message = f"JSON Exception: {e.msg}"
            if self.show_debug: print(
                f"{self.vendor.name}-{self.report_type}: JSON Exception: {e.msg}")
        except RetryLaterException as e:
            if not self.retried_request:
                if self.show_debug:
                    print(f"{self.vendor.name}-{self.report_type}: Retry Later Exception: {e}")
                    print(f"{self.vendor.name}-{self.report_type}: Retrying in {RETRY_WAIT_TIME} seconds...")
                QThread.currentThread().sleep(RETRY_WAIT_TIME)  # Wait some time before retrying request
                self.retried_request = True
                self.make_request()
            else:
                self.process_result.message = "Retry later exception received"
                message = exception_models_to_message(e.exceptions)
                if message: self.process_result.message += "\n\n" + message
                self.process_result.completion_status = CompletionStatus.FAILED
                self.process_result.retry = True
                if self.show_debug: print(
                    f"{self.vendor.name}-{self.report_type}: Retry Later Exception: {e}")
        except ReportHeaderMissingException as e:
            self.process_result.message = "Report_Header not received, no file was created"
            message = exception_models_to_message(e.exceptions)
            if message: self.process_result.message += "\n\n" + message
            self.process_result.completion_status = CompletionStatus.FAILED
            if self.show_debug: print(
                f"{self.vendor.name}-{self.report_type}: Report Header Missing Exception: {e}")
        except UnacceptableCodeException as e:
            self.process_result.message = "Unsupported exception code received"
            message = exception_models_to_message(e.exceptions)
            if message: self.process_result.message += "\n\n" + message
            self.process_result.completion_status = CompletionStatus.FAILED
            if self.show_debug: print(
                f"{self.vendor.name}-{self.report_type}: Unsupported Code Exception: {e}")
        except Exception as e:
            self.process_result.completion_status = CompletionStatus.FAILED
            self.process_result.message = str(e)
            if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Exception: {e}")

    def process_report_model(self, report_model: ReportModel):
        """Processes the report model into a TSV report

        :param report_model: The report model
        """
        report_type = report_model.report_header.report_id
        major_report_type = report_model.report_header.major_report_type
        report_items = report_model.report_items
        report_rows = []
        file_dir = f"{self.save_dir}{self.begin_date.toString('yyyy')}/{self.vendor.name}/"
        file_name = f"{self.begin_date.toString('yyyy')}_{self.vendor.name}_{report_type}.tsv"
        file_path = f"{file_dir}{file_name}"

        if self.show_debug: print(f"{self.vendor.name}-{self.report_type}: Processing report")

        for report_item in report_items:
            metric_row_dict = {}  # <k = metric_type, v = ReportRow> Some metric_types have a list of components
            # Some Item report metric_types have a list of components
            components = []  # list({component_values_as_dict})

            performance: PerformanceModel
            for performance in report_item.performances:
                begin_month = QDate.fromString(performance.period.begin_date, "yyyy-MM-dd").toString("MMM-yyyy")

                instance: InstanceModel
                for instance in performance.instances:
                    metric_type = instance.metric_type
                    if metric_type not in metric_row_dict:
                        metric_row = ReportRow(self.begin_date, self.end_date)
                        metric_row.metric_type = metric_type

                        metric_row_dict[metric_type] = metric_row
                    else:
                        metric_row = metric_row_dict[metric_type]

                    if major_report_type == MajorReportType.PLATFORM:
                        report_item: PlatformReportItemModel
                        if report_item.platform: metric_row.platform = report_item.platform
                        if report_item.data_type: metric_row.data_type = report_item.data_type
                        if report_item.access_method: metric_row.access_method = report_item.access_method

                    elif major_report_type == MajorReportType.DATABASE:
                        report_item: DatabaseReportItemModel
                        if report_item.database: metric_row.database = report_item.database
                        if report_item.publisher: metric_row.publisher = report_item.publisher
                        if report_item.platform: metric_row.platform = report_item.platform
                        if report_item.data_type: metric_row.data_type = report_item.data_type
                        if report_item.access_method: metric_row.access_method = report_item.access_method

                        pub_id_str = ""
                        for pub_id in report_item.publisher_ids:
                            if pub_id.item_type == "Proprietary":
                                continue
                            pub_id_str += f"{pub_id.item_type}:{pub_id.value}; "
                        if pub_id_str: metric_row.publisher_id = pub_id_str.rstrip("; ")

                        for item_id in report_item.item_ids:
                            if item_id.item_type == "Proprietary" or item_id.item_type == "Proprietary_ID":
                                metric_row.proprietary_id = item_id.value

                    elif major_report_type == MajorReportType.TITLE:
                        report_item: TitleReportItemModel
                        if report_item.title: metric_row.title = report_item.title
                        if report_item.publisher: metric_row.publisher = report_item.publisher
                        if report_item.platform: metric_row.platform = report_item.platform
                        if report_item.data_type: metric_row.data_type = report_item.data_type
                        if report_item.section_type: metric_row.section_type = report_item.section_type
                        if report_item.yop: metric_row.yop = report_item.yop
                        if report_item.access_type: metric_row.access_type = report_item.access_type
                        if report_item.access_method: metric_row.access_method = report_item.access_method

                        pub_id_str = ""
                        for pub_id in report_item.publisher_ids:
                            if pub_id.item_type == "Proprietary":
                                continue
                            pub_id_str += f"{pub_id.item_type}:{pub_id.value}; "
                        if pub_id_str: metric_row.publisher_id = pub_id_str.rstrip("; ")

                        item_id: TypeValueModel
                        for item_id in report_item.item_ids:
                            item_type = item_id.item_type

                            if item_type == "DOI":
                                metric_row.doi = item_id.value
                            elif item_type == "Proprietary" or item_type == "Proprietary_ID":
                                metric_row.proprietary_id = item_id.value
                            elif item_type == "ISBN":
                                metric_row.isbn = item_id.value
                            elif item_type == "Print_ISSN":
                                metric_row.print_issn = item_id.value
                            elif item_type == "Online_ISSN":
                                metric_row.online_issn = item_id.value
                            elif item_type == "Linking_ISSN":
                                metric_row.linking_issn = item_id.value
                            elif item_type == "URI":
                                metric_row.uri = item_id.value

                    elif major_report_type == MajorReportType.ITEM:
                        report_item: ItemReportItemModel
                        if report_item.item: metric_row.item = report_item.item
                        if report_item.publisher: metric_row.publisher = report_item.publisher
                        if report_item.platform: metric_row.platform = report_item.platform
                        if report_item.data_type: metric_row.data_type = report_item.data_type
                        if report_item.yop: metric_row.yop = report_item.yop
                        if report_item.access_type: metric_row.access_type = report_item.access_type
                        if report_item.access_method: metric_row.access_method = report_item.access_method

                        # Publisher ID
                        pub_id_str = ""
                        for pub_id in report_item.publisher_ids:
                            if pub_id.item_type == "Proprietary":
                                continue
                            pub_id_str += f"{pub_id.item_type}:{pub_id.value}; "
                        if pub_id_str: metric_row.publisher_id = pub_id_str.rstrip("; ")

                        # Authors
                        authors_str = ""
                        item_contributor: ItemContributorModel
                        for item_contributor in report_item.item_contributors:
                            if item_contributor.item_type == "Author":
                                authors_str += f"{item_contributor.name}"
                                if item_contributor.identifier:
                                    authors_str += f" ({item_contributor.identifier})"
                                authors_str += "; "
                        if authors_str: metric_row.authors = authors_str.rstrip("; ")

                        # Publication date
                        item_date: TypeValueModel
                        for item_date in report_item.item_dates:
                            if item_date.item_type == "Publication_Date":
                                metric_row.publication_date = item_date.value

                        # Article version
                        item_attribute: TypeValueModel
                        for item_attribute in report_item.item_attributes:
                            if item_attribute.item_type == "Article_Version":
                                metric_row.article_version = item_attribute.value

                        # Base IDs
                        item_id: TypeValueModel
                        for item_id in report_item.item_ids:
                            item_type = item_id.item_type

                            if item_type == "DOI":
                                metric_row.doi = item_id.value
                            elif item_type == "Proprietary" or item_type == "Proprietary_ID":
                                metric_row.proprietary_id = item_id.value
                            elif item_type == "ISBN":
                                metric_row.isbn = item_id.value
                            elif item_type == "Print_ISSN":
                                metric_row.print_issn = item_id.value
                            elif item_type == "Online_ISSN":
                                metric_row.online_issn = item_id.value
                            elif item_type == "Linking_ISSN":
                                metric_row.linking_issn = item_id.value
                            elif item_type == "URI":
                                metric_row.uri = item_id.value

                        # Parent
                        if report_item.item_parent is not None:
                            item_parent: ItemParentModel
                            item_parent = report_item.item_parent
                            if item_parent.item_name: metric_row.parent_title = item_parent.item_name
                            if item_parent.data_type: metric_row.parent_data_type = item_parent.data_type

                            # Authors
                            authors_str = ""
                            item_contributor: ItemContributorModel
                            for item_contributor in report_item.item_contributors:
                                if item_contributor.item_type == "Author":
                                    authors_str += f"{item_contributor.name}"
                                    if item_contributor.identifier:
                                        authors_str += f" ({item_contributor.identifier})"
                                    authors_str += "; "
                            authors_str.rstrip("; ")
                            if authors_str: metric_row.authors = authors_str

                            # Publication date
                            item_date: TypeValueModel
                            for item_date in item_parent.item_dates:
                                if item_date.item_type == "Publication_Date" or item_date.item_type == "Pub_Date":
                                    metric_row.parent_publication_date = item_date.value

                            # Article version
                            item_attribute: TypeValueModel
                            for item_attribute in item_parent.item_attributes:
                                if item_attribute.item_type == "Article_Version":
                                    metric_row.parent_article_version = item_attribute.value

                            # Parent IDs
                            item_id: TypeValueModel
                            for item_id in item_parent.item_ids:
                                item_type = item_id.item_type

                                if item_type == "DOI":
                                    metric_row.parent_doi = item_id.value
                                elif item_type == "Proprietary" or item_type == "Proprietary_ID":
                                    metric_row.parent_proprietary_id = item_id.value
                                elif item_type == "ISBN":
                                    metric_row.parent_isbn = item_id.value
                                elif item_type == "Print_ISSN":
                                    metric_row.parent_print_issn = item_id.value
                                elif item_type == "Online_ISSN":
                                    metric_row.parent_online_issn = item_id.value
                                elif item_type == "URI":
                                    metric_row.parent_uri = item_id.value

                    else:
                        if self.show_debug: print(
                            f"{self.vendor.name}-{self.report_type}: Unexpected report type")

                    month_counts = metric_row.month_counts
                    month_counts[begin_month] += instance.count

                    metric_row.total_count += instance.count

            if major_report_type == MajorReportType.ITEM:
                # Item Components
                item_component: ItemComponentModel
                for item_component in report_item.item_components:
                    component_dict = {
                        "component_title": "",
                        "component_authors": "",
                        "component_publication_date": "",
                        "component_data_type": "",
                        "component_doi": "",
                        "component_proprietary_id": "",
                        "component_isbn": "",
                        "component_print_issn": "",
                        "component_online_issn": "",
                        "component_uri": ""
                    }

                    if item_component.item_name: component_dict["component_title"] = item_component.item_name
                    if item_component.data_type: component_dict["component_data_type"] = item_component.data_type

                    # Authors
                    authors_str = ""
                    item_contributor: ItemContributorModel
                    for item_contributor in item_component.item_contributors:
                        if item_contributor.item_type == "Author":
                            authors_str += f"{item_contributor.name}"
                            if item_contributor.identifier:
                                authors_str += f" ({item_contributor.identifier})"
                            authors_str += "; "
                    authors_str.rstrip("; ")
                    if authors_str: component_dict["component_authors"] = authors_str

                    # Publication date
                    item_date: TypeValueModel
                    for item_date in item_component.item_dates:
                        if item_date.item_type == "Publication_Date" or item_date.item_type == "Pub_Date":
                            component_dict["component_publication_date"] = item_date.value

                    # Component IDs
                    item_id: TypeValueModel
                    for item_id in item_component.item_ids:
                        item_type = item_id.item_type

                        if item_type == "DOI":
                            component_dict["component_doi"] = item_id.value
                        elif item_type == "Proprietary" or item_type == "Proprietary_ID":
                            component_dict["component_proprietary_id"] = item_id.value
                        elif item_type == "ISBN":
                            component_dict["component_isbn"] = item_id.value
                        elif item_type == "Print_ISSN":
                            component_dict["component_print_issn"] = item_id.value
                        elif item_type == "Online_ISSN":
                            component_dict["component_online_issn"] = item_id.value
                        elif item_type == "URI":
                            component_dict["component_uri"] = item_id.value

                    components.append(component_dict)

            for metric_type in metric_row_dict:
                metric_row = metric_row_dict[metric_type]

                if major_report_type == MajorReportType.ITEM:
                    if len(components) > 0 and \
                            (metric_type == "Total_Item_Investigations" or metric_type == "Total_Item_Requests"):
                        for component in components:
                            row = copy.copy(metric_row)
                            row.component_title = component["component_title"]
                            row.component_authors = component["component_authors"]
                            row.component_publication_date = component["component_publication_date"]
                            row.component_data_type = component["component_data_type"]
                            row.component_doi = component["component_doi"]
                            row.component_proprietary_id = component["component_proprietary_id"]
                            row.component_isbn = component["component_isbn"]
                            row.component_print_issn = component["component_print_issn"]
                            row.component_online_issn = component["component_online_issn"]
                            row.component_uri = component["component_uri"]
                            report_rows.append(row)
                    else:
                        report_rows.append(metric_row)
                else:
                    report_rows.append(metric_row)

        report_rows = self.sort_rows(report_rows, major_report_type)
        self.save_tsv_files(report_model.report_header, report_rows)

    @staticmethod
    def sort_rows(report_rows: list, major_report_type: MajorReportType) -> list:
        """Sorts the rows of the report

        :param report_rows: The report's rows
        :param major_report_type: The major report type of this report type
        """
        if major_report_type == MajorReportType.PLATFORM:
            return sorted(report_rows, key=lambda row: row.platform.lower())
        elif major_report_type == MajorReportType.DATABASE:
            return sorted(report_rows, key=lambda row: row.database.lower())
        elif major_report_type == MajorReportType.TITLE:
            return sorted(report_rows, key=lambda row: (row.title.lower(), row.yop))
        elif major_report_type == MajorReportType.ITEM:
            return sorted(report_rows, key=lambda row: row.item.lower())

    def save_tsv_files(self, report_header, report_rows: list):
        """Saves the TSV file in the target directories

        :param report_header: The SUSHI report header model
        :param report_rows: The report's rows
        """
        report_type = report_header.report_id
        major_report_type = report_header.major_report_type

        if self.is_yearly:
            file_dir = GeneralUtils.get_yearly_file_dir(self.save_dir, self.vendor.name, self.begin_date)
            file_name = GeneralUtils.get_yearly_file_name(self.vendor.name, self.report_type, self.begin_date)
        elif self.is_special:
            file_dir = GeneralUtils.get_special_file_dir(self.save_dir, self.vendor.name)
            file_name = GeneralUtils.get_special_file_name(self.vendor.name, self.report_type, self.begin_date,
                                                           self.end_date)
        else:
            file_dir = GeneralUtils.get_other_file_dir(self.save_dir, self.vendor.name)
            file_name = GeneralUtils.get_other_file_name(self.vendor.name, self.report_type, self.begin_date,
                                                         self.end_date)

        # Save user tsv file
        if not path.isdir(file_dir):
            makedirs(file_dir)

        file_path = f"{file_dir}{file_name}"
        file = open(file_path, 'w', encoding="utf-8", newline='')
        if self.is_special:
            self.add_report_header_to_file(report_header, file, True)
        else:
            self.add_report_header_to_file(report_header, file, False)

        if not self.add_report_rows_to_file(report_type, report_rows, self.begin_date, self.end_date, file, False,
                                            self.is_special, self.special_options):
            self.process_result.completion_status = CompletionStatus.WARNING

        file.close()
        self.process_result.file_name = file_name
        self.process_result.file_dir = file_dir
        self.process_result.file_path = file_path
        self.process_result.year = self.begin_date.toString('yyyy')

        # Save protected tsv file
        if self.is_yearly:
            protected_file_dir = GeneralUtils.get_yearly_file_dir(PROTECTED_DATABASE_FILE_DIR, self.vendor.name,
                                                                  self.begin_date)
            if not path.isdir(protected_file_dir):
                makedirs(protected_file_dir)
                if platform.system() == "Windows":
                    ctypes.windll.kernel32.SetFileAttributesW(PROTECTED_DATABASE_FILE_DIR, 2)  # Hide folder

            protected_file_path = f"{protected_file_dir}{file_name}"
            protected_file = open(protected_file_path, 'w', encoding="utf-8", newline='')
            self.add_report_header_to_file(report_header, protected_file, True)
            self.add_report_rows_to_file(report_type, report_rows, self.begin_date, self.end_date, protected_file, True)

            protected_file.close()
            self.process_result.protected_file_path = protected_file_path

    @staticmethod
    def add_report_header_to_file(report_header: ReportHeaderModel, file, include_attributes: bool):
        """Adds the report header to a TSV file

        :param report_header: The report header model
        :param file: The TSV file to write to
        :param include_attributes: Include the Report_Attributes value
        """
        tsv_writer = csv.writer(file, delimiter='\t')
        tsv_writer.writerow(["Report_Name", report_header.report_name])
        tsv_writer.writerow(["Report_ID", report_header.report_id])
        tsv_writer.writerow(["Release", report_header.release])
        tsv_writer.writerow(["Institution_Name", report_header.institution_name])

        institution_ids_str = ""
        for institution_id in report_header.institution_ids:
            institution_ids_str += f"{institution_id.value}; "
        tsv_writer.writerow(["Institution_ID", institution_ids_str.rstrip("; ")])

        metric_types_str = ""
        reporting_period_str = ""
        report_filters_str = ""
        for report_filter in report_header.report_filters:
            if report_filter.name == "Metric_Type":
                metric_types_str += f"{report_filter.value}; "
            elif report_filter.name == "Begin_Date" or report_filter.name == "End_Date":
                reporting_period_str += f"{report_filter.name}={report_filter.value}; "
            else:
                report_filters_str += f"{report_filter.name}={report_filter.value}; "
        tsv_writer.writerow(["Metric_Types", metric_types_str.replace("|", "; ").rstrip("; ")])
        tsv_writer.writerow(["Report_Filters", report_filters_str.rstrip("; ")])

        report_attributes_str = ""
        if include_attributes:
            for report_attribute in report_header.report_attributes:
                report_attributes_str += f"{report_attribute.name}={report_attribute.value}; "
        tsv_writer.writerow(["Report_Attributes", report_attributes_str.rstrip("; ")])

        exceptions_str = ""
        for exception in report_header.exceptions:
            exceptions_str += f"{exception.code}: {exception.message} ({exception.data}); "
        tsv_writer.writerow(["Exceptions", exceptions_str.rstrip("; ")])

        tsv_writer.writerow(["Reporting_Period", reporting_period_str.rstrip("; ")])
        tsv_writer.writerow(["Created", report_header.created])
        tsv_writer.writerow(["Created_By", report_header.created_by])
        tsv_writer.writerow([])

    @staticmethod
    def add_report_rows_to_file(report_type: str, report_rows: list, begin_date: QDate, end_date: QDate, file,
                                include_all_attributes: bool, is_special: bool = False,
                                special_options: SpecialReportOptions = None) -> bool:
        """Adds the report's rows to a TSV file

        :param report_type: The report type
        :param report_rows: The report's rows
        :param begin_date: The first date in the report
        :param end_date: The last date in the report
        :param file: The TSV file to write to
        :param include_all_attributes: Option to include all possible attributes for this report type to the report
        :param is_special: If this is a special report
        :param special_options: The special options if this is a special report
        """
        column_names = []
        row_dicts = []

        if report_type == "PR":
            column_names += ["Platform"]
            if is_special:
                special_options_dict = special_options.__dict__
                if special_options_dict["data_type"][0]: column_names.append("Data_Type")
                if special_options_dict["access_method"][0]: column_names.append("Access_Method")
            elif include_all_attributes:
                column_names.append("Data_Type")
                column_names.append("Access_Method")

            row: ReportRow
            for row in report_rows:
                row_dict = {"Platform": row.platform}
                if is_special:
                    special_options_dict = special_options.__dict__
                    if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type
                    if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method

                    if not special_options_dict["exclude_monthly_details"][0]:
                        row_dict.update(row.month_counts)
                else:
                    if include_all_attributes:
                        row_dict["Data_Type"] = row.data_type
                        row_dict["Access_Method"] = row.access_method
                    row_dict.update(row.month_counts)

                row_dict.update({"Metric_Type": row.metric_type,
                                 "Reporting_Period_Total": row.total_count})
                row_dicts.append(row_dict)

        elif report_type == "PR_P1":
            column_names += ["Platform"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Platform": row.platform,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "DR":
            column_names += ["Database", "Publisher", "Publisher_ID", "Platform", "Proprietary_ID"]
            if is_special:
                special_options_dict = special_options.__dict__
                if special_options_dict["data_type"][0]: column_names.append("Data_Type")
                if special_options_dict["access_method"][0]: column_names.append("Access_Method")
            elif include_all_attributes:
                column_names.append("Data_Type")
                column_names.append("Access_Method")

            row: ReportRow
            for row in report_rows:
                row_dict = {"Database": row.database,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "Proprietary_ID": row.proprietary_id}

                if is_special:
                    special_options_dict = special_options.__dict__
                    if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type
                    if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method

                    if not special_options_dict["exclude_monthly_details"][0]:
                        row_dict.update(row.month_counts)
                else:
                    if include_all_attributes:
                        row_dict["Data_Type"] = row.data_type
                        row_dict["Access_Method"] = row.access_method
                    row_dict.update(row.month_counts)

                row_dict.update({"Metric_Type": row.metric_type,
                                 "Reporting_Period_Total": row.total_count})
                row_dicts.append(row_dict)

        elif report_type == "DR_D1" or report_type == "DR_D2":
            column_names += ["Database", "Publisher", "Publisher_ID", "Platform", "Proprietary_ID"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Database": row.database,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "Proprietary_ID": row.proprietary_id,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "TR":
            column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "ISBN",
                             "Print_ISSN", "Online_ISSN", "URI"]
            if is_special:
                special_options_dict = special_options.__dict__
                if special_options_dict["data_type"][0]: column_names.append("Data_Type")
                if special_options_dict["section_type"][0]: column_names.append("Section_Type")
                if special_options_dict["yop"][0]: column_names.append("YOP")
                if special_options_dict["access_type"][0]: column_names.append("Access_Type")
                if special_options_dict["access_method"][0]: column_names.append("Access_Method")
            elif include_all_attributes:
                column_names.append("Data_Type")
                column_names.append("Section_Type")
                column_names.append("YOP")
                column_names.append("Access_Type")
                column_names.append("Access_Method")

            row: ReportRow
            for row in report_rows:
                row_dict = {"Title": row.title,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "ISBN": row.isbn,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri}

                if is_special:
                    special_options_dict = special_options.__dict__
                    if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type
                    if special_options_dict["section_type"][0]: row_dict["Section_Type"] = row.section_type
                    if special_options_dict["yop"][0]: row_dict["YOP"] = row.yop
                    if special_options_dict["access_type"][0]: row_dict["Access_Type"] = row.access_type
                    if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method

                    if not special_options_dict["exclude_monthly_details"][0]:
                        row_dict.update(row.month_counts)
                else:
                    if include_all_attributes:
                        row_dict["Data_Type"] = row.data_type
                        row_dict["Section_Type"] = row.section_type
                        row_dict["YOP"] = row.yop
                        row_dict["Access_Type"] = row.access_type
                        row_dict["Access_Method"] = row.access_method
                    row_dict.update(row.month_counts)

                row_dict.update({"Metric_Type": row.metric_type,
                                 "Reporting_Period_Total": row.total_count})
                row_dicts.append(row_dict)

        elif report_type == "TR_B1" or report_type == "TR_B2":
            column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "ISBN",
                             "Print_ISSN", "Online_ISSN", "URI", "YOP"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Title": row.title,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "ISBN": row.isbn,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri,
                            "YOP": row.yop,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "TR_B3":
            column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "ISBN",
                             "Print_ISSN", "Online_ISSN", "URI", "YOP", "Access_Type"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Title": row.title,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "ISBN": row.isbn,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri,
                            "YOP": row.yop,
                            "Access_Type": row.access_type,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "TR_J1" or report_type == "TR_J2":
            column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "Print_ISSN",
                             "Online_ISSN", "URI"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Title": row.title,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "TR_J3":
            column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "Print_ISSN",
                             "Online_ISSN", "URI", "Access_Type"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Title": row.title,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri,
                            "Access_Type": row.access_type,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "TR_J4":
            column_names += ["Title", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "Print_ISSN",
                             "Online_ISSN", "URI", "YOP"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Title": row.title,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri,
                            "YOP": row.yop,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "IR":
            column_names += ["Item", "Publisher", "Publisher_ID", "Platform"]
            if is_special:
                special_options_dict = special_options.__dict__
                if special_options_dict["authors"][0]: column_names.append("Authors")
                if special_options_dict["publication_date"][0]: column_names.append("Publication_Date")
                if special_options_dict["article_version"][0]: column_names.append("Article_version")
            elif include_all_attributes:
                column_names.append("Authors")
                column_names.append("Publication_Date")
                column_names.append("Article_version")
            column_names += ["DOI", "Proprietary_ID", "ISBN", "Print_ISSN", "Online_ISSN", "URI"]
            if is_special:
                special_options_dict = special_options.__dict__
                if special_options_dict["include_parent_details"][0]:
                    column_names += ["Parent_Title", "Parent_Authors", "Parent_Publication_Date",
                                     "Parent_Article_Version", "Parent_Data_Type", "Parent_DOI",
                                     "Parent_Proprietary_ID", "Parent_ISBN", "Parent_Print_ISSN", "Parent_Online_ISSN",
                                     "Parent_URI"]
                if special_options_dict["include_component_details"][0]:
                    column_names += ["Component_Title", "Component_Authors", "Component_Publication_Date",
                                     "Component_Data_Type", "Component_DOI", "Component_Proprietary_ID",
                                     "Component_ISBN", "Component_Print_ISSN", "Component_Online_ISSN", "Component_URI"]
                if special_options_dict["data_type"][0]: column_names.append("Data_Type")
                if special_options_dict["yop"][0]: column_names.append("YOP")
                if special_options_dict["access_type"][0]: column_names.append("Access_Type")
                if special_options_dict["access_method"][0]: column_names.append("Access_Method")
            elif include_all_attributes:
                column_names += ["Parent_Title", "Parent_Authors", "Parent_Publication_Date", "Parent_Article_Version",
                                 "Parent_Data_Type", "Parent_DOI", "Parent_Proprietary_ID", "Parent_ISBN",
                                 "Parent_Print_ISSN", "Parent_Online_ISSN", "Parent_URI"]
                column_names += ["Component_Title", "Component_Authors", "Component_Publication_Date",
                                 "Component_Data_Type", "Component_DOI", "Component_Proprietary_ID", "Component_ISBN",
                                 "Component_Print_ISSN", "Component_Online_ISSN", "Component_URI"]
                column_names.append("Data_Type")
                column_names.append("YOP")
                column_names.append("Access_Type")
                column_names.append("Access_Method")

            row: ReportRow
            for row in report_rows:
                row_dict = {"Item": row.item,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "ISBN": row.isbn,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri}

                if is_special:
                    special_options_dict = special_options.__dict__
                    if special_options_dict["authors"][0]: row_dict["Authors"] = row.authors
                    if special_options_dict["publication_date"][0]: row_dict["Publication_Date"] = row.publication_date
                    if special_options_dict["article_version"][0]: row_dict["Article_version"] = row.article_version
                    if special_options_dict["include_parent_details"][0]:
                        row_dict.update({"Parent_Title": row.parent_title,
                                         "Parent_Authors": row.parent_authors,
                                         "Parent_Publication_Date": row.parent_publication_date,
                                         "Parent_Article_Version": row.parent_article_version,
                                         "Parent_Data_Type": row.parent_data_type,
                                         "Parent_DOI": row.parent_doi,
                                         "Parent_Proprietary_ID": row.parent_proprietary_id,
                                         "Parent_ISBN": row.parent_isbn,
                                         "Parent_Print_ISSN": row.parent_print_issn,
                                         "Parent_Online_ISSN": row.parent_online_issn,
                                         "Parent_URI": row.parent_uri})
                    if special_options_dict["include_component_details"][0]:
                        row_dict.update({"Component_Title": row.component_title,
                                         "Component_Authors": row.component_authors,
                                         "Component_Publication_Date": row.component_publication_date,
                                         "Component_Data_Type": row.component_data_type,
                                         "Component_DOI": row.component_doi,
                                         "Component_Proprietary_ID": row.component_proprietary_id,
                                         "Component_ISBN": row.component_isbn,
                                         "Component_Print_ISSN": row.component_print_issn,
                                         "Component_Online_ISSN": row.component_online_issn,
                                         "Component_URI": row.component_uri})
                    if special_options_dict["data_type"][0]: row_dict["Data_Type"] = row.data_type
                    if special_options_dict["yop"][0]: row_dict["YOP"] = row.yop
                    if special_options_dict["access_type"][0]: row_dict["Access_Type"] = row.access_type
                    if special_options_dict["access_method"][0]: row_dict["Access_Method"] = row.access_method

                    if not special_options_dict["exclude_monthly_details"][0]:
                        row_dict.update(row.month_counts)
                else:
                    if include_all_attributes:
                        row_dict["Authors"] = row.authors
                        row_dict["Publication_Date"] = row.publication_date
                        row_dict["Article_version"] = row.article_version
                        row_dict.update({"Parent_Title": row.parent_title,
                                         "Parent_Authors": row.parent_authors,
                                         "Parent_Publication_Date": row.parent_publication_date,
                                         "Parent_Article_Version": row.parent_article_version,
                                         "Parent_Data_Type": row.parent_data_type,
                                         "Parent_DOI": row.parent_doi,
                                         "Parent_Proprietary_ID": row.parent_proprietary_id,
                                         "Parent_ISBN": row.parent_isbn,
                                         "Parent_Print_ISSN": row.parent_print_issn,
                                         "Parent_Online_ISSN": row.parent_online_issn,
                                         "Parent_URI": row.parent_uri})
                        row_dict.update({"Component_Title": row.component_title,
                                         "Component_Authors": row.component_authors,
                                         "Component_Publication_Date": row.component_publication_date,
                                         "Component_Data_Type": row.component_data_type,
                                         "Component_DOI": row.component_doi,
                                         "Component_Proprietary_ID": row.component_proprietary_id,
                                         "Component_ISBN": row.component_isbn,
                                         "Component_Print_ISSN": row.component_print_issn,
                                         "Component_Online_ISSN": row.component_online_issn,
                                         "Component_URI": row.component_uri})
                        row_dict["Data_Type"] = row.data_type
                        row_dict["YOP"] = row.yop
                        row_dict["Access_Type"] = row.access_type
                        row_dict["Access_Method"] = row.access_method
                    row_dict.update(row.month_counts)

                row_dict.update({"Metric_Type": row.metric_type,
                                 "Reporting_Period_Total": row.total_count})
                row_dicts.append(row_dict)

        elif report_type == "IR_A1":
            column_names += ["Item", "Publisher", "Publisher_ID", "Platform", "Authors", "Publication_Date",
                             "Article_version", "DOI", "Proprietary_ID", "Print_ISSN", "Online_ISSN", "URI",
                             "Parent_Title", "Parent_Authors", "Parent_Article_Version", "Parent_DOI",
                             "Parent_Proprietary_ID", "Parent_Print_ISSN", "Parent_Online_ISSN", "Parent_URI",
                             "Access_Type"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Item": row.item,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "Authors": row.authors,
                            "Publication_Date": row.publication_date,
                            "Article_version": row.article_version,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "Print_ISSN": row.print_issn,
                            "Online_ISSN": row.online_issn,
                            "URI": row.uri,
                            "Parent_Title": row.parent_title,
                            "Parent_Authors": row.parent_authors,
                            "Parent_Article_Version": row.parent_article_version,
                            "Parent_DOI": row.parent_doi,
                            "Parent_Proprietary_ID": row.parent_proprietary_id,
                            "Parent_Print_ISSN": row.parent_print_issn,
                            "Parent_Online_ISSN": row.parent_online_issn,
                            "Parent_URI": row.parent_uri,
                            "Access_Type": row.access_type,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        elif report_type == "IR_M1":
            column_names += ["Item", "Publisher", "Publisher_ID", "Platform", "DOI", "Proprietary_ID", "URI"]

            row: ReportRow
            for row in report_rows:
                row_dict = {"Item": row.item,
                            "Publisher": row.publisher,
                            "Publisher_ID": row.publisher_id,
                            "Platform": row.platform,
                            "DOI": row.doi,
                            "Proprietary_ID": row.proprietary_id,
                            "URI": row.uri,
                            "Metric_Type": row.metric_type,
                            "Reporting_Period_Total": row.total_count}
                row_dict.update(row.month_counts)

                row_dicts.append(row_dict)

        column_names += ["Metric_Type", "Reporting_Period_Total"]

        if is_special:
            special_options_dict = special_options.__dict__
            if not special_options_dict["exclude_monthly_details"][0]:
                column_names += get_month_years(begin_date, end_date)
        else:
            column_names += get_month_years(begin_date, end_date)

        tsv_dict_writer = csv.DictWriter(file, column_names, delimiter='\t')
        tsv_dict_writer.writeheader()

        if len(row_dicts) == 0:
            return False

        tsv_dict_writer.writerows(row_dicts)
        return True

    def save_json_file(self, json_string: str):
        """Saves a raw JSON file of the report

        :param json_string: The JSON string
        """
        file_dir = f"{PROTECTED_DATABASE_FILE_DIR}_json/{self.begin_date.toString('yyyy')}/{self.vendor.name}/"
        file_name = f"{self.begin_date.toString('yyyy')}_{self.vendor.name}_{self.report_type}.json"
        file_path = f"{file_dir}{file_name}"

        if not path.isdir(file_dir):
            makedirs(file_dir)

        json_file = open(file_path, 'w', encoding="utf-8")
        json_file.write(json_string)
        json_file.close()

    def notify_worker_finished(self):
        """Notifies any listeners that this worker has finished"""
        self.worker_finished_signal.emit(self.worker_id)

# El Psy Kongroo