main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data management imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as kungfupanda                    # pandas for data manipulation and markdown\n",
    "from pandas import DataFrame                    # export\n",
    "\n",
    "import argparse                                 # For command line arguments when calling py script with flags\n",
    "import pickle                                   # for saving/loading json records and file \n",
    "                                                # modification date history\n",
    "import json\n",
    "\n",
    "from questionDataclass import questionDataclass as Question"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OS and directory management imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir                          # for file retrieval and path calculations\n",
    "from os.path import isfile, join\n",
    "from os import stat\n",
    "\n",
    "from os.path import isdir                       # for creation of topic markdown folder if \n",
    "from os import mkdir                            # not present\n",
    "\n",
    "from os import getcwd                           # gets current working DIR for calculating git \n",
    "                                                # root of submissions folder \n",
    "\n",
    "from os import chdir                            # for changing the working directory to ensure\n",
    "from os.path import abspath, dirname            # relative paths used are from this script's\n",
    "import sys                                      # location rather than the calling location\n",
    "                                                # e.g. if you call `python someFolder/main.py`\n",
    "                                                #      then it will still work."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Environment variable imports + file log and creation time imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import getenv, environ                  # for environment variables\n",
    "from dotenv import load_dotenv, find_dotenv     # for config purposes (.env file)\n",
    "\n",
    "import subprocess                               # tracing git log history for ctimes and mtimes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os.path import getmtime, getctime          # retreiving file creation/modification times\n",
    "from datetime import datetime, timedelta\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "QOL and anti-reundancy imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Set, Dict, List, Tuple       # misc. QOL imports\n",
    "from collections import defaultdict\n",
    "from icecream import ic                         # for debugging / outputs\n",
    "\n",
    "import re                                       # for regex file name matching / question number matching\n",
    "from functools import cache                     # for redundancy protection\n",
    "\n",
    "# TQDM import done separately below after checking if this is a .py or .ipynb file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Script Configuration\n",
    "#### `.env` variables and `working directories`\n",
    "1. Loads `env` variables for reference.\n",
    "    1. Tries to retrieve it from `../` if found (prioritizing template).\n",
    "    2. If failure, use the `.env` found in the current script directory (in the updater).\n",
    "2. If is a script run, denotes it as such for script flag references and ensures working directory is the script's location rather than the calling directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading env variables\n",
    "print('Default .env activated from script directory (.readme_updater/)')\n",
    "load_dotenv(find_dotenv(), override=True)\n",
    "\n",
    "if '.env' in listdir('../') :\n",
    "    print('.env found in ../ directory. Overriding default...')\n",
    "    load_dotenv(find_dotenv('../.env'), override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: if the script is being run from a jupyter notebook, then it should\n",
    "# already be in the correct directory.\n",
    "IS_NOTEBOOK = True\n",
    "try:\n",
    "    if 'ipykernel' not in sys.modules:\n",
    "        print('Working directory being set to script location.')\n",
    "        IS_NOTEBOOK = False\n",
    "        chdir(dirname(abspath(__file__)))\n",
    "    else :\n",
    "        print('Working directory already set to script location. No need for adjustment')\n",
    "except NameError:\n",
    "    print('NameError')\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TQDM import based off if current running script is a jupyter notebook\n",
    "# or a python script\n",
    "\n",
    "if IS_NOTEBOOK :\n",
    "    print('Importing tqdm.notebook')\n",
    "    from tqdm.notebook import tqdm\n",
    "else :\n",
    "    print('Importing tqdm (non-notebook)')\n",
    "    from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# README_ABS_DIR will get confirmed in if name==main prior to running\n",
    "README_ABS_DIR      = getcwd().replace('\\\\', '/')\n",
    "NOTEBOOK_ABS_DIR    = README_ABS_DIR\n",
    "MAIN_DIRECTORY      = NOTEBOOK_ABS_DIR[NOTEBOOK_ABS_DIR.rfind('/')+1:]\n",
    "\n",
    "print(f'{NOTEBOOK_ABS_DIR = }')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "README_PATH                 = getenv('README_PATH')\n",
    "LEETCODE_PATH_FROM_README   = getenv('QUESTIONS_PATH_FROM_README')\n",
    "LEETCODE_PATH_REFERENCE     = join(README_PATH, LEETCODE_PATH_FROM_README)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Methods\n",
    "\n",
    "AddCase $\\rightarrow$ takes information for a new question file and formats it accordingly for a row.\n",
    "\n",
    "UpdateLanguage $\\rightarrow$ if a question already has a solution, this is called instead to insert the new file link to the existing row details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Categories besides those in lists\n",
    "PRIMARY_CATEGORIES = set(['Daily', 'Weekly Premium', 'Contest', 'Favourite'])\n",
    "_oldest_date = datetime.now()\n",
    "# _oldest_date = datetime(2024, 7, 23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def individualCTimeViaGit(cmd: List[str]) -> Tuple[datetime, datetime] :\n",
    "    process = subprocess.Popen(cmd,\n",
    "                               shell=False,\n",
    "                               stdin=None,\n",
    "                               stdout=subprocess.PIPE,\n",
    "                               stderr=subprocess.PIPE)\n",
    "    result = process.stdout.readlines()\n",
    "    modifiedTimes = []\n",
    "    \n",
    "    for line in result:\n",
    "        temp = line.decode(\"utf-8\").replace('\\n', '')\n",
    "    \n",
    "        # In case of a redundant '\\n' at the end of an output\n",
    "        if temp :\n",
    "            modifiedTimes.append(temp)\n",
    "        \n",
    "    # Debugging\n",
    "    if '1404' in cmd[-1] :\n",
    "        print(f'{cmd = }')\n",
    "        print(f'{modifiedTimes = }')\n",
    "        print(f'{result = }')\n",
    "    \n",
    "    try :\n",
    "        creationDate = datetime.strptime(time.ctime(int(min(modifiedTimes))), '%a %b %d %H:%M:%S %Y')\n",
    "        modifiedDate = datetime.strptime(time.ctime(int(max(modifiedTimes))), '%a %b %d %H:%M:%S %Y')\n",
    "        \n",
    "    except ValueError as ve:\n",
    "        print(f'Error in parsing {path}')\n",
    "        print(f'{modifiedTimes}')\n",
    "        print(ve)\n",
    "        exit()\n",
    "    \n",
    "    return (creationDate, modifiedDate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ALL_GIT_CM_TIMES = {}\n",
    "def getAllCTimesViaGit(paths: List[str]) -> Dict[str, Tuple[datetime, datetime]] :\n",
    "    '''\n",
    "    WARNING: DO NOT USE LOCALLY. SLOW IF RAN LOCALLY.\n",
    "    \n",
    "    GITHUB ACTIONS ARE ABLE TO PERFORM THIS QUICKLY (~10s for the script for ~700 files) \n",
    "    BUT A LOCAL RUN OF `-g` CAN TAKE UPWARDS OF 10 MINUTES FOR THE SAMEN NUMBER OF FILES.\n",
    "\n",
    "    To avoid having to constantly swap directories, this function parses all the ctimes and mtimes \n",
    "    in one block of time. This gets activated with the `-g` flag. Default otherwise is to use the \n",
    "    regular `getctime` and `getmtime` functions locally which is much much faster. This only exists \n",
    "    to compensate for the inability for ctime and mtime checking with git actions.\n",
    "    '''\n",
    "    print(f'Beginning parsing of git logs for file creation and modification dates...')\n",
    "    print(f'Script path: {getcwd() = }')\n",
    "    chdir('../')\n",
    "    print(f'README path: {getcwd() = }')\n",
    "\n",
    "    cmd = r\"git log -M --format=%ct --reverse --\".split()\n",
    "    # cmd = r\"git log -M --follow --format=%ct --reverse --\".split()\n",
    "    # cmd = r\"git log --follow --format=%ct --reverse --\".split()\n",
    "    output = {}\n",
    "    \n",
    "    oldest_date = datetime.now()\n",
    "\n",
    "    # if getenv(\"GITHUB_ACTIONS\") == 'true' :\n",
    "    #     print('\\n')\n",
    "    #     print('/' + '=' * 15 + '\\\\')\n",
    "    #     print(' ', end='')\n",
    "    #     pqBarsPrinted = 0\n",
    "    #     for i, path in enumerate(paths) :\n",
    "    #         path = join(LEETCODE_PATH_FROM_README, path)\n",
    "    #         output[path] = individualCTimeViaGit(cmd + [path])\n",
    "            \n",
    "    #         if output[path][0] < oldest_date :\n",
    "    #             oldest_date = output[path][0]\n",
    "            \n",
    "    #         curChunk = int((i / len(paths)) * 15)\n",
    "    #         if curChunk > pqBarsPrinted :\n",
    "    #             print('=' * (curChunk - pqBarsPrinted), end='')\n",
    "    #             pqBarsPrinted = curChunk\n",
    "                \n",
    "    #     print((15 - pqBarsPrinted) * '=', '\\n\\n')\n",
    "    # else :\n",
    "    \n",
    "    \n",
    "    with tqdm(total=len(paths)) as pbar :\n",
    "    # with tqdm(total=len(paths), position=0, leave=True) as pbar :\n",
    "        for i, path in enumerate(paths) :\n",
    "            path = join(LEETCODE_PATH_FROM_README, path)\n",
    "            output[path] = individualCTimeViaGit(cmd + [path])\n",
    "            \n",
    "            if output[path][0] < oldest_date :\n",
    "                oldest_date = output[path][0]\n",
    "            \n",
    "            pbar.update(1)\n",
    "\n",
    "    global _oldest_date\n",
    "    _oldest_date = oldest_date.replace(hour=0, minute=0, second=0, microsecond=0)\n",
    "\n",
    "    # Usually I'd avoid using global for this but this is a personal project so it should be fine.\n",
    "    _ALL_GIT_CM_TIMES.update(output)\n",
    "    print(f'{_ALL_GIT_CM_TIMES = }')\n",
    "    \n",
    "    chdir(MAIN_DIRECTORY)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def getCtimesMtimesGitHistory(path: str) -> Tuple[datetime, datetime] :\n",
    "    '''\n",
    "    WARNING: DO NOT USE LOCALLY. SLOW IF RAN LOCALLY RELATIVE TO THE REGULAR CTIME FUNCTION.\n",
    "\n",
    "    IF RUNNING LOCALLY, RUN (getCtimeMtimesMain) I.E. WITHOUT THE `-g` FLAG.\n",
    "\n",
    "    The cost for a single file isn't significant however when you reach ~100+ files, \n",
    "    the cumulative wait can go into the minutes compared to the seconds it would take \n",
    "    with the regular `getctime` and `getmtime` functions (without the `-g` flag)\n",
    "    '''\n",
    "    path = path[path.find('/') + 1:]\n",
    "    chdir('../')\n",
    "    cmd = r\"git log --follow --format=%ct --reverse --\".split() + [f'{path}']\n",
    "\n",
    "    cmDates = individualCTimeViaGit(cmd)\n",
    "\n",
    "    chdir(MAIN_DIRECTORY)\n",
    "\n",
    "    return cmDates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_GIT_DATES = False\n",
    "\n",
    "@cache\n",
    "def getCtimeMtimesMain(path: str) -> Tuple[datetime, datetime] :\n",
    "    '''\n",
    "    Returns the a tuple containing the datetime objs of \n",
    "    (create date and time, modification date and time)\n",
    "\n",
    "    @param useGitDates: bool = False\n",
    "        If true, it will track the creation/modification dates of the file \n",
    "        according to the git history. This is mainly to counter the issue in \n",
    "        GitHub actions where the file creation date is the time of the action.\n",
    "    '''\n",
    "    \n",
    "    if USE_GIT_DATES :\n",
    "        return getCtimesMtimesGitHistory(path)\n",
    "    \n",
    "    creation_date = time.ctime(getctime(path))\n",
    "    modification_date = time.ctime(getmtime(path))\n",
    "\n",
    "    creation_date = datetime.strptime(creation_date, \"%a %b %d %H:%M:%S %Y\")\n",
    "    modification_date = datetime.strptime(modification_date, \"%a %b %d %H:%M:%S %Y\")\n",
    "\n",
    "    # I've sometimes encountered weird meta data issues so just as a precaution\n",
    "    if creation_date > modification_date :\n",
    "        return (modification_date, creation_date)\n",
    "    \n",
    "    return (creation_date, modification_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCtimeMtimes(path: str, *, preCalculated: Dict[str, Tuple[datetime, datetime]] = None) -> Tuple[datetime, datetime] :\n",
    "    # Due to readme realtive and script relative paths\n",
    "    readme_path = path if ('../' not in path) else path[path.find('../') + len('../'):]\n",
    "    if _ALL_GIT_CM_TIMES and readme_path in _ALL_GIT_CM_TIMES :\n",
    "        return _ALL_GIT_CM_TIMES[readme_path]\n",
    "    \n",
    "    if preCalculated and readme_path in preCalculated :\n",
    "        return preCalculated[readme_path]\n",
    "\n",
    "    return getCtimeMtimesMain(path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addCase(level:              str,\n",
    "            number:             int, \n",
    "            title:              str, \n",
    "            categories:         Set[str],\n",
    "            language:           str,\n",
    "            notebook_path:      str,\n",
    "            readme_path:        str,\n",
    "            fileLatestTimes:    dict,\n",
    "            contestTitle:       str=None,\n",
    "            contestQNo:         str=None) -> dict :\n",
    "    '''\n",
    "    Takes the data found on a question not encountered before and \n",
    "    converts it into a callable dictionary with all the relevant \n",
    "    information\n",
    "\n",
    "    ### Parameters (Required) :\n",
    "    level : str\n",
    "        Difficulty indicator of the question (e, m, h)\n",
    "    number : int\n",
    "        The official LeetCode question number\n",
    "    title : str\n",
    "        The title of the question (colloquial name)\n",
    "    categories : Set[str]\n",
    "        The categories the question falls under (e.g. Contest, Daily, etc.)\n",
    "    language : str\n",
    "        The programming language used to solve the question\n",
    "    notebook_path : str\n",
    "        The path from the main.py/ipynb script to the code file in question\n",
    "    readme_path : str\n",
    "        The path from the README.md file to be exported to the code file in question\n",
    "    fileLatestTimes : dict\n",
    "        A dictionary containing the latest modification times of all files\n",
    "        in the repository\n",
    "    \n",
    "    ### Parameters (Optional) :\n",
    "    contestTitle : str\n",
    "        The title of the contest the question was a part of if applicable\n",
    "    contestQNo : str\n",
    "        The question number in the contest if applicable (e.g. q1, q2, etc.)\n",
    "\n",
    "    ### Returns :\n",
    "    output : dict\n",
    "        A dictionary containing all the relevant information for the question\n",
    "        to be used in the final output\n",
    "    '''\n",
    "\n",
    "    creation_date, modification_date = getCtimeMtimes(notebook_path)\n",
    "    fileLatestTimes[readme_path] = modification_date\n",
    "\n",
    "    try :\n",
    "        fileSize = stat(notebook_path).st_size\n",
    "    except FileNotFoundError as fnfe :\n",
    "        fileSize = 0\n",
    "        print(fnfe)\n",
    "    \n",
    "\n",
    "    if not categories :\n",
    "        categories = set()\n",
    "\n",
    "    match level[0].lower() :\n",
    "        case 'e' :\n",
    "            level = 'Easy'\n",
    "        case 'm' : \n",
    "            level = 'Medium'\n",
    "        case 'h' :\n",
    "            level = 'Hard'\n",
    "        case _ :\n",
    "            level = 'Unknown'\n",
    "\n",
    "    output = {\n",
    "                'level':                level,\n",
    "                'number':               number,\n",
    "                'title':                title, \n",
    "                'categories':           categories,\n",
    "                'contestTitle':         contestTitle,\n",
    "                'contestQNo':           contestQNo,\n",
    "                'date_done':            creation_date,          # First time completed\n",
    "                'date_modified':        modification_date,      # Most recent date\n",
    "                'solution':             '',\n",
    "                'solutions':            {language: [readme_path]},\n",
    "                'languages':            set([language]),\n",
    "                'bytes':                fileSize\n",
    "             }\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateQuestion(orig:               dict, \n",
    "                   *,\n",
    "                   language:           str,\n",
    "                   categories:         Set[str],\n",
    "                   notebook_path:      str,\n",
    "                   readme_path:        str,\n",
    "                   fileLatestTimes:    dict,\n",
    "                   contestTitle:       str=None,\n",
    "                   contestQNo:         str=None) -> dict :\n",
    "    '''\n",
    "    Takes question data of a question that's already been encountered and \n",
    "    updates the relevant dictionary with the new information found. Similar \n",
    "    to addCase but for questions that have already been encountered.\n",
    "\n",
    "    ### Parameters :\n",
    "    orig : dict\n",
    "        The original dictionary containing all the relevant information from previous encounters\n",
    "    \n",
    "    All other parameters are the same as addCase and are optional in order to update them.\n",
    "        \n",
    "    ### Returns :\n",
    "    orig : dict\n",
    "        The updated dictionary containing all the relevant information from previous encounters\n",
    "    '''\n",
    "    \n",
    "    # Another question file found\n",
    "    if language and language not in orig['languages'] :\n",
    "        orig['languages'].add(language)\n",
    "\n",
    "    if contestTitle :\n",
    "        orig['contestTitle'] = contestTitle\n",
    "        \n",
    "    if contestQNo :\n",
    "        orig['contestQNo'] = contestQNo\n",
    "          \n",
    "    if categories :\n",
    "        orig['categories'] |= categories\n",
    "\n",
    "    if notebook_path and readme_path :\n",
    "        creation_date, modification_date = getCtimeMtimes(notebook_path)\n",
    "        \n",
    "        if creation_date < orig['date_done'] :\n",
    "            orig['date_done'] = creation_date\n",
    "        if modification_date > orig['date_modified'] :\n",
    "            orig['date_modified'] = modification_date\n",
    "            fileLatestTimes[readme_path] = modification_date\n",
    "\n",
    "        if language not in orig['solutions'] :\n",
    "            orig['solutions'][language] = []\n",
    "        orig['solutions'][language].append(readme_path)\n",
    "\n",
    "        try :\n",
    "            fileSize = stat(notebook_path).st_size\n",
    "        except FileNotFoundError as fnfe :\n",
    "            fileSize = 0\n",
    "            print(fnfe)\n",
    "        orig['bytes'] += fileSize\n",
    "  \n",
    "    return orig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pickle Processes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUESTION_DATA_FOLDER = join(getenv('SUBMODULE_DATA_PATH'), getenv('LEETCODE_QUESTION_DETAILS'))\n",
    "SUBMODULE_DATA_PATH = getenv('SUBMODULE_DATA_PATH')\n",
    "\n",
    "@cache\n",
    "def retrieveQuestionDetails() -> dict :\n",
    "    '''\n",
    "    Retrieves the question details (i.e. title, acRates, difficulties, etc.) from\n",
    "    the relevant `.pkl` file containing parsed official LeetCode json data.\n",
    "\n",
    "    ### Returns :\n",
    "    questionDetailsDict : dict[int, details]\n",
    "        A dictionary containing the question details matched to the question's assigned number\n",
    "    '''\n",
    "\n",
    "    print(f'{getcwd() = }')\n",
    "    \n",
    "    print(f'{listdir(SUBMODULE_DATA_PATH)}')\n",
    "    print(f'Question details path: {QUESTION_DATA_FOLDER = }')\n",
    "\n",
    "    if not isfile(QUESTION_DATA_FOLDER) :\n",
    "        print('\\nError in parsing official question data. leetcode.pkl not found. Exiting...')\n",
    "        print()\n",
    "        exit()\n",
    "    else : \n",
    "        print('\\nFiles found. Importing now...\\n')\n",
    "\n",
    "\n",
    "    # schema: key=int(questionNumber)   val=(title, titleSlug, paidOnly, difficulty, acRate)\n",
    "    with open(join(QUESTION_DATA_FOLDER), 'rb') as fp:\n",
    "        questionDetailsDict = pickle.load(fp)\n",
    "\n",
    "    return questionDetailsDict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HISTORY_PATH = join(getenv('USER_DATA_PATH'), getenv('FILE_MODIFICATION_NAME'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def writeRecentFileTimes(fileLatestTimes: dict) -> bool :\n",
    "    '''Pickles the newly found most recent modification times of each question for reference in future runs'''\n",
    "    \n",
    "    with open(HISTORY_PATH, 'wb') as fp:\n",
    "        pickle.dump(fileLatestTimes, fp)\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getRecentFileTimes() -> dict :\n",
    "    '''Retrieves the pickled data from previous cases of `writeRecentFileTimes()`'''\n",
    "\n",
    "    if isfile(HISTORY_PATH) :\n",
    "        with open(HISTORY_PATH, 'rb') as fp:\n",
    "            return pickle.load(fp)\n",
    "        \n",
    "    return {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Daily and Weekly Challenges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAILIES_DATA_PATH = join(getenv('SUBMODULE_DATA_PATH'), getenv('DAILIES_FOLDER'), getenv('DAILIES_FILE'))\n",
    "WEEKLIES_DATA_PATH = join(getenv('SUBMODULE_DATA_PATH'), getenv('DAILIES_FOLDER'), getenv('WEEKLIES_FILE'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getDailies(firstDate: datetime = None) -> List[Tuple[datetime, int]] :\n",
    "    '''\n",
    "    Retrieves the daily questions from the official LeetCode json data\n",
    "    and returns them as a set of strings\n",
    "\n",
    "    ### Returns :\n",
    "    dailies : List[Tuple[date, questionNo]]\n",
    "    '''\n",
    "    if not firstDate :\n",
    "        global _oldest_date\n",
    "        firstDate = _oldest_date\n",
    "    \n",
    "    print('Oldest date found:', firstDate)\n",
    "    \n",
    "    with open(DAILIES_DATA_PATH, 'rb') as fp:\n",
    "        dailies = json.load(fp)\n",
    "    \n",
    "    output = []\n",
    "\n",
    "    for k in list(dailies.keys()) :\n",
    "        newK = datetime.strptime(k, '%Y-%m-%d')\n",
    "        if newK < firstDate :\n",
    "            continue\n",
    "\n",
    "        output.append((newK, int(dailies[k]['question']['questionFrontendId'])))\n",
    "\n",
    "    # print(f'{firstDate = }')\n",
    "    # print(f'dailies: {output = }')\n",
    "    return sorted(output, key=lambda x: x[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getWeeklies(firstDate: datetime = None) -> List[Tuple[datetime, int]] :\n",
    "    '''\n",
    "    Retrieves the weekly premium questions from the official LeetCode json data\n",
    "    and returns them as a set of strings\n",
    "\n",
    "    ### Returns :\n",
    "    weeklies : List[Tuple[date, questionNo]]\n",
    "    '''\n",
    "    if not firstDate :\n",
    "        global _oldest_date\n",
    "        firstDate = _oldest_date\n",
    "    \n",
    "    print('Oldest date found:', firstDate)\n",
    "    \n",
    "    with open(WEEKLIES_DATA_PATH, 'rb') as fp:\n",
    "        weeklies = json.load(fp)\n",
    "    \n",
    "    output = []\n",
    "\n",
    "    for k in list(weeklies.keys()) :\n",
    "        newK = datetime.strptime(k, '%Y-%m-%d')\n",
    "        if newK < firstDate :\n",
    "            continue\n",
    "\n",
    "        output.append((newK, int(weeklies[k]['question']['questionFrontendId'])))\n",
    "\n",
    "    return sorted(output, key=lambda x: x[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # NOTE: TESTING\n",
    "# temp = getWeeklies(datetime(2022, 1, 1))\n",
    "# ic(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseQuestionsForDailies(questionData: dict) -> Dict[int, Question] :\n",
    "    '''\n",
    "    Parses the official LeetCode json data for the daily and weekly premium questions\n",
    "    and returns them as a dictionary of question numbers to question objects\n",
    "\n",
    "    ### Returns :\n",
    "    dailies : Dict[int, Question]\n",
    "        A dictionary containing the daily questions as question objects\n",
    "    weeklies : Dict[int, Question]\n",
    "        A dictionary containing the weekly premium questions as question objects\n",
    "    '''\n",
    "    dailies = getDailies()\n",
    "    dailiesDict = {}\n",
    "\n",
    "    # I have ~12 hours of leeway due to potential to forget to commit\n",
    "    for date, qNo in dailies :\n",
    "        # if qNo == 1334 :\n",
    "        #     print('\\n\\nhi')\n",
    "        #     print(f'{bool(qNo in questionData) = }')\n",
    "        #     print(f\"{questionData[qNo]['date_done'] = }\")\n",
    "        #     print(f\"{date + timedelta(days=1, hours=12) = }\")\n",
    "        if qNo in questionData and questionData[qNo]['date_done'] <= date + timedelta(days=1, hours=12) :\n",
    "            dailiesDict[date] = questionData[qNo].copy()\n",
    "            dailiesDict[date]['date_done'] = date\n",
    "            questionData[qNo]['categories'].add('Daily')\n",
    "            # print(f'{dailiesDict[date] = }')\n",
    "\n",
    "    # print(f'{dailiesDict = }')\n",
    "\n",
    "    return dailiesDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseQuestionsForWeeklies(questionData: dict) -> Dict[int, Question] :\n",
    "    '''\n",
    "    Parses the official LeetCode json data for the daily and weekly premium questions\n",
    "    and returns them as a dictionary of question numbers to question objects\n",
    "\n",
    "    ### Returns :\n",
    "    dailies : Dict[int, Question]\n",
    "        A dictionary containing the daily questions as question objects\n",
    "    weeklies : Dict[int, Question]\n",
    "        A dictionary containing the weekly premium questions as question objects\n",
    "    '''\n",
    "    weeklies = getWeeklies()\n",
    "    weekliesDict = {}\n",
    "\n",
    "    # I gave ~1 day of leeway for the weeklies in case you forget to commit\n",
    "    for date, qNo in weeklies :\n",
    "        if qNo in questionData and questionData[qNo]['date_done'] <= date + timedelta(days=8) :\n",
    "            weekliesDict[date] = questionData[qNo].copy()\n",
    "            weekliesDict[date]['date_done'] = date\n",
    "            questionData[qNo]['categories'].add('Weekly Premium')\n",
    "\n",
    "    return weekliesDict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parsing Files\n",
    "Question file parsing occurs here. It organizes it into 3 different lists, separated by difficulty and sorted by question number afterwards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse one leetcode answer file in the submissions folder\n",
    "def parseCase(leetcodeFile:         str,  # file name\n",
    "              questionData:         dict, # dictionary of question data\n",
    "              fileLatestTimes:      dict,\n",
    "              reprocessMarkdown:    set,\n",
    "              *,\n",
    "              questionDetailsDict:  dict = retrieveQuestionDetails(),\n",
    "              subFolderPath:        str = '',\n",
    "              altTitle:             str = '',\n",
    "              contest:              str = None,\n",
    "              contestQNo:           str = None) -> bool:\n",
    "\n",
    "    path = join(LEETCODE_PATH_FROM_README, subFolderPath, leetcodeFile).replace(\"\\\\\", \"/\")\n",
    "\n",
    "    try :\n",
    "        number      = int(re.search(\"\\d{1,4}\", leetcodeFile).group())   # Takes the first full number as the question\n",
    "        level       = questionDetailsDict[number].level                 # number and uses that as reference\n",
    "                                                                        # e.g. 'e123 v1.py' becomes 123\n",
    "    except (AttributeError) as ae :\n",
    "        print(f'Error in parsing {leetcodeFile}: {ae.name} encountered while trying to extract question number int(...).',\n",
    "                '\\nparseCase(...)',\n",
    "                '\\nSkipping')\n",
    "        return False\n",
    "    except (KeyError) as ke :\n",
    "        print(f'Error in parsing {leetcodeFile}: {ke} encountered while trying to extract question level from questionDetailsDict.',\n",
    "                '\\nparseCase(...)',\n",
    "                '\\nAttempting to pull from the name...')\n",
    "        level = leetcodeFile[0].lower()\n",
    "        \n",
    "        if level in ['e', 'm', 'h'] :\n",
    "            print(f'Level found: {level}')\n",
    "        else :\n",
    "            print(f'Level not found. Defaulting to \"Unknown\"')\n",
    "            level = 'Unknown'\n",
    "\n",
    "\n",
    "    creationtime, modificationtime = getCtimeMtimes(join(README_PATH, path))\n",
    "\n",
    "    if path not in fileLatestTimes or max(creationtime, modificationtime) > fileLatestTimes[path] :\n",
    "        reprocessMarkdown.add(number)\n",
    "        fileLatestTimes[path] = max(creationtime, modificationtime)\n",
    "        \n",
    "\n",
    "    if number in questionDetailsDict :\n",
    "        title   = f'[{questionDetailsDict[number].title}](<https://leetcode.com/problems/{questionDetailsDict[number].slug}>)'\n",
    "    else :\n",
    "        title   = f'Question {number}'\n",
    "    categories  = set()\n",
    "    language    = leetcodeFile[leetcodeFile.rfind('.') + 1:]\n",
    "\n",
    "\n",
    "    if len(altTitle) > 0 :\n",
    "        title = altTitle + ' - ' + title\n",
    "\n",
    "\n",
    "    # Question is from a contest folder\n",
    "    if contest :\n",
    "        temp = re.findall('q\\d{1}', leetcodeFile)                       # Checking if file name has a question number (e.g. q1 of the contest)\n",
    "        if not len(temp) == 0 :\n",
    "            contestQNo = temp[0]\n",
    "\n",
    "        categories.add('Contest')\n",
    "\n",
    "\n",
    "    # for cat in PRIMARY_CATEGORIES :\n",
    "    #     if cat.lower() in leetcodeFile.lower() :\n",
    "    #         categories.add(cat)\n",
    "\n",
    "    if number in questionData :                                     # If solution already found for this question\n",
    "        questionData[number] = updateQuestion(questionData[number], \n",
    "                                              language=language, \n",
    "                                              categories=categories, \n",
    "                                              notebook_path=join(README_PATH, path), \n",
    "                                              readme_path=path,\n",
    "                                              contestTitle=contest,\n",
    "                                              contestQNo=contestQNo,\n",
    "                                              fileLatestTimes=fileLatestTimes)\n",
    "    else :\n",
    "        questionData[number] = addCase(level=level, \n",
    "                                       number=number, \n",
    "                                       title=title,\n",
    "                                       categories=categories, \n",
    "                                       language=language, \n",
    "                                       notebook_path=join(README_PATH, path), \n",
    "                                       readme_path=path,\n",
    "                                       contestTitle=contest,\n",
    "                                       contestQNo=contestQNo,\n",
    "                                       fileLatestTimes=fileLatestTimes)\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@cache\n",
    "def getCodeFiles() -> List[str] :\n",
    "    return [x for x in listdir(LEETCODE_PATH_REFERENCE) if isfile(join(LEETCODE_PATH_REFERENCE, x))\n",
    "                                                           and not x.endswith('.txt')\n",
    "                                                           and not x.endswith('.md')\n",
    "                                                           and not x.endswith('.gitignore')\n",
    "                                                           and '.' in x]\n",
    "\n",
    "@cache\n",
    "def getContestFolders() -> List[str] :\n",
    "    return [x for x in listdir(LEETCODE_PATH_REFERENCE) if not isfile(join(LEETCODE_PATH_REFERENCE, x))]\n",
    "\n",
    "@cache\n",
    "def getContextFiles(contestFolders: List[str] = getContestFolders()) -> List[str] :\n",
    "    output = [x for x in listdir(LEETCODE_PATH_REFERENCE) if isfile(join(LEETCODE_PATH_REFERENCE, x)) \n",
    "                                                             and (x.endswith('.txt') \n",
    "                                                                  or x.endswith('.md') \n",
    "                                                                  or '.' not in x)\n",
    "                                                             and not x.endswith('.gitignore')]\n",
    "    for folder in contestFolders :\n",
    "        output.extend([join(folder, y) for y in listdir(join(LEETCODE_PATH_REFERENCE, folder)) \n",
    "                                if isfile(join(LEETCODE_PATH_REFERENCE, folder, y))\n",
    "                                   and (y.endswith('.txt') \n",
    "                                        or y.endswith('.md') \n",
    "                                        or '.' not in y)\n",
    "                                   and not y.endswith('.gitignore')])\n",
    "    return output\n",
    "\n",
    "def getContestFiles(contestFolders: List[str]) -> List[Tuple[str, str]] :\n",
    "    contestLeetcodeFiles    = []\n",
    "\n",
    "    for contestFolder in contestFolders :\n",
    "        contestLeetcodeFiles.extend([(contestFolder, fileName) for fileName in listdir(join(LEETCODE_PATH_REFERENCE, contestFolder)) \n",
    "                                                                if isfile(join(LEETCODE_PATH_REFERENCE, contestFolder, fileName))\n",
    "                                                                    and not fileName.endswith('.txt')\n",
    "                                                                    and not fileName.endswith('.md')\n",
    "                                                                    and '.' in fileName\n",
    "                                                                    and not fileName.endswith('.gitignore')])\n",
    "    \n",
    "    return contestLeetcodeFiles\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sort TXT Context\n",
    "If .txt notes are placed, this adds them to their respective entry."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseContextFiles(txtFiles: str, \n",
    "                      questionData: dict,\n",
    "                      fileLatestTimes: dict, \n",
    "                      reprocessMarkdown: Set[int]) -> None:\n",
    "    for fileName in txtFiles :\n",
    "        # print(f'Context file found: {fileName}')\n",
    "\n",
    "        try :\n",
    "            if '\\\\' in fileName :\n",
    "                number = int(re.search(\"\\d{1,4}\", fileName[fileName.rfind('\\\\'):]).group())\n",
    "            elif '/' in fileName :\n",
    "                number = int(re.search(\"\\d{1,4}\", fileName[fileName.rfind('/'):]).group())\n",
    "            else :\n",
    "                number = int(re.search(\"\\d{1,4}\", fileName).group())\n",
    "        except AttributeError as ae :\n",
    "            print(f'Error in parsing {fileName}: Attribute Error encountered while trying to extract question number int(...).',\n",
    "                  '\\nparseContextFiles(...)',\n",
    "                  '\\nSkipping')\n",
    "            continue\n",
    "    \n",
    "        if number not in questionData :\n",
    "            print(f'Error. No question solution found for context file ({fileName = })')\n",
    "            continue\n",
    "        \n",
    "        questionData[number]['contextFile'] = join(LEETCODE_PATH_FROM_README, fileName)\n",
    "        path = join(LEETCODE_PATH_REFERENCE, fileName)\n",
    "        \n",
    "        creationtime, modificationtime = getCtimeMtimes(path)\n",
    "        if path not in fileLatestTimes or max(creationtime, modificationtime) > fileLatestTimes[path] :\n",
    "            fileLatestTimes[path] = max(creationtime, modificationtime)\n",
    "            reprocessMarkdown.add(number)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# List-Based Categories\n",
    "Updating `Category` columns based on the lists in the `Lists` directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LISTSDIR = getenv('LISTS_LOCATION')\n",
    "\n",
    "@cache\n",
    "def getLists() -> List[str] :\n",
    "\n",
    "    listFileNames = [x for x in listdir(LISTSDIR) if isfile(join(LISTSDIR, x)) \n",
    "                                                    and not x.startswith('.')\n",
    "                                                    and not x == 'README.md']\n",
    "    print(listFileNames)\n",
    "\n",
    "    return listFileNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Format for lists file is as follows:\n",
    "\n",
    "        [Question #]. [Question Name]\n",
    "\n",
    "        [Easy, Med., Hard]\n",
    "        Topic1\n",
    "        Topic2\n",
    "        Topic3\n",
    "        ...\n",
    "'''\n",
    "\n",
    "@cache\n",
    "def getList(fileName, filePath) -> Set[int] :\n",
    "    output = set() # can change to dict later if we want to output category info\n",
    "\n",
    "    count = 0\n",
    "    with open(filePath, 'r') as file :\n",
    "        lines = file.readlines()\n",
    "        for line in lines :\n",
    "            if re.match(r'\\d{1,4}\\.', line) :\n",
    "                count += 1\n",
    "                output.add(int(line[:line.find('.')]))\n",
    "    \n",
    "    return output\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processListData(questionData: dict,\n",
    "                    *,\n",
    "                    listFileNames: List[str] = getLists()) -> dict :\n",
    "    \n",
    "    listData = {}\n",
    "    for file in listFileNames :\n",
    "        listData[file] = getList(file, join(LISTSDIR, file))\n",
    "        for q in listData[file] :\n",
    "            if q in questionData :\n",
    "                questionData[q]['categories'].add(file)\n",
    "                \n",
    "    # print(listData)\n",
    "\n",
    "    return listData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Question Topic Grouping\n",
    "Parses the questions in `questionData` and adds their numbers to appropriate lists so that they can be parsed into their own lists as well as counted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCompletedQuestionsTopicLists(questionData: dict,\n",
    "                                    *,\n",
    "                                    questionTopicsDict: dict = retrieveQuestionDetails()) -> defaultdict :\n",
    "    \n",
    "    completedTopicLists = defaultdict(set)\n",
    "\n",
    "    for question in questionData.keys() :\n",
    "        # Shouldn't occur but just in case\n",
    "        if question not in questionTopicsDict :\n",
    "            continue\n",
    "        for topic in questionTopicsDict[question].topics :\n",
    "            completedTopicLists[topic].add(question)\n",
    "\n",
    "    return completedTopicLists\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Individual Markdown Generation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "README_PATH                     = getenv('README_PATH')\n",
    "QUESTIONS_FOLDER_FROM_README    = getenv('QUESTIONS_PATH_FROM_README')\n",
    "QUESTIONS_FOLDER                = join(README_PATH, QUESTIONS_FOLDER_FROM_README)\n",
    "\n",
    "MARKDOWN_PATH = getenv('QUESTION_MARKDOWNS_PATH_FROM_README')\n",
    "MARKDOWN_TO_SUBMISSIONS = '../' * MARKDOWN_PATH.count('/') + QUESTIONS_FOLDER_FROM_README\n",
    "\n",
    "QUESTION_DATA_FOLDER_PATH    = getenv('QUESTION_DATA_PATH')\n",
    "QUESTION_TOPICS_FILE         = getenv('LEETCODE_QUESTION_TOPICS')\n",
    "QUESTION_DETAILS_FILE        = getenv('LEETCODE_QUESTION_DETAILS')\n",
    "\n",
    "with open('question_data/language_equivs.json') as f :\n",
    "    LANGUAGE_EQUIVS = json.load(f)\n",
    "    \n",
    "BY_TOPIC_FOLDER_PATH = getenv('TOPIC_MARKDOWN_PATH_IN_MARKDOWNS_FOLDER')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MARKDOWN_TO_SUBMISSIONS\n",
    "def generate_markdown(questionNo: int, \n",
    "                      questionData: dict,\n",
    "                      *,\n",
    "                      questionDetailsDict: dict = retrieveQuestionDetails(),\n",
    "                      export: bool = False) -> str :\n",
    "    \n",
    "    if questionNo in questionData :\n",
    "        questionData = questionData[questionNo]\n",
    "\n",
    "    title = questionData[\"title\"]\n",
    "\n",
    "    # Only if title has already been modified and matched to a LeetCode url\n",
    "    # E.g. some contest files will be unmatched\n",
    "    if '[' in questionData[\"title\"] :\n",
    "        title = title[title.find('[') + 1:title.find(']')]\n",
    "\n",
    "    title = f'{questionNo}. {title}'\n",
    "    \n",
    "    generate_file_name = f'_{title}.md'\n",
    "    generate_path = join(README_PATH, MARKDOWN_PATH, generate_file_name)\n",
    "    output_path = join(MARKDOWN_PATH, generate_file_name)\n",
    "    \n",
    "    questionData['solution'] = generate_file_name\n",
    "\n",
    "    if not export :\n",
    "        return generate_path\n",
    "\n",
    "    with open(generate_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# {questionNo}. {questionData[\"title\"]}\\n\\n')\n",
    "\n",
    "        date_done = questionData['date_done']\n",
    "        date_modified = questionData['date_modified']\n",
    "        \n",
    "        f.write(f'*All prompts are owned by LeetCode. To view the prompt, click the title link above.*\\n\\n')\n",
    "        \n",
    "        if questionData['contestTitle'] and questionData['contestQNo']:\n",
    "            f.write(f'*Completed during {questionData[\"contestTitle\"]} ({questionData[\"contestQNo\"]})*\\n\\n')\n",
    "\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(f'> *First completed : {date_done:%B %d, %Y}*\\n>\\n')\n",
    "        f.write(f'> *Last updated : {date_modified:%B %d, %Y}*\\n')\n",
    "\n",
    "        f.write('\\n------\\n\\n')\n",
    "\n",
    "        tpcs = 'N/A' if questionNo not in questionDetailsDict or len(questionDetailsDict[questionNo].topics) == 0 \\\n",
    "                     else ', '.join([f'[{x}](<{join(BY_TOPIC_FOLDER_PATH, x)}.md>)' for x in questionDetailsDict[questionNo].topics])\n",
    "        \n",
    "        f.write(f'> **Related Topics** : **{tpcs}**\\n>\\n')\n",
    "\n",
    "        acrate = 'Unknown' if questionNo not in questionDetailsDict else f'{round(questionDetailsDict[questionNo].acRate, 2)} %'\n",
    "        f.write(f'> **Acceptance Rate** : **{acrate}**\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "\n",
    "        if 'contextFile' in questionData:\n",
    "            with open(join(README_PATH, questionData['contextFile']), 'r') as contextFile:\n",
    "                f.write('> ' + contextFile.read().replace('\\n', '\\n> '))\n",
    "            f.write(f'\\n\\n------\\n\\n')\n",
    "        \n",
    "\n",
    "        f.write(f'## Solutions\\n\\n')\n",
    "        for lang, solutions in questionData['solutions'].items() :\n",
    "            solutions.sort()\n",
    "            for solution in solutions :\n",
    "                name = solution[solution.find('/') + 1:]\n",
    "                f.write(f'- [{name}](<{join(README_PATH, solution)}>)\\n')\n",
    "\n",
    "        for lang, solutions in questionData['solutions'].items() :\n",
    "            if lang.lower() in LANGUAGE_EQUIVS :\n",
    "                lang = LANGUAGE_EQUIVS[lang.lower()]\n",
    "            else :\n",
    "                print()\n",
    "                print(f'Lang equiv not found: {lang = }')\n",
    "            f.write(f'### {lang}\\n')\n",
    "            for solution in solutions :\n",
    "                name = solution.rfind('/') + 1\n",
    "                f.write(f'#### [{solution[name:]}](<{join(README_PATH, solution)}>)\\n')\n",
    "                f.write(f'```{lang}\\n')\n",
    "                with open(join(README_PATH, solution), 'r', encoding='utf-8') as solutionFile:\n",
    "                    fileData = solutionFile.read()\n",
    "                    if '# @lc code=start' in fileData :\n",
    "                        lcStart = '# @lc code=start'\n",
    "                        lcEnd   = '# @lc code=end'\n",
    "                        fileData = fileData[fileData.find(lcStart) + len(lcStart):fileData.rfind(lcEnd)]\n",
    "                    f.write(fileData)\n",
    "                f.write('\\n```\\n\\n')\n",
    "\n",
    "    return output_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processMarkdownGeneration(questionData: dict,\n",
    "                              reprocessMarkdown: Set[int],\n",
    "                              *,\n",
    "                              questionDetailsDict: dict = retrieveQuestionDetails()) -> None :\n",
    "    \n",
    "    # Create a folder to avoid errors if it doesn't already exist\n",
    "    markdownFolder = join(README_PATH, MARKDOWN_PATH)\n",
    "    if not isdir(markdownFolder) :\n",
    "        mkdir(markdownFolder)\n",
    "\n",
    "    for questionNo, dta in questionData.items() :\n",
    "        if questionNo in reprocessMarkdown :\n",
    "            generate_markdown(questionNo, \n",
    "                              questionData, \n",
    "                              questionDetailsDict=questionDetailsDict, \n",
    "                              export=True)\n",
    "        else : # In order to assign the markdown paths\n",
    "            generate_markdown(questionNo, \n",
    "                              questionData, \n",
    "                              questionDetailsDict=questionDetailsDict, \n",
    "                              export=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataFrames\n",
    "Conversion into DataFrames and declaration of respective column headers occurs here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "COLUMNS = [ \n",
    "            '#',\n",
    "            'Title', \n",
    "            'Level',\n",
    "            'Cats',\n",
    "            'Solution',\n",
    "            'Languages',\n",
    "            'Date Complete'\n",
    "          ]\n",
    "\n",
    "TYPE_CLARIFICATION = {\n",
    "                      '#':                  int,\n",
    "                      'Title':              str, \n",
    "                      'Level':              str,\n",
    "                      'Cats':               str,\n",
    "                      'Solution':           str,\n",
    "                      'Languages':          str,\n",
    "                      'Date Complete':      str\n",
    "                    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertDataToMatrix(questionData: dict,\n",
    "                        *,\n",
    "                        sortBy:       str = 'number',\n",
    "                        includeDate:  bool = False,\n",
    "                        includeQuestions: set[int] = set(),\n",
    "                        relativeFolderAdjustment: int = 0,\n",
    "                        includeMarkdownFolder: bool = False) -> List[list] :\n",
    "    dataframe_array = []\n",
    "\n",
    "    for question in questionData.values() :\n",
    "        # If it's not an empty set and the value isn't in there, skip\n",
    "        if includeQuestions and question['number'] not in includeQuestions :\n",
    "            continue\n",
    "\n",
    "        if sortBy == 'number' and includeMarkdownFolder :\n",
    "            solution_path = join(MARKDOWN_PATH, question['solution'])    \n",
    "        else :\n",
    "            solution_path = question['solution']\n",
    "        \n",
    "        solution_path = '../' * abs(relativeFolderAdjustment) + solution_path\n",
    "\n",
    "        title_to_use = question['title']\n",
    "        \n",
    "        if question['contestTitle'] and question['contestQNo'] :\n",
    "            title_to_use = f'{question[\"contestTitle\"]} - {question[\"contestQNo\"]} - {title_to_use}'\n",
    "\n",
    "        currentRow = [question['number'],\n",
    "                      title_to_use, \n",
    "                      question['level'], \n",
    "                      ', '.join(sorted(list(question['categories']))), \n",
    "                      f'[solution](<{solution_path}>)', \n",
    "                      ', '.join(sorted(list(question['languages'])))]\n",
    "        \n",
    "        if includeDate :\n",
    "            currentRow.append(question['date_done'].strftime('%b %d, %Y'))\n",
    "        \n",
    "        dataframe_array.append(currentRow)\n",
    "\n",
    "    # Usually, x[0] is the question number but the qustionData set might be keyed by the date if it's a daily case\n",
    "    if sortBy == 'date_done' and type(list(questionData.keys())[0]) == datetime :\n",
    "        dataframe_array.sort(key=lambda x: datetime.strptime(x[-1], '%b %d, %Y'))\n",
    "    else :\n",
    "        dataframe_array.sort(key=lambda x: questionData.get(x[0])[sortBy])\n",
    "\n",
    "    return dataframe_array\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertQuestionDataToDataframe(questionData: dict,\n",
    "                                   *,\n",
    "                                   sortBy: str = 'number',\n",
    "                                   includeDate:  bool = False,\n",
    "                                   includeQuestions: set[int] = set(),\n",
    "                                   relativeFolderAdjustment: int = 0,\n",
    "                                   includeMarkdownFolder: bool = False) -> DataFrame :\n",
    "    \n",
    "    questionData = convertDataToMatrix(questionData, \n",
    "                                       sortBy=sortBy, \n",
    "                                       includeDate=includeDate, \n",
    "                                       includeQuestions=includeQuestions,\n",
    "                                       relativeFolderAdjustment=relativeFolderAdjustment,\n",
    "                                       includeMarkdownFolder=includeMarkdownFolder)\n",
    "    \n",
    "    # Protects against empty cases (e.g. if you have no daily files)\n",
    "    dfQuestions = kungfupanda.DataFrame()\n",
    "    if questionData :\n",
    "        dfQuestions = kungfupanda.DataFrame(data=questionData, columns=COLUMNS[:len(questionData[0])])\n",
    "    \n",
    "    # dfQuestions   = dfQuestions.astype(TYPE_CLARIFICATION[:len(questionData[0])])\n",
    "\n",
    "    return dfQuestions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# List & Other Markdowns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sorted by Most Recent\n",
    "Using creation dates of code files only; not modification dates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: Reversed due to default sorting being in ascending order\n",
    "def byRecentQuestionDataDataframe(questionData: dict) -> DataFrame :\n",
    "    return convertQuestionDataToDataframe(questionData, sortBy='date_done', includeDate=True).iloc[::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sorted by Amount of Code\n",
    "Questions with more files on the question and longer submissions will be prioritized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def byCodeLengthDataDataframe(questionData: dict) -> DataFrame :\n",
    "    return convertQuestionDataToDataframe(questionData, sortBy='bytes', includeDate=True).iloc[::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generation of Markdowns for Each Related Topic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def questionTopicDataframes(questionData: dict,\n",
    "                            *,\n",
    "                            topicGroupings: defaultdict) -> List[Tuple[str, int, DataFrame]] : # [topic, number of questions, dataframe]\n",
    "    if not topicGroupings :\n",
    "        topicGroupings = getCompletedQuestionsTopicLists(questionData)\n",
    "    \n",
    "    output = []\n",
    "    for topic, qs in topicGroupings.items() :\n",
    "        output.append((topic, \n",
    "                       len(qs), \n",
    "                       convertQuestionDataToDataframe(questionData,\n",
    "                                                      includeDate=True,\n",
    "                                                      includeQuestions=qs,\n",
    "                                                      relativeFolderAdjustment= \\\n",
    "                                                        -getenv('TOPIC_MARKDOWN_PATH_IN_MARKDOWNS_FOLDER').count('/'))))\n",
    "        \n",
    "    output.sort(key=lambda x: x[1], reverse=True)\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Topic based markdown generation and any of the large list markdowns in general sometimes suddenly show massive edits for what should be a regular usual question update. This is likely due to the dataframe.to_markdown method increasing the width of the table due to a larger than before seen input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOPIC_FOLDER = getenv('TOPIC_MARKDOWN_PATH_IN_MARKDOWNS_FOLDER')\n",
    "\n",
    "# For each topic case\n",
    "NOTEBOOK_PATH = join(README_PATH, MARKDOWN_PATH, TOPIC_FOLDER)\n",
    "\n",
    "# For the overal hosting markdown\n",
    "OVERALL_FILE_NOTEBOOK_PATH = join(README_PATH, MARKDOWN_PATH, 'Topics.md')\n",
    "OVERALL_FILE_README_PATH   = join(MARKDOWN_PATH, 'Topics.md')\n",
    "\n",
    "def topicBasedMarkdowns(questionData: dict,\n",
    "                         *,\n",
    "                         topicGroupings: defaultdict) -> List[Tuple[str, str]] :    # path of all outputs\n",
    "                                                                                    # list[0]  = overall mardown\n",
    "                                                                                    # list[1:] = order of count\n",
    "                                                                                    # doesn't include markdown 'markdown/'\n",
    "    if not topicGroupings :\n",
    "        topicGroupings = getCompletedQuestionsTopicLists(questionData)\n",
    "\n",
    "    topicDataframes = questionTopicDataframes(questionData=questionData, topicGroupings=topicGroupings)\n",
    "\n",
    "    if not isdir(NOTEBOOK_PATH) :\n",
    "        mkdir(NOTEBOOK_PATH)\n",
    "\n",
    "    output = [OVERALL_FILE_README_PATH]\n",
    "    with open(OVERALL_FILE_NOTEBOOK_PATH, 'w', encoding='utf-8') as topic_file :\n",
    "        topic_file.write('# Topics\\n\\n')\n",
    "        topic_file.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        topic_file.write('------\\n\\n')\n",
    "\n",
    "        for topic, cnt, df in topicDataframes :\n",
    "            file_name = f'{topic}.md'\n",
    "            notebook_path = join(NOTEBOOK_PATH, file_name)\n",
    "            readme_path = join(TOPIC_FOLDER, file_name)\n",
    "            with open(notebook_path, 'w', encoding='utf-8') as f :\n",
    "                url = f'https://leetcode.com/tag/{topic.replace(\" \", \"-\")}/'\n",
    "                f.write(f'# [{topic}](<{url}>) ({cnt} completed)\\n\\n')\n",
    "                f.write(f'*[Back to top](<../../README.md>)*\\n\\n')\n",
    "                f.write('------\\n\\n')\n",
    "                f.write(df.to_markdown(index=False))\n",
    "        \n",
    "            topic_file.write(f'- [{topic}](<{readme_path}>) ({cnt} completed)\\n')\n",
    "            output.append((topic, readme_path))\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Markdowns for Easy/Medium/Hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIFFICULTY_MARKDOWNS_PATH = MARKDOWN_PATH\n",
    "\n",
    "def generateDifficultyLevelMarkdowns(questionData: dict) -> Tuple[Tuple[int, str], Tuple[int, str], Tuple[int, str]] :\n",
    "    '''\n",
    "    ### Returns:\n",
    "    - Tuple[Easy, Medium, Hard]\n",
    "        - Tuple[int, str] : (count, path from readme)\n",
    "    '''\n",
    "    \n",
    "    easyQuestions = {}\n",
    "    mediumQuestions = {}\n",
    "    hardQuestions = {}\n",
    "    \n",
    "    for q, d in questionData.items() :\n",
    "        lvlIndicator = d['level'][0].lower()\n",
    "        match lvlIndicator :\n",
    "            case 'e' :\n",
    "                easyQuestions[q] = d\n",
    "            case 'm' :\n",
    "                mediumQuestions[q] = d\n",
    "            case 'h' :\n",
    "                hardQuestions[q] = d\n",
    "            case _ :\n",
    "                print(f'Error identifying level of {q = }')\n",
    "    \n",
    "    easyMarkdown    = convertQuestionDataToDataframe(easyQuestions, \n",
    "                                                     includeDate=True, \n",
    "                                                     includeMarkdownFolder=False)\n",
    "    mediumMarkdown  = convertQuestionDataToDataframe(mediumQuestions, \n",
    "                                                     includeDate=True, \n",
    "                                                     includeMarkdownFolder=False)\n",
    "    hardMarkdown    = convertQuestionDataToDataframe(hardQuestions, \n",
    "                                                     includeDate=True, \n",
    "                                                     includeMarkdownFolder=False)\n",
    "    \n",
    "    \n",
    "    easy_path   = join(DIFFICULTY_MARKDOWNS_PATH, 'Easy.md')\n",
    "    medium_path = join(DIFFICULTY_MARKDOWNS_PATH, 'Medium.md')\n",
    "    hard_path   = join(DIFFICULTY_MARKDOWNS_PATH, 'Hard.md')\n",
    "    \n",
    "    with open('../' + easy_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# Easy Questions ({len(easyQuestions)})\\n\\n')\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(easyMarkdown.to_markdown(index=False))\n",
    "        \n",
    "    with open('../' + medium_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# Medium Questions ({len(mediumQuestions)})\\n\\n')\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(mediumMarkdown.to_markdown(index=False))\n",
    "        \n",
    "    with open('../' + hard_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(f'# Hard Questions ({len(hardQuestions)})\\n\\n')\n",
    "        f.write('*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write('------\\n\\n')\n",
    "        f.write(hardMarkdown.to_markdown(index=False))\n",
    "    \n",
    "    return ((len(easyQuestions),    easy_path),\n",
    "            (len(mediumQuestions),  medium_path),\n",
    "            (len(hardQuestions),    hard_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dailies, Recents, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAILY_URL = ''\n",
    "\n",
    "def miscMarkdownGenerations(questionData:   dict,\n",
    "                            *,\n",
    "                            code_length:    bool = False,\n",
    "                            recent:         bool = False,\n",
    "                            daily:          bool = False,\n",
    "                            weekly:         bool = False) -> str : # output path\n",
    "    \n",
    "    df = None\n",
    "    fileName = None\n",
    "    header_data = None\n",
    "    details = None\n",
    "\n",
    "    # print(f'{code_length, recent = }')\n",
    "    if code_length :\n",
    "        df = byCodeLengthDataDataframe(questionData)\n",
    "        fileName    = 'Questions_By_Code_Length.md'\n",
    "        header_data = '# Questions By Code Length\\n\\n'\n",
    "        details     = 'Calculations are based on the code files\\'s byte sizes.\\n\\n'\n",
    "    elif recent :\n",
    "        df = byRecentQuestionDataDataframe(questionData)\n",
    "        fileName    = 'Questions_By_Recent.md'\n",
    "        header_data = '# Most Recently Solved Questions\\n\\n'\n",
    "        details     = 'Calculations are based on the date of the first solve.\\n\\n'\n",
    "    elif daily :\n",
    "        dailyQuestionData = parseQuestionsForDailies(questionData)\n",
    "        global _oldest_date\n",
    "        print(f'{dailyQuestionData = }')\n",
    "        print(f'{_oldest_date = }')\n",
    "        # for qNo, qData in questionData.items() :\n",
    "        #     if 'Daily' in qData['categories'] :\n",
    "        #         dailyQuestionData[qNo] = qData\n",
    "        df = byRecentQuestionDataDataframe(dailyQuestionData)\n",
    "        fileName    = 'Daily_Questions.md'\n",
    "        # header_data = f'# [Daily Questions](<{DAILY_URL}>)\\n\\n'\n",
    "        header_data = f'# Daily Questions\\n\\n'\n",
    "        details     = 'Dates are for the date I completed the ' + \\\n",
    "                      'question so due to the my time zone and how it lines up with ' + \\\n",
    "                      'UTC, it may be off by a day.\\n\\n'\n",
    "    elif weekly :\n",
    "        weeklyQuestionData = parseQuestionsForWeeklies(questionData)\n",
    "    \n",
    "        df = byRecentQuestionDataDataframe(weeklyQuestionData)\n",
    "        fileName    = 'Weekly_Questions.md'\n",
    "        # header_data = f'# [Daily Questions](<{DAILY_URL}>)\\n\\n'\n",
    "        header_data = f'# Weekly Premium Questions\\n\\n'\n",
    "        details     = 'Dates are for the date I completed the ' + \\\n",
    "                      'question so due to the my time zone and how it lines up with ' + \\\n",
    "                      'UTC, it may be off by a day.\\n\\n'\n",
    "    else :\n",
    "        print('Error. No markdown generation specified.')\n",
    "        print()\n",
    "        return ''\n",
    "\n",
    "    # print(f'{fileName = }')\n",
    "\n",
    "    output_path = join(MARKDOWN_PATH, fileName)\n",
    "    readme_path = join(README_PATH, MARKDOWN_PATH, fileName)\n",
    "\n",
    "    with open(readme_path, 'w', encoding='utf-8') as f :\n",
    "        f.write(header_data)\n",
    "        f.write(f'*[Back to top](<../README.md>)*\\n\\n')\n",
    "        f.write(details)\n",
    "        f.write(df.to_markdown(index=False))\n",
    "\n",
    "    return output_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Outputing to README File\n",
    "Takes all the above and overwrites the current [README.md](README.md) file with the data calculated above.\n",
    "\n",
    "Inputs values in order of:\n",
    "- Profile link\n",
    "- Stats\n",
    "- Stat clarification\n",
    "- Question link tables Easy-Medium-Hard\n",
    "\n",
    "Uses the built-in DataFrame `.to_markdown()` for outputting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exportPrimaryReadme(dfQuestions:        DataFrame,\n",
    "                        *,\n",
    "                        difficultyBasedMarkdowns: Tuple[Tuple[int, str], \n",
    "                                                        Tuple[int, str], \n",
    "                                                        Tuple[int, str]] = None,\n",
    "                        additionalSorts:    List[str] = [],\n",
    "                        topicLinks:         List[Tuple[str, str]] = []) -> None :\n",
    "    readmePath = join(README_PATH, 'README.md')\n",
    "    print(readmePath)\n",
    "\n",
    "    # No. Questions Solved\n",
    "    qSolvedHeader = f'{len(dfQuestions.index)} solved'\n",
    "    \n",
    "    print(difficultyBasedMarkdowns)\n",
    "    # if difficultyBasedMarkdowns :\n",
    "    #     qSolvedHeader += f' - [{difficultyBasedMarkdowns[0][0]}e](<{difficultyBasedMarkdowns[0][1]}>), ' + \\\n",
    "    #                         f'[{difficultyBasedMarkdowns[1][0]}m](<{difficultyBasedMarkdowns[1][1]}>), ' + \\\n",
    "    #                         f'[{difficultyBasedMarkdowns[2][0]}h](<{difficultyBasedMarkdowns[2][1]}>)'\n",
    "    \n",
    "    with open(readmePath, 'w') as file :\n",
    "        username = getenv('LEETCODE_USERNAME')\n",
    "        file.write(f'# **[LeetCode Records](https://leetcode.com/u/{username}/)** ({qSolvedHeader})\\n\\n')\n",
    "        file.write(f'<!-- This readme was generated using [WikiLeet](<https://github.com/Zanger67/WikiLeet>) -->\\n\\n')\n",
    "        file.write(f'> My LeetCode Profile: [{username}](https://leetcode.com/u/{username}/)\\n')\n",
    "        \n",
    "        # if difficultyBasedMarkdowns :\n",
    "        #     file.write(f'> [{difficultyBasedMarkdowns[0][0]} easy](<{difficultyBasedMarkdowns[0][1]}>), ' + \\\n",
    "        #                f'[{difficultyBasedMarkdowns[1][0]} medium](<{difficultyBasedMarkdowns[1][1]}>), ' + \\\n",
    "        #                f'[{difficultyBasedMarkdowns[2][0]} hard](<{difficultyBasedMarkdowns[2][1]}>)')\n",
    "        \n",
    "        file.write('\\n\\n')\n",
    "\n",
    "        file.write('## About this Repo\\n\\n')\n",
    "        file.write('This repo is a collection of my LeetCode solutions, primarily written in Python, Java, and C. ' + \n",
    "                   'On any page, `click the main title` to be redirected to the official `LeetCode` page for the ' + \n",
    "                   'question, topic, list, etc. See the `Additional Categories` section for pages that group' + \n",
    "                   ' questions by different criteria -- e.g. by their *related topics*.')\n",
    "        file.write('\\n\\n\\n')\n",
    "\n",
    "        file.write('------\\n\\n')\n",
    "\n",
    "        file.write('## Category Notes\\n')\n",
    "        file.write('1. **Daily** - Daily challenge questions that were done on the day of\\n')\n",
    "        file.write('2. **Weekly Premium** - Weekly premium questions that were done on week of\\n')\n",
    "        file.write('3. **Contest** - Questions that I completed during a live contest\\n')\n",
    "        # file.write('4. **Favourite** - Questions that I liked and wanted to keep a record of\\n')\n",
    "        file.write('\\n\\n')\n",
    "\n",
    "        file.write('------\\n\\n')\n",
    "\n",
    "        file.write('## Additional Categories Stats\\n')\n",
    "\n",
    "        for altSorts in additionalSorts :\n",
    "            file.write(altSorts)\n",
    "            file.write('\\n\\n')\n",
    "        \n",
    "        if topicLinks :\n",
    "            file.write('------\\n\\n')\n",
    "            file.write(', '.join([f'[{topic}](<{join(MARKDOWN_PATH, link)}>)' for topic, link in topicLinks[1:]]))       \n",
    "            file.write('\\n\\n')\n",
    "            file.write('------\\n\\n')\n",
    "        \n",
    "        file.write('\\n\\n')\n",
    "\n",
    "        file.write('## Questions\\n')\n",
    "        file.write(dfQuestions.to_markdown(index=False))\n",
    "        \n",
    "        \n",
    "        file.write('\\n\\n')\n",
    "        file.write('<p align=\"right\"><i>This README was generated using <a href=\"https://github.com/Zanger67/WikiLeet\">WikiLeet</a></i></p>\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recalculateAll: forces recalcualtion markdowns for each question irregardless if its\n",
    "#                 source files have been modified or not\n",
    "def main(*, recalculateAll: bool = False, noRecord: bool = False) -> None :\n",
    "    leetcodeFiles           = getCodeFiles()\n",
    "    additionalInfoFiles     = getContextFiles()     # For later use when generating the individual readme files\n",
    "\n",
    "    contestFolders          = getContestFolders()\n",
    "    contestLeetcodeFiles    = getContestFiles(contestFolders)\n",
    "\n",
    "    if USE_GIT_DATES :\n",
    "        getAllCTimesViaGit(additionalInfoFiles \n",
    "                           + leetcodeFiles \n",
    "                           + [join(x[0], x[1]) for x in contestLeetcodeFiles])\n",
    "\n",
    "\n",
    "    questionDetailsDict     = retrieveQuestionDetails()\n",
    "    # print(questionDetailsDict)\n",
    "\n",
    "    leetcodeFiles.sort()\n",
    "    contestLeetcodeFiles.sort()\n",
    "\n",
    "\n",
    "    # Files for leetcode questions found\n",
    "    print(f'Total of {len(leetcodeFiles)} files found.')\n",
    "\n",
    "    # Files in contest folders found\n",
    "    print(f'Total of {len(contestLeetcodeFiles)} contest files found.')\n",
    "\n",
    "\n",
    "    # Parsing primary files\n",
    "    fileLatestTimes = getRecentFileTimes() if (not recalculateAll and not noRecord) else {}\n",
    "\n",
    "    reprocessMarkdown = set()\n",
    "    questionData = {}\n",
    "\n",
    "    # Parsing primary files\n",
    "    print('Parsing code files...')\n",
    "    for leetcodeFile in leetcodeFiles :\n",
    "        parseCase(leetcodeFile=leetcodeFile,\n",
    "                  questionData=questionData,\n",
    "                  fileLatestTimes=fileLatestTimes, \n",
    "                  reprocessMarkdown=reprocessMarkdown,\n",
    "                  questionDetailsDict=questionDetailsDict)\n",
    "        \n",
    "    # Parsing contest files & folforders\n",
    "    print('Parsing contest files...')\n",
    "    for leetcodeContestFile in contestLeetcodeFiles :\n",
    "        contestFolder, leetcodeFile = leetcodeContestFile\n",
    "        parseCase(leetcodeFile=leetcodeFile,\n",
    "                  questionData=questionData,\n",
    "                  fileLatestTimes=fileLatestTimes,\n",
    "                  reprocessMarkdown=reprocessMarkdown, \n",
    "                  subFolderPath=contestFolder, \n",
    "                  questionDetailsDict=questionDetailsDict,\n",
    "                  contest=contestFolder)\n",
    "        \n",
    "\n",
    "    # Parsing additional information files\n",
    "    print('Parsing additional information/context files...')\n",
    "    parseContextFiles(txtFiles=additionalInfoFiles, \n",
    "                      questionData=questionData,\n",
    "                      fileLatestTimes=fileLatestTimes,\n",
    "                      reprocessMarkdown=reprocessMarkdown)\n",
    "    \n",
    "    # Identifying members of lists\n",
    "    print('Sorting questions to their lists...')\n",
    "    processListData(questionData=questionData)\n",
    "\n",
    "    # Generating markdowns for each individual question\n",
    "    print('Generating markdowns for each individual question...')\n",
    "    processMarkdownGeneration(questionData=questionData, \n",
    "                              reprocessMarkdown=reprocessMarkdown, \n",
    "                              questionDetailsDict=questionDetailsDict)\n",
    "\n",
    "    \n",
    "    # Produces a markdown where questions are sorted by the amount of code\n",
    "    # written for the question\n",
    "    # code_length_md_path = exportCodeLengthMarkdown(questionData)\n",
    "    print('Generating category lists...')\n",
    "    dailyQuestions      = miscMarkdownGenerations(questionData, daily=True)\n",
    "    weeklyQuestions     = miscMarkdownGenerations(questionData, weekly=True)\n",
    "    byCodeLength        = miscMarkdownGenerations(questionData, code_length=True)\n",
    "    byRecentlySolved    = miscMarkdownGenerations(questionData, recent=True)\n",
    "    altSorts            = [f'- [Daily Questions](<{dailyQuestions}>)',\n",
    "                           f'- [Weekly Questions](<{weeklyQuestions}>)',\n",
    "                           f'- [Questions By Code Length](<{byCodeLength}>)',\n",
    "                           f'- [Questions By Recent](<{byRecentlySolved}>)']\n",
    "    \n",
    "\n",
    "    difficultyBasedMarkdowns = generateDifficultyLevelMarkdowns(questionData)\n",
    "    \n",
    "\n",
    "    completedQsTopicGroupings = getCompletedQuestionsTopicLists(questionData)\n",
    "    topicMarkdownLinks = topicBasedMarkdowns(questionData, topicGroupings=completedQsTopicGroupings)\n",
    "    altSorts.append(f'- [Grouped by Topic](<{topicMarkdownLinks[0]}>)')\n",
    "\n",
    "\n",
    "    # Exporting the primary README.md file\n",
    "    print('Exporting primary README.md file...')\n",
    "    dfQuestions = convertQuestionDataToDataframe(questionData, \n",
    "                                                 includeDate=False, \n",
    "                                                 includeMarkdownFolder=True)\n",
    "    exportPrimaryReadme(dfQuestions, \n",
    "                        additionalSorts=altSorts, \n",
    "                        topicLinks=topicMarkdownLinks, \n",
    "                        difficultyBasedMarkdowns=difficultyBasedMarkdowns)\n",
    "\n",
    "\n",
    "    print(f'Number of individual questions updated/added: {len(reprocessMarkdown)}')\n",
    "\n",
    "\n",
    "    if not noRecord :\n",
    "        print('Pickling most recent modification times for future reference...')\n",
    "        writeRecentFileTimes(fileLatestTimes)           # restore for next use\n",
    "\n",
    "\n",
    "    print('All processes complete. Exiting...')\n",
    "    return questionData, reprocessMarkdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__' :\n",
    "    '''\n",
    "    ### Flags\n",
    "    `-r` : \n",
    "        Recalculate all markdown files irregardless of whether there are modified or new code files for that question or not\n",
    "    `-n` :\n",
    "        Don't use the previous modified dates and don't store them (in effect, the same as `-r` but it doesn't save the \n",
    "        new modification dates). Primarily for use with GitHub actions.\n",
    "    `-g` :\n",
    "        Uses the repository's git log history for each file to trace the creation and last modification dates of each file \n",
    "        rather than use the default `getctime()` and `getmtime()` of each file. GitHub actions seem to default the ctime \n",
    "        and mtimes to time.now due to not tracking the actual mtime ctime metadata.\n",
    "\n",
    "        WARNING: Only for use with GitHub actions as this ends up being very slow due to low subprocess speends.\n",
    "    '''\n",
    "    recalcaulateAll = False\n",
    "    noRecord = False\n",
    "\n",
    "    if not IS_NOTEBOOK :\n",
    "        parser = argparse.ArgumentParser()\n",
    "\n",
    "        parser.add_argument(\"-r\", \n",
    "                            help=\"Recompile all markdown files\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "        \n",
    "        parser.add_argument(\"-n\", \n",
    "                            help=\"Don't use the previous modified dates and don't store them\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "        parser.add_argument(\"-norecord\", \n",
    "                            help=\"Don't use the previous modified dates and don't store them\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "        parser.add_argument(\"-g\", \n",
    "                            help=\"Use Git repo's dates for determining if a file has been modified and created (WARNING SLOW)\", \n",
    "                            required=False, \n",
    "                            action=argparse.BooleanOptionalAction)\n",
    "\n",
    "        parser.add_argument('-user', type=str, default='', required=False, help='LeetCode Username')\n",
    "        if parser.parse_args().user :\n",
    "            environ['LEETCODE_USERNAME'] = parser.parse_args().user\n",
    "        \n",
    "        \n",
    "        recalcaulateAll = parser.parse_args().r\n",
    "        noRecord = parser.parse_args().norecord or parser.parse_args().n\n",
    "\n",
    "        USE_GIT_DATES = parser.parse_args().g\n",
    "        \n",
    "        \n",
    "\n",
    "    # README_ABS_DIR = README_ABS_DIR[:README_ABS_DIR.rindex('/')]\n",
    "    # print(README_ABS_DIR, '\\n')\n",
    "\n",
    "    print('No record'.ljust(20), 'on' if noRecord else 'off')\n",
    "    print('Recalculate'.ljust(20), 'on' if recalcaulateAll else 'off')\n",
    "    print('Use Git dates'.ljust(20), 'on' if USE_GIT_DATES else 'off')\n",
    "    print()\n",
    "\n",
    "\n",
    "    main(recalculateAll=recalcaulateAll, noRecord=noRecord)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}