92_split_paragraphs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DRS document\n",
    "Defining the functions first and then applying that on a selection of suitable document types (ACs and Orders like)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "def get_bookmarks(outlines, pdf, parent_position_in_bookmark_tree):\n",
    "\n",
    "    def get_bookmark_content(outline, pdf):\n",
    "        if pdf.get_destination_page_number(outline)>=0:\n",
    "            try:\n",
    "                repattern_title_number_ac = 'AC 2[13759]'+\\\n",
    "                                            '('+\\\n",
    "                                                '\\.[0-9]{1,5}[A-Z]?\\.?[\\s]{1,3}§'+\\\n",
    "                                                    '|'+\\\n",
    "                                                '(?= MG [0-9]{1,2}\\.?)'+\\\n",
    "                                            ')' \n",
    "                repattern_title_text_ac = '('+\\\n",
    "                                            '2[13759]\\.[0-9]{1,5}\\.?'+\\\n",
    "                                            ' (\\(A[A-Za-z 0-9-]*\\))?' +\\\n",
    "                                                    '|'+\\\n",
    "                                            'MG [0-9]{1,2}\\.?'+\\\n",
    "                                            ')'      \n",
    "                                                                                                                         \n",
    "                repattern_title_prefix = '(VOLUME|volume|Volume|Section|SECTION|section|Section|PART|SUBPART|CHAPTER|chapter|Chapter|APPENDIX|Appendix|appendix)+[\\s]{1,2}'\n",
    "                repattern_greek = '[ivx]{2,3}|[IVX]{2,3}'\n",
    "                repattern_title_prevent_firstletter_mismatch = '(?!([A-Z]{2,10}|[a-z]{2,10}))'    \n",
    "                repattern_title_number_basic = '[\\(]?('+repattern_greek+'|[1-9][0-9]{0,3}|[A-Za-z])'+repattern_title_prevent_firstletter_mismatch\n",
    "                repattern_title_delimiter = '(-|\\)|\\.)'\n",
    "\n",
    "                repattern_title_number ='('+\\\n",
    "                                            repattern_title_number_ac+\\\n",
    "                                        '|'+\\\n",
    "                                            '('+repattern_title_prefix+')?'+\\\n",
    "                                            '('+\\\n",
    "                                                repattern_title_delimiter.join([repattern_title_number_basic, \n",
    "                                                                                repattern_title_number_basic, \n",
    "                                                                                '[ ]?'+repattern_title_number_basic, \n",
    "                                                                                repattern_title_number_basic])+\\\n",
    "                                            '|'+\\\n",
    "                                                repattern_title_delimiter.join([repattern_title_number_basic, \n",
    "                                                                                repattern_title_number_basic, \n",
    "                                                                                '[ ]?'+repattern_title_number_basic])+\\\n",
    "                                            '|'+\\\n",
    "                                                repattern_title_delimiter.join([repattern_title_number_basic, \n",
    "                                                                                repattern_title_number_basic])+\\\n",
    "                                            '|'+\\\n",
    "                                                repattern_title_delimiter.join([repattern_title_number_basic])+\\\n",
    "                                            ')'+\\\n",
    "                                        ')'+\\\n",
    "                                        '[\\s]{0,2}'+repattern_title_delimiter+'?'+'[\\s]{0,2}'\n",
    "                \n",
    "                repattern_before_title = '(?<=( |\\n))'\n",
    "                \n",
    "                for offset in [0,-1,-2,1,2]:                \n",
    "                    line_with_title = None\n",
    "                    nb_keywords = 5\n",
    "                    while line_with_title is None and nb_keywords>0:\n",
    "                        outline_title_cleaned = re.sub('[^a-zA-Z ,/\\d\\s\\.:-]', '.', outline.title)\n",
    "\n",
    "                        title_shortened_variable_spaces = re.sub('[ ]+', ' ', outline_title_cleaned).split(' ')[:nb_keywords]\n",
    "                        title_shortened_variable_spaces = title_shortened_variable_spaces[:2]+\\\n",
    "                                                        [keyword[:2]+'-?'.join(keyword[2:]) for keyword in title_shortened_variable_spaces[2:]]\n",
    "                        title_shortened_variable_spaces = '[ ]*'.join(title_shortened_variable_spaces)\n",
    "\n",
    "                        repattern_title_line = '('+repattern_before_title+repattern_title_number+')?'+\\\n",
    "                                                title_shortened_variable_spaces+\\\n",
    "                                                '.*(\\n|\\r|$)'\n",
    "                        try:\n",
    "                            line_with_title = re.search(repattern_title_line,\n",
    "                                                        pdf.pages[max(0,pdf.get_destination_page_number(outline)+offset)].extract_text())\n",
    "                        except:\n",
    "                            pass                        \n",
    "\n",
    "                        if line_with_title is not None:\n",
    "                            break\n",
    "                        else:\n",
    "                            nb_keywords = nb_keywords - 1\n",
    "\n",
    "                    if line_with_title is not None:\n",
    "                        break\n",
    "                    \n",
    "                sentence_around_title = pdf.pages[max(0,\n",
    "                                                      pdf.get_destination_page_number(outline)+offset)].extract_text()[line_with_title.span()[0]:\\\n",
    "                                                                                                                        max(line_with_title.span()[1],\\\n",
    "                                                                                                                            line_with_title.span()[0]+len(outline.title))]\n",
    "                title_number = re.search(repattern_title_number, sentence_around_title)\n",
    "\n",
    "                title_max_length = 200\n",
    "                title_text_end_delimiter = '(\\.|\\:|\\?|\\n|\\r|$)?'\n",
    "                repattern_title_text = '('+repattern_title_text_ac+')?'+\\\n",
    "                                        '[A-Za-z,\\/\\-\\s0-1\\(\\)\\[\\]]'+\\\n",
    "                                        '{3,'+str(title_max_length)+'}'+title_text_end_delimiter# [\\s]{1,3}\n",
    "                if title_number is not None:\n",
    "                    title_text = re.search(repattern_title_text, \n",
    "                                           sentence_around_title[len(title_number.group(0)):])\n",
    "                    string_after_title = title_text.string[title_text.span()[1]:\n",
    "                                                            (title_text.span()[1]+30)]\n",
    "\n",
    "                    bookmark_content = [outline.title,\n",
    "                                        str(pdf.get_destination_page_number(outline)),\n",
    "                                        str(pdf.get_destination_page_number(outline)+offset),\n",
    "                                        title_number.group(0),\n",
    "                                        title_text.group(0),\n",
    "                                        string_after_title,\n",
    "                                        str(nb_keywords)\n",
    "                                        ]\n",
    "                    pass\n",
    "                else:\n",
    "                    title_text = re.search(repattern_title_text, sentence_around_title[len(title_number.group(0)):])\n",
    "                    string_after_title = title_text.string[title_text.span()[1]:\n",
    "                                                           (title_text.span()[1]+30)]\n",
    "\n",
    "                    bookmark_content = [outline.title,\n",
    "                                        str(pdf.get_destination_page_number(outline)),\n",
    "                                        str(pdf.get_destination_page_number(outline)+offset),\n",
    "                                        '',\n",
    "                                        title_text.group(0),\n",
    "                                        string_after_title,\n",
    "                                        str(nb_keywords)\n",
    "                                        ]\n",
    "                    pass\n",
    "            except:\n",
    "                bookmark_content = [outline.title,\n",
    "                                    pdf.get_destination_page_number(outline),\n",
    "                                    '',\n",
    "                                    '',\n",
    "                                    '',\n",
    "                                    '',\n",
    "                                    ''\n",
    "                                    ]\n",
    "                pass\n",
    "        else:\n",
    "            bookmark_content = [outline.title,\n",
    "                                pdf.get_destination_page_number(outline),\n",
    "                                '',\n",
    "                                '',\n",
    "                                '',\n",
    "                                '',\n",
    "                                ''\n",
    "                                ]\n",
    "            pass\n",
    "        \n",
    "        return bookmark_content\n",
    "\n",
    "\n",
    "    headers = ['position in bookmark tree',\n",
    "                'original pdf bookmark title (outline.title)',\n",
    "                'original pdf bookmark page number',\n",
    "                'corrected pdf bookmark page number',\n",
    "                'extracted title number',\n",
    "                'extracted title text',\n",
    "                'first 20 characters after title',\n",
    "                'nb_keywords for title search']\n",
    "\n",
    "    df_content = pd.DataFrame(columns=headers,\n",
    "                            dtype=\"string\")\n",
    "    \n",
    "    n_section = -1\n",
    "\n",
    "    for outline in outlines:\n",
    "        if type(outline) == PyPDF2.generic._data_structures.Destination:\n",
    "            n_section = n_section + 1\n",
    "            if parent_position_in_bookmark_tree=='':\n",
    "                current_position_in_bookmark_tree = str(n_section)\n",
    "            else:\n",
    "                current_position_in_bookmark_tree = parent_position_in_bookmark_tree+'>'+str(n_section)\n",
    "\n",
    "            df_thisbookmark_content = get_bookmark_content(outline, pdf)\n",
    "            df_thisbookmark_content = [current_position_in_bookmark_tree]+df_thisbookmark_content\n",
    "\n",
    "            try:\n",
    "                df_thisbookmark_content = pd.DataFrame(dict(zip(headers, df_thisbookmark_content)),\n",
    "                                                    dtype=\"string\", \n",
    "                                                    index=[0])\n",
    "            except UnicodeDecodeError:\n",
    "                df_thisbookmark_content[1] = df_thisbookmark_content[1].decode('utf-8','ignore')+' (Python Exception: title UnicodeDecodeError)'\n",
    "                df_thisbookmark_content = pd.DataFrame(dict(zip(headers, df_thisbookmark_content)),\n",
    "                                                    dtype=\"string\", \n",
    "                                                    index=[0])\n",
    "\n",
    "            df_content = pd.concat([df_content, \n",
    "                                    df_thisbookmark_content], \n",
    "                                    ignore_index=True)\n",
    "        elif type(outline) == list:\n",
    "            if parent_position_in_bookmark_tree=='':\n",
    "                current_position_in_bookmark_tree = str(n_section)\n",
    "            else:\n",
    "                current_position_in_bookmark_tree = parent_position_in_bookmark_tree+'>'+str(n_section)\n",
    "\n",
    "            extracted_content = get_bookmarks(outline, pdf, current_position_in_bookmark_tree)\n",
    "\n",
    "            df_content = pd.concat([df_content, \n",
    "                                    extracted_content], \n",
    "                                    ignore_index=True)\n",
    "        else:\n",
    "            raise Exception\n",
    "        \n",
    "    return df_content\n",
    "\n",
    "# drs_doc = PyPDF2.PdfReader(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\raw data\\AC\\pdf\\Current__083AD53732DC0D4686258347005037DF.pdf\") # AC 43-13\n",
    "# drs_doc = PyPDF2.PdfReader(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\raw data\\AC\\pdf\\Current__99C827DB9BAAC81B86256B4500596C4E.pdf\") # AC 43-13\n",
    "# drs_doc = PyPDF2.PdfReader(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\raw data\\AC\\pdf\\Current__C7CCE9FCA6D7E34786257D41004C3E63.pdf\") # AC 29\n",
    "# drs_doc = PyPDF2.PdfReader(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\raw data\\ORDER_8900.1\\pdf\\Current__DRSDOCID166392477020230614171412.pdf\") # Order\n",
    "# drs_doc = PyPDF2.PdfReader(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\raw data\\AC\\pdf\\Current__C2774DBA5FFEE8D286258088005528A9.pdf\") # Order\n",
    "\n",
    "\n",
    "# import glob, os\n",
    "# matching_file = glob.glob(os.path.join(os.getcwd(), 'data', 'DRS', 'raw data', '**','**','*5934B72DA528D2CC862585230041269E*'))\n",
    "# drs_doc = PyPDF2.PdfReader(matching_file[0])\n",
    "\n",
    "# df_content = get_bookmarks(outlines = drs_doc.outline, pdf = drs_doc, parent_position_in_bookmark_tree='') # [8:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AC 21-43A: E7E9387A5386881C86257ED9005A1B5A\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "import PyPDF2\n",
    "import datetime\n",
    "\n",
    "drs_doc_types = ['AC',\n",
    "                 'ORDER_8300.10',\n",
    "                 'ORDER_8400.10',\n",
    "                 'ORDER_8700.1',\n",
    "                 'ORDER_8740.1',\n",
    "                 'ORDER_8900.1',\n",
    "                 'ORDERS']\n",
    "\n",
    "df_all_content = None\n",
    "\n",
    "for drs_doc_type in drs_doc_types:\n",
    "    df_index = glob.glob(os.path.join(os.getcwd(), 'data', 'DRS', 'index', drs_doc_type+'*.parquet'))[-1]\n",
    "    df_index = pd.read_parquet(df_index)\n",
    "\n",
    "    for index, row in df_index.iterrows():\n",
    "        matching_file = glob.glob(os.path.join(os.getcwd(), 'data', 'DRS', 'raw data', drs_doc_type, 'pdf','*'+row['documentGuid']+'*.pdf'))\n",
    "        if len(matching_file)>0:\n",
    "            if row['drs:status']=='Current':\n",
    "                if 'drs:docID'  in row.index:\n",
    "                    print(row['drs:docID']+': '+row['documentGuid'])\n",
    "                elif 'drs:documentNumber' in row.index:\n",
    "                    print(row['drs:documentNumber']+': '+row['documentGuid'])\n",
    "\n",
    "                drs_doc = PyPDF2.PdfReader(matching_file[0])\n",
    "                df_content = get_bookmarks(outlines = drs_doc.outline, \n",
    "                                           pdf = drs_doc, \n",
    "                                           parent_position_in_bookmark_tree='')\n",
    "\n",
    "                if len(df_content)>0:                    \n",
    "                    df_content.insert(loc=0, \n",
    "                                      column='documentGuid', \n",
    "                                      value=pd.Series(data=row['documentGuid'], index=df_content.index, dtype=\"string\"))\n",
    "                    if df_all_content is not None:\n",
    "                        df_all_content = pd.concat([df_all_content, \n",
    "                                                    df_content], \n",
    "                                                    ignore_index=True)\n",
    "                    else:\n",
    "                        df_all_content = df_content\n",
    "                    print('   -> '+str(df_all_content.shape))\n",
    "\n",
    "                \n",
    "df_all_content.to_parquet(os.path.join(os.getcwd(), \n",
    "                                       'data', \n",
    "                                       'DRS', \n",
    "                                       'extracted_text_for_'+'-'.join(drs_doc_types)+'-'+datetime.datetime.now().strftime(\"%Y%m%d - %H%M%S\")+'.parquet'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# df_all_content = pd.read_parquet(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\extracted_text_for_AC-ORDER_8300.10-ORDER_8400.10-ORDER_8700.1-ORDER_8740.1-ORDER_8900.1-ORDERS.parquet\")\n",
    "df_all_content = pd.read_parquet(r\"C:\\Users\\victor\\Documents\\DeepLearning\\FAA NLP Project\\data\\DRS\\extracted_text_for_AC-ORDER_8300.10-ORDER_8400.10-ORDER_8700.1-ORDER_8740.1-ORDER_8900.1-ORDERS-20231214 - 020735.parquet\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Non DRS document (AIM...)\n",
    "Defining the functions first and then applying that on a selection of Non DRS documents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "import pandas as pd\n",
    "\n",
    "# pdf.pages[pdf.get_destination_page_number(pdf.outline[n]):\n",
    "#                             pdf.get_destination_page_number(pdf.outline[n+1])].extract_text()\n",
    "\n",
    "def get_bookmarks(outlines, pdf, parent_position_in_bookmark_tree):\n",
    "    headers = ['position in bookmark tree',\n",
    "                'original pdf bookmark title (outline.title)',\n",
    "                'original pdf bookmark page number'\n",
    "                ]\n",
    "    df_content = pd.DataFrame(columns=headers,\n",
    "                            dtype=\"string\")    \n",
    "    n_section = -1\n",
    "\n",
    "    for outline in outlines:\n",
    "        if type(outline) == PyPDF2.generic._data_structures.Destination:\n",
    "            n_section = n_section + 1\n",
    "            if parent_position_in_bookmark_tree=='':\n",
    "                current_position_in_bookmark_tree = str(n_section)\n",
    "            else:\n",
    "                current_position_in_bookmark_tree = parent_position_in_bookmark_tree+'>'+str(n_section)\n",
    "\n",
    "            bookmark_content = [current_position_in_bookmark_tree,\n",
    "                                outline.title,\n",
    "                                str(pdf.get_destination_page_number(outline))\n",
    "                                ]                        \n",
    "            df_thisbookmark_content = pd.DataFrame(dict(zip(headers, bookmark_content)),\n",
    "                                                dtype=\"string\", \n",
    "                                                index=[0])            \n",
    "            df_content = pd.concat([df_content, \n",
    "                                    df_thisbookmark_content], \n",
    "                                    ignore_index=True)\n",
    "                \n",
    "        elif type(outline) == list:\n",
    "            if parent_position_in_bookmark_tree=='':\n",
    "                current_position_in_bookmark_tree = str(n_section)\n",
    "            else:\n",
    "                current_position_in_bookmark_tree = parent_position_in_bookmark_tree+'>'+str(n_section)\n",
    "\n",
    "            extracted_content = get_bookmarks(outline, pdf, current_position_in_bookmark_tree)\n",
    "\n",
    "            df_content = pd.concat([df_content, \n",
    "                                    extracted_content], \n",
    "                                    ignore_index=True)\n",
    "        \n",
    "    return df_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   -> (73, 4)\n",
      "   -> (187, 4)\n",
      "   -> (372, 4)\n",
      "   -> (679, 4)\n",
      "   -> (704, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\victor\\anaconda3\\envs\\faa-nlp-drs\\Lib\\site-packages\\pyarrow\\pandas_compat.py:373: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if _pandas_api.is_sparse(col):\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "import PyPDF2\n",
    "import datetime\n",
    "\n",
    "raw_documents = glob.glob(os.path.join(os.getcwd(), 'data', 'NotInDRS', 'raw data', 'NotInDRS', 'pdf', '*.pdf'))\n",
    "df_all_content = None\n",
    "\n",
    "for raw_document in raw_documents:\n",
    "    drs_doc = PyPDF2.PdfReader(raw_document)\n",
    "    df_content = get_bookmarks(outlines = drs_doc.outline, pdf = drs_doc, parent_position_in_bookmark_tree='')\n",
    "\n",
    "    if len(df_content)>0:                    \n",
    "        df_content.insert(loc=0, column='filename', value=os.path.basename(raw_document))\n",
    "        if df_all_content is not None:\n",
    "            df_all_content = pd.concat([df_all_content, \n",
    "                                        df_content], \n",
    "                                        ignore_index=True)\n",
    "        else:\n",
    "            df_all_content = df_content\n",
    "        print('   -> '+str(df_all_content.shape))\n",
    "\n",
    "                \n",
    "df_all_content.to_parquet(os.path.join(os.getcwd(), \n",
    "                                       'data', \n",
    "                                       'NotInDRS', \n",
    "                                       'dataset',\n",
    "                                       'extracted_text',\n",
    "                                       'extracted_text_for_DocNotInDRS'+'-'+datetime.datetime.now().strftime(\"%Y%m%d - %H%M%S\")+'.parquet'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Post-process to split AIP and AIM with one more section level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 41\u001b[0m\n\u001b[0;32m     39\u001b[0m pattern_findings \u001b[38;5;241m=\u001b[39m re_pattern\u001b[38;5;241m.\u001b[39mfindall(pdf\u001b[38;5;241m.\u001b[39mpages[n]\u001b[38;5;241m.\u001b[39mextract_text())\n\u001b[0;32m     40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(pattern_findings)\u001b[38;5;241m>\u001b[39m\u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m---> 41\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m pattern_finding \u001b[38;5;129;01min\u001b[39;00m pattern_findings:\n\u001b[0;32m     42\u001b[0m         first_letter_lower_case \u001b[38;5;241m=\u001b[39m [w[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mislower() \u001b[38;5;28;01mfor\u001b[39;00m w \u001b[38;5;129;01min\u001b[39;00m pattern_finding\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mlen\u001b[39m(w)\u001b[38;5;241m>\u001b[39m\u001b[38;5;241m3\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m w[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m0123456789\u001b[39m\u001b[38;5;124m'\u001b[39m)][\u001b[38;5;241m1\u001b[39m:]\n\u001b[0;32m     43\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m np\u001b[38;5;241m.\u001b[39msum(first_letter_lower_case)\u001b[38;5;241m<\u001b[39m\u001b[38;5;241m2\u001b[39m:\n",
      "Cell \u001b[1;32mIn[4], line 41\u001b[0m\n\u001b[0;32m     39\u001b[0m pattern_findings \u001b[38;5;241m=\u001b[39m re_pattern\u001b[38;5;241m.\u001b[39mfindall(pdf\u001b[38;5;241m.\u001b[39mpages[n]\u001b[38;5;241m.\u001b[39mextract_text())\n\u001b[0;32m     40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(pattern_findings)\u001b[38;5;241m>\u001b[39m\u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m---> 41\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m pattern_finding \u001b[38;5;129;01min\u001b[39;00m pattern_findings:\n\u001b[0;32m     42\u001b[0m         first_letter_lower_case \u001b[38;5;241m=\u001b[39m [w[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mislower() \u001b[38;5;28;01mfor\u001b[39;00m w \u001b[38;5;129;01min\u001b[39;00m pattern_finding\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mlen\u001b[39m(w)\u001b[38;5;241m>\u001b[39m\u001b[38;5;241m3\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m w[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m0123456789\u001b[39m\u001b[38;5;124m'\u001b[39m)][\u001b[38;5;241m1\u001b[39m:]\n\u001b[0;32m     43\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m np\u001b[38;5;241m.\u001b[39msum(first_letter_lower_case)\u001b[38;5;241m<\u001b[39m\u001b[38;5;241m2\u001b[39m:\n",
      "File \u001b[1;32m_pydevd_bundle/pydevd_cython.pyx:1457\u001b[0m, in \u001b[0;36m_pydevd_bundle.pydevd_cython.SafeCallWrapper.__call__\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m_pydevd_bundle/pydevd_cython.pyx:701\u001b[0m, in \u001b[0;36m_pydevd_bundle.pydevd_cython.PyDBFrame.trace_dispatch\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m_pydevd_bundle/pydevd_cython.pyx:1152\u001b[0m, in \u001b[0;36m_pydevd_bundle.pydevd_cython.PyDBFrame.trace_dispatch\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m_pydevd_bundle/pydevd_cython.pyx:1135\u001b[0m, in \u001b[0;36m_pydevd_bundle.pydevd_cython.PyDBFrame.trace_dispatch\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m_pydevd_bundle/pydevd_cython.pyx:312\u001b[0m, in \u001b[0;36m_pydevd_bundle.pydevd_cython.PyDBFrame.do_wait_suspend\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mc:\\Users\\victor\\anaconda3\\envs\\faa-nlp-drs\\Lib\\site-packages\\debugpy\\_vendored\\pydevd\\pydevd.py:2070\u001b[0m, in \u001b[0;36mPyDB.do_wait_suspend\u001b[1;34m(self, thread, frame, event, arg, exception_type)\u001b[0m\n\u001b[0;32m   2067\u001b[0m             from_this_thread\u001b[38;5;241m.\u001b[39mappend(frame_custom_thread_id)\n\u001b[0;32m   2069\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_threads_suspended_single_notification\u001b[38;5;241m.\u001b[39mnotify_thread_suspended(thread_id, thread, stop_reason):\n\u001b[1;32m-> 2070\u001b[0m         keep_suspended \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_do_wait_suspend(thread, frame, event, arg, suspend_type, from_this_thread, frames_tracker)\n\u001b[0;32m   2072\u001b[0m frames_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   2074\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m keep_suspended:\n\u001b[0;32m   2075\u001b[0m     \u001b[38;5;66;03m# This means that we should pause again after a set next statement.\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\victor\\anaconda3\\envs\\faa-nlp-drs\\Lib\\site-packages\\debugpy\\_vendored\\pydevd\\pydevd.py:2106\u001b[0m, in \u001b[0;36mPyDB._do_wait_suspend\u001b[1;34m(self, thread, frame, event, arg, suspend_type, from_this_thread, frames_tracker)\u001b[0m\n\u001b[0;32m   2103\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_input_hook()\n\u001b[0;32m   2105\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_internal_commands()\n\u001b[1;32m-> 2106\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m0.01\u001b[39m)\n\u001b[0;32m   2108\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel_async_evaluation(get_current_thread_id(thread), \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mid\u001b[39m(frame)))\n\u001b[0;32m   2110\u001b[0m \u001b[38;5;66;03m# process any stepping instructions\u001b[39;00m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import os\n",
    "from pypdf import PdfReader\n",
    "import datetime\n",
    "# import PyPDF2 import PdfReader\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "predatasets = sorted(glob.glob(os.path.join(os.getcwd(), 'data', 'NotInDRS', 'dataset', 'extracted_text', '*.parquet')), \n",
    "                    key=os.path.getctime, \n",
    "                    reverse=True)\n",
    "predatasets = [val for val in predatasets if '(augmented)' not in val]\n",
    "df_predataset = pd.read_parquet(predatasets[0])\n",
    "\n",
    "filename_aim = 'Aeronautical Information Manual (AIM) Basic with Change 1.pdf'\n",
    "filename_aip = 'Aeronautical Information Publication (AIP) Basic with Amendments 1, 2 and 3.pdf'\n",
    "\n",
    "pdf_aim = PdfReader(os.path.join(os.getcwd(), 'data', 'NotInDRS', 'raw data', 'NotInDRS', 'pdf', filename_aim))\n",
    "pdf_aip = PdfReader(os.path.join(os.getcwd(), 'data', 'NotInDRS', 'raw data', 'NotInDRS', 'pdf', filename_aip))\n",
    "\n",
    "re_prefix = '[ \\s\\r\\n]'\n",
    "re_sufix = '[\\r\\n]'\n",
    "re_pattern_aim = re.compile(re_prefix+\n",
    "                            '[0-9]{1,2}[−-][0-9]{1,2}[−-][0-9]{1,2}\\. [A-Za-z \\(\\)\\-−\\/]{6,80}'+\n",
    "                            re_sufix)\n",
    "re_pattern_aip1 = re.compile(re_prefix+\n",
    "                             '[0-9]{1,2}\\. [A-Za-z \\(\\)\\-−\\/]{6,80}'+\n",
    "                             re_sufix)\n",
    "\n",
    "list_split4nonDRS = [[filename_aim, pdf_aim, re_pattern_aim],\n",
    "                     [filename_aip, pdf_aip, re_pattern_aip1]]\n",
    "\n",
    "for settings in list_split4nonDRS:\n",
    "    filename, pdf, re_pattern = settings\n",
    "    df_tobeadded = pd.DataFrame(columns=['previous_index']+list(df_predataset),\n",
    "                                dtype=\"string\")\n",
    "    for n in range(0, len(pdf.pages)):\n",
    "        pattern_findings = re_pattern.findall(pdf.pages[n].extract_text())\n",
    "        if len(pattern_findings)>0:\n",
    "            for pattern_finding in pattern_findings:\n",
    "                first_letter_lower_case = [w[0].islower() for w in pattern_finding.split(' ') if (len(w)>3 and w[0] not in '0123456789')][1:]\n",
    "                if np.sum(first_letter_lower_case)<2:\n",
    "                    previous_index = df_predataset[(df_predataset[\"filename\"]==filename) & \n",
    "                                                (df_predataset[\"original pdf bookmark page number\"].apply(int)<=n) &\n",
    "                                                (df_predataset[\"original pdf bookmark page number\"].apply(int)>=0)].index[-1]\n",
    "                    df_new_content = pd.DataFrame(dict(zip(['previous_index']+list(df_predataset), \n",
    "                                                            [previous_index,\n",
    "                                                            filename,\n",
    "                                                            df_predataset.loc[previous_index,\"position in bookmark tree\"]+'>0',\n",
    "                                                            pattern_finding[1:-1],\n",
    "                                                            str(n)])),\n",
    "                                                        dtype=\"string\", \n",
    "                                                        index=[0])            \n",
    "                    df_tobeadded = pd.concat([df_tobeadded, \n",
    "                                            df_new_content], \n",
    "                                            ignore_index=True)\n",
    "\n",
    "    counter_next_section = 0\n",
    "\n",
    "    df_tobeadded = df_tobeadded.iloc[::-1]\n",
    "    for idx, row in df_tobeadded.iterrows():\n",
    "        df_predataset = pd.concat([df_predataset.iloc[:int(row['previous_index'])+1+counter_next_section], \n",
    "                                pd.DataFrame(dict(zip(list(df_predataset), row[list(df_predataset)])),\n",
    "                                                dtype=\"string\",\n",
    "                                                index=[0]), \n",
    "                                    df_predataset.iloc[int(row['previous_index'])+1+counter_next_section:]]).reset_index(drop=True)\n",
    "\n",
    "df_predataset.to_parquet(os.path.join(os.getcwd(), \n",
    "                                    'data', \n",
    "                                    'NotInDRS', \n",
    "                                    'dataset',\n",
    "                                    'extracted_text',\n",
    "                                    'extracted_text (augmented)_for_DocNotInDRS'+'-'+datetime.datetime.now().strftime(\"%Y%m%d - %H%M%S\")+'.parquet'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "faa-nlp-drs",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}