Converting the CMU dictionary to Unicode IPA symbols.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Eric Meinhardt / emeinhardt@ucsd.edu**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:06.870083Z",
     "start_time": "2019-04-15T16:11:06.866695Z"
    }
   },
   "outputs": [],
   "source": [
    "#Prints **all** console output, not just last item in cell \n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading the CMU dict in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.011797Z",
     "start_time": "2019-04-15T16:11:06.875376Z"
    }
   },
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CMU pronouncing dictionary v0.7b obtained from `http://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict-0.7b`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.129301Z",
     "start_time": "2019-04-15T16:11:07.014353Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " cmudict-0.7b\r\n",
      " cmudict-0.7b_IPA_destressed.tsv\r\n",
      " cmudict-0.7b_IPA_stressed.tsv\r\n",
      " cmudict-0.7b.tsv\r\n",
      "'Converting the CMU dictionary to use IPA representations.ipynb'\r\n"
     ]
    }
   ],
   "source": [
    "%ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.246674Z",
     "start_time": "2019-04-15T16:11:07.131265Z"
    }
   },
   "outputs": [],
   "source": [
    "cmudict_raw_filename = 'cmudict-0.7b'\n",
    "\n",
    "cmudict_raw = []\n",
    "with open(cmudict_raw_filename, 'r', encoding='latin-1') as the_file:\n",
    "    for row in the_file:\n",
    "        cmudict_raw.append(row.rstrip('\\n'))#(row.rstrip('\\r\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.361892Z",
     "start_time": "2019-04-15T16:11:07.248689Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     1\t;;; # CMUdict  --  Major Version: 0.07\r\n",
      "     2\t;;; \r\n",
      "     3\t;;; # $HeadURL$\r\n",
      "     4\t;;; # $Date::                                                   $:\r\n",
      "     5\t;;; # $Id::                                                     $:\r\n",
      "     6\t;;; # $Rev::                                                    $: \r\n",
      "     7\t;;; # $Author::                                                 $:\r\n",
      "     8\t;;;\r\n",
      "     9\t;;; #\r\n",
      "    10\t;;; # ========================================================================\r\n",
      "    11\t;;; # Copyright (C) 1993-2015 Carnegie Mellon University. All rights reserved.\r\n",
      "    12\t;;; #\r\n",
      "    13\t;;; # Redistribution and use in source and binary forms, with or without\r\n",
      "    14\t;;; # modification, are permitted provided that the following conditions\r\n",
      "    15\t;;; # are met:\r\n",
      "    16\t;;; #\r\n",
      "    17\t;;; # 1. Redistributions of source code must retain the above copyright\r\n",
      "    18\t;;; #    notice, this list of conditions and the following disclaimer.\r\n",
      "    19\t;;; #    The contents of this file are deemed to be source code.\r\n",
      "    20\t;;; #\r\n",
      "    21\t;;; # 2. Redistributions in binary form must reproduce the above copyright\r\n",
      "    22\t;;; #    notice, this list of conditions and the following disclaimer in\r\n",
      "    23\t;;; #    the documentation and/or other materials provided with the\r\n",
      "    24\t;;; #    distribution.\r\n",
      "    25\t;;; #\r\n",
      "    26\t;;; # This work was supported in part by funding from the Defense Advanced\r\n",
      "    27\t;;; # Research Projects Agency, the Office of Naval Research and the National\r\n",
      "    28\t;;; # Science Foundation of the United States of America, and by member\r\n",
      "    29\t;;; # companies of the Carnegie Mellon Sphinx Speech Consortium. We acknowledge\r\n",
      "    30\t;;; # the contributions of many volunteers to the expansion and improvement of\r\n",
      "    31\t;;; # this dictionary.\r\n",
      "    32\t;;; #\r\n",
      "    33\t;;; # THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND\r\n",
      "    34\t;;; # ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,\r\n",
      "    35\t;;; # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r\n",
      "    36\t;;; # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY\r\n",
      "    37\t;;; # NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r\n",
      "    38\t;;; # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r\n",
      "    39\t;;; # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r\n",
      "    40\t;;; # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r\n",
      "    41\t;;; # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r\n",
      "    42\t;;; # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\n",
      "    43\t;;; # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r\n",
      "    44\t;;; #\r\n",
      "    45\t;;; # ========================================================================\r\n",
      "    46\t;;; #\r\n",
      "    47\t;;;\r\n",
      "    48\t;;;  NOTES  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\r\n",
      "    49\t;;; \r\n",
      "    50\t;;;  [20080401] (air)  New dict file format introduced \r\n",
      "    51\t;;;   - comments (like this section) are allowed\r\n",
      "    52\t;;;   - file name is major version; vers/rev information is now in the header\r\n",
      "    53\t;;;\r\n",
      "    54\t;;; \r\n",
      "    55\t;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\r\n",
      "    56\t;;; \r\n",
      "    57\t!EXCLAMATION-POINT  EH2 K S K L AH0 M EY1 SH AH0 N P OY2 N T\r\n",
      "    58\t\"CLOSE-QUOTE  K L OW1 Z K W OW1 T\r\n",
      "    59\t\"DOUBLE-QUOTE  D AH1 B AH0 L K W OW1 T\r\n",
      "    60\t\"END-OF-QUOTE  EH1 N D AH0 V K W OW1 T\r\n",
      "cat: write error: Broken pipe\r\n"
     ]
    }
   ],
   "source": [
    "%cat -n cmudict-0.7b | head -60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.376694Z",
     "start_time": "2019-04-15T16:11:07.365200Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "';;; # CMUdict  --  Major Version: 0.07'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "';;; '"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'!EXCLAMATION-POINT  EH2 K S K L AH0 M EY1 SH AH0 N P OY2 N T'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmudict_raw[0] #header...\n",
    "cmudict_raw[55] #end header\n",
    "cmudict_raw[56]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.385782Z",
     "start_time": "2019-04-15T16:11:07.378056Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!EXCLAMATION-POINT  EH2 K S K L AH0 M EY1 SH AH0 N P OY2 N T'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'ALBINIA  AA0 L B IY1 N IY0 AH0'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmudict_entry_lines = cmudict_raw[56:]\n",
    "cmudict_entry_lines[0]\n",
    "cmudict_entry_lines[2434]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert to tsv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each entry line is split by a double space..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.392551Z",
     "start_time": "2019-04-15T16:11:07.387248Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\"IN-QUOTES', 'IH1 N K W OW1 T S']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['ALBINIA', 'AA0 L B IY1 N IY0 AH0']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['PATRICOF', 'P AE1 T R IH0 K AO2 F']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmudict_entry_lines[5].split('  ')\n",
    "cmudict_entry_lines[2434].split('  ')\n",
    "cmudict_entry_lines[89347].split('  ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.482681Z",
     "start_time": "2019-04-15T16:11:07.393779Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['!EXCLAMATION-POINT', 'EH2 K S K L AH0 M EY1 SH AH0 N P OY2 N T'],\n",
       " ['\"CLOSE-QUOTE', 'K L OW1 Z K W OW1 T'],\n",
       " ['\"DOUBLE-QUOTE', 'D AH1 B AH0 L K W OW1 T'],\n",
       " ['\"END-OF-QUOTE', 'EH1 N D AH0 V K W OW1 T'],\n",
       " ['\"END-QUOTE', 'EH1 N D K W OW1 T'],\n",
       " ['\"IN-QUOTES', 'IH1 N K W OW1 T S'],\n",
       " ['\"QUOTE', 'K W OW1 T'],\n",
       " ['\"UNQUOTE', 'AH1 N K W OW1 T'],\n",
       " ['#HASH-MARK', 'HH AE1 M AA2 R K'],\n",
       " ['#POUND-SIGN', 'P AW1 N D S AY2 N']]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab_separated_cmudict_lines = [entry.split('  ') for entry in cmudict_entry_lines]\n",
    "tab_separated_cmudict_lines[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.557279Z",
     "start_time": "2019-04-15T16:11:07.484324Z"
    }
   },
   "outputs": [],
   "source": [
    "cmudict = list(map(lambda tscdl: {'Orthography':tscdl[0],\n",
    "                                  'Transcription':tscdl[1]},\n",
    "                   tab_separated_cmudict_lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.563541Z",
     "start_time": "2019-04-15T16:11:07.558678Z"
    }
   },
   "outputs": [],
   "source": [
    "from csv import DictWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:07.882197Z",
     "start_time": "2019-04-15T16:11:07.564773Z"
    }
   },
   "outputs": [],
   "source": [
    "cmudict_tsv_fn = 'cmudict-0.7b.tsv'\n",
    "\n",
    "with open(cmudict_tsv_fn, 'w', newline='\\n') as dl:\n",
    "    fieldnames = ['Orthography', 'Transcription']\n",
    "    writer = csv.DictWriter(dl, fieldnames=fieldnames, delimiter='\\t', quoting=csv.QUOTE_NONE, quotechar='@')\n",
    "    writer.writeheader()\n",
    "    for entry in cmudict:\n",
    "        _ = writer.writerow(entry) #assignment suppresses return value (= number of characters, iirc?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.002345Z",
     "start_time": "2019-04-15T16:11:07.884370Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     1\tOrthography\tTranscription\r",
      "\r\n",
      "     2\t!EXCLAMATION-POINT\tEH2 K S K L AH0 M EY1 SH AH0 N P OY2 N T\r",
      "\r\n",
      "     3\t\"CLOSE-QUOTE\tK L OW1 Z K W OW1 T\r",
      "\r\n",
      "     4\t\"DOUBLE-QUOTE\tD AH1 B AH0 L K W OW1 T\r",
      "\r\n",
      "     5\t\"END-OF-QUOTE\tEH1 N D AH0 V K W OW1 T\r",
      "\r\n",
      "     6\t\"END-QUOTE\tEH1 N D K W OW1 T\r",
      "\r\n",
      "     7\t\"IN-QUOTES\tIH1 N K W OW1 T S\r",
      "\r\n",
      "     8\t\"QUOTE\tK W OW1 T\r",
      "\r\n",
      "     9\t\"UNQUOTE\tAH1 N K W OW1 T\r",
      "\r\n",
      "    10\t#HASH-MARK\tHH AE1 M AA2 R K\r",
      "\r\n",
      "    11\t#POUND-SIGN\tP AW1 N D S AY2 N\r",
      "\r\n",
      "    12\t#SHARP-SIGN\tSH AA1 R P S AY2 N\r",
      "\r\n",
      "    13\t%PERCENT\tP ER0 S EH1 N T\r",
      "\r\n",
      "    14\t&AMPERSAND\tAE1 M P ER0 S AE2 N D\r",
      "\r\n",
      "    15\t'ALLO\tAA2 L OW1\r",
      "\r\n",
      "    16\t'APOSTROPHE\tAH0 P AA1 S T R AH0 F IY0\r",
      "\r\n",
      "    17\t'BOUT\tB AW1 T\r",
      "\r\n",
      "    18\t'CAUSE\tK AH0 Z\r",
      "\r\n",
      "    19\t'COURSE\tK AO1 R S\r",
      "\r\n",
      "    20\t'CUSE\tK Y UW1 Z\r",
      "\r\n",
      "    21\t'EM\tAH0 M\r",
      "\r\n",
      "    22\t'END-INNER-QUOTE\tEH1 N D IH1 N ER0 K W OW1 T\r",
      "\r\n",
      "    23\t'END-QUOTE\tEH1 N D K W OW1 T\r",
      "\r\n",
      "    24\t'FRISCO\tF R IH1 S K OW0\r",
      "\r\n",
      "    25\t'GAIN\tG EH1 N\r",
      "\r\n",
      "    26\t'INNER-QUOTE\tIH1 N ER0 K W OW1 T\r",
      "\r\n",
      "    27\t'KAY\tK EY1\r",
      "\r\n",
      "    28\t'M\tAH0 M\r",
      "\r\n",
      "    29\t'N\tAH0 N\r",
      "\r\n",
      "    30\t'QUOTE\tK W OW1 T\r",
      "\r\n",
      "    31\t'ROUND\tR AW1 N D\r",
      "\r\n",
      "    32\t'S\tEH1 S\r",
      "\r\n",
      "    33\t'SINGLE-QUOTE\tS IH1 NG G AH0 L K W OW1 T\r",
      "\r\n",
      "    34\t'TIL\tT IH1 L\r",
      "\r\n",
      "    35\t'TIS\tT IH1 Z\r",
      "\r\n",
      "    36\t'TWAS\tT W AH1 Z\r",
      "\r\n",
      "    37\t(BEGIN-PARENS\tB IH0 G IH1 N P ER0 EH1 N Z\r",
      "\r\n",
      "    38\t(IN-PARENTHESES\tIH1 N P ER0 EH1 N TH AH0 S IY2 Z\r",
      "\r\n",
      "    39\t(LEFT-PAREN\tL EH1 F T P ER0 EH1 N\r",
      "\r\n",
      "    40\t(OPEN-PARENTHESES\tOW1 P AH0 N P ER0 EH1 N TH AH0 S IY2 Z\r",
      "\r\n",
      "    41\t(PAREN\tP ER0 EH1 N\r",
      "\r\n",
      "    42\t(PARENS\tP ER0 EH1 N Z\r",
      "\r\n",
      "    43\t(PARENTHESES\tP ER0 EH1 N TH AH0 S IY2 Z\r",
      "\r\n",
      "    44\t)CLOSE-PAREN\tK L OW1 Z P ER0 EH1 N\r",
      "\r\n",
      "    45\t)CLOSE-PARENTHESES\tK L OW1 Z P ER0 EH1 N TH AH0 S IY2 Z\r",
      "\r\n",
      "    46\t)END-PAREN\tEH1 N D P ER0 EH1 N\r",
      "\r\n",
      "    47\t)END-PARENS\tEH1 N D P ER0 EH1 N Z\r",
      "\r\n",
      "    48\t)END-PARENTHESES\tEH1 N D P ER0 EH1 N TH AH0 S IY2 Z\r",
      "\r\n",
      "    49\t)END-THE-PAREN\tEH1 N D DH AH0 P ER0 EH1 N\r",
      "\r\n",
      "    50\t)PAREN\tP ER0 EH1 N\r",
      "\r\n",
      "    51\t)PARENS\tP ER0 EH1 N Z\r",
      "\r\n",
      "    52\t)RIGHT-PAREN\tR AY1 T P EH2 R AH0 N\r",
      "\r\n",
      "    53\t)UN-PARENTHESES\tAH1 N P ER0 EH1 N TH AH0 S IY1 Z\r",
      "\r\n",
      "    54\t+PLUS\tP L UH1 S\r",
      "\r\n",
      "    55\t,COMMA\tK AA1 M AH0\r",
      "\r\n",
      "    56\t--DASH\tD AE1 SH\r",
      "\r\n",
      "    57\t-DASH\tD AE1 SH\r",
      "\r\n",
      "    58\t-HYPHEN\tHH AY1 F AH0 N\r",
      "\r\n",
      "    59\t...ELLIPSIS\tIH2 L IH1 P S IH0 S\r",
      "\r\n",
      "    60\t.DECIMAL\tD EH1 S AH0 M AH0 L\r",
      "\r\n",
      "cat: write error: Broken pipe\r\n"
     ]
    }
   ],
   "source": [
    "%cat -n cmudict-0.7b.tsv | head -60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert wordform representations to dotted strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.009368Z",
     "start_time": "2019-04-15T16:11:08.004674Z"
    }
   },
   "outputs": [],
   "source": [
    "def modify_dict(the_dict, the_key, the_new_value):\n",
    "    '''\n",
    "    Composable and (naively-implemented) non-mutating dictionary update.\n",
    "    '''\n",
    "    new_dict = {k:the_dict[k] for k in the_dict}\n",
    "    new_dict.update({the_key: the_new_value})\n",
    "    return new_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.019150Z",
     "start_time": "2019-04-15T16:11:08.011162Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AH1', 'N', 'K', 'W', 'OW1', 'T']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'AH1.N.K.W.OW1.T'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'AH1 N K W OW1 T'.split(' ')\n",
    "'.'.join('AH1 N K W OW1 T'.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.188566Z",
     "start_time": "2019-04-15T16:11:08.020814Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Orthography': '!EXCLAMATION-POINT',\n",
       "  'Transcription': 'EH2.K.S.K.L.AH0.M.EY1.SH.AH0.N.P.OY2.N.T'},\n",
       " {'Orthography': '\"CLOSE-QUOTE', 'Transcription': 'K.L.OW1.Z.K.W.OW1.T'},\n",
       " {'Orthography': '\"DOUBLE-QUOTE', 'Transcription': 'D.AH1.B.AH0.L.K.W.OW1.T'},\n",
       " {'Orthography': '\"END-OF-QUOTE', 'Transcription': 'EH1.N.D.AH0.V.K.W.OW1.T'},\n",
       " {'Orthography': '\"END-QUOTE', 'Transcription': 'EH1.N.D.K.W.OW1.T'}]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_lexicon = list(map(lambda row: modify_dict(row, \n",
    "                                              'Transcription', \n",
    "                                              '.'.join( row['Transcription'].split(' ') )),\n",
    "                      cmudict))\n",
    "ds_lexicon[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.193020Z",
     "start_time": "2019-04-15T16:11:08.190129Z"
    }
   },
   "outputs": [],
   "source": [
    "def tupleToDottedString(pair): \n",
    "    return '.'.join(pair)\n",
    "\n",
    "def dottedStringToTuple(s): \n",
    "    return tuple(s.split('.'))\n",
    "\n",
    "t2ds = tupleToDottedString\n",
    "ds2t = dottedStringToTuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.199588Z",
     "start_time": "2019-04-15T16:11:08.194215Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('AH1', 'N', 'K', 'W', 'OW1', 'T')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds2t('AH1.N.K.W.OW1.T')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert ARPABET to IPA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.206409Z",
     "start_time": "2019-04-15T16:11:08.204027Z"
    }
   },
   "outputs": [],
   "source": [
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.230444Z",
     "start_time": "2019-04-15T16:11:08.208360Z"
    }
   },
   "outputs": [],
   "source": [
    "def union(Ss):\n",
    "    return reduce(set.union, Ss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identify all ARPABET symbols in use in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.250617Z",
     "start_time": "2019-04-15T16:11:08.232152Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['EH2.K.S.K.L.AH0.M.EY1.SH.AH0.N.P.OY2.N.T',\n",
       " 'K.L.OW1.Z.K.W.OW1.T',\n",
       " 'D.AH1.B.AH0.L.K.W.OW1.T',\n",
       " 'EH1.N.D.AH0.V.K.W.OW1.T',\n",
       " 'EH1.N.D.K.W.OW1.T',\n",
       " 'IH1.N.K.W.OW1.T.S',\n",
       " 'K.W.OW1.T',\n",
       " 'AH1.N.K.W.OW1.T',\n",
       " 'HH.AE1.M.AA2.R.K',\n",
       " 'P.AW1.N.D.S.AY2.N']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transcriptions = list(map(lambda entry: entry['Transcription'],\n",
    "                          ds_lexicon))\n",
    "transcriptions[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.694819Z",
     "start_time": "2019-04-15T16:11:08.252214Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'EY2', 'OY1', 'G', 'F', 'OW1', 'UH1', 'DH', 'JH', 'SH', 'P', 'IY0', 'UH2', 'IY2', 'OY2', 'UW1', 'AA1', 'OW2', 'M', 'AY1', 'AE0', 'Z', 'AH1', 'ER2', 'AW1', 'AE2', 'ZH', 'ER1', 'R', 'OW0', 'T', 'W', 'AH0', 'K', 'IH1', 'ER0', 'AA2', 'IH0', 'AO2', 'AE1', 'TH', 'AO0', 'S', 'AY0', 'EH0', 'N', 'EH1', 'HH', 'CH', 'UW2', 'AW0', 'V', 'EY1', 'AW2', 'NG', 'UH0', 'EY0', 'AO1', 'AA0', 'Y', 'B', 'D', 'IY1', 'EH2', 'IH2', 'L', 'UW0', 'AH2', 'AY2', 'OY0'}\n"
     ]
    }
   ],
   "source": [
    "ARPABET_in_cmudict = union([set(t.split('.')) for t in transcriptions])\n",
    "len(ARPABET_in_cmudict)\n",
    "print(ARPABET_in_cmudict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.698648Z",
     "start_time": "2019-04-15T16:11:08.696228Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'EY'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'EY2'[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.705250Z",
     "start_time": "2019-04-15T16:11:08.699614Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AA',\n",
       " 'AE',\n",
       " 'AH',\n",
       " 'AO',\n",
       " 'AW',\n",
       " 'AY',\n",
       " 'B',\n",
       " 'CH',\n",
       " 'D',\n",
       " 'DH',\n",
       " 'EH',\n",
       " 'ER',\n",
       " 'EY',\n",
       " 'F',\n",
       " 'G',\n",
       " 'HH',\n",
       " 'IH',\n",
       " 'IY',\n",
       " 'JH',\n",
       " 'K',\n",
       " 'L',\n",
       " 'M',\n",
       " 'N',\n",
       " 'NG',\n",
       " 'OW',\n",
       " 'OY',\n",
       " 'P',\n",
       " 'R',\n",
       " 'S',\n",
       " 'SH',\n",
       " 'T',\n",
       " 'TH',\n",
       " 'UH',\n",
       " 'UW',\n",
       " 'V',\n",
       " 'W',\n",
       " 'Y',\n",
       " 'Z',\n",
       " 'ZH'}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ARPABET_in_cmudict_nostress = set(map(lambda symbol: symbol if symbol[-1] != \"0\" and symbol[-1] != \"1\" and symbol[-1] != \"2\" else symbol[:-1],\n",
    "                                       ARPABET_in_cmudict))\n",
    "ARPABET_in_cmudict_nostress"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Map ARPABET symbols to IPA symbols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See https://en.wikipedia.org/wiki/ARPABET."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.715537Z",
     "start_time": "2019-04-15T16:11:08.706242Z"
    }
   },
   "outputs": [],
   "source": [
    "arpabetUnicodeIPArelation = set([ \\\n",
    "('AO', 'ɔ'),\n",
    "('AA', 'ɑ'),\n",
    "('IY', 'i'),\n",
    "('UW', 'u'),\n",
    "('EH', 'ɛ'),\n",
    "('IH', 'ɪ'),\n",
    "('UH', 'ʊ'),\n",
    "('AH', 'ʌ'),\n",
    "('AX', 'ə'),\n",
    "('AE', 'æ'),\n",
    "('EY', 'eɪ'),\n",
    "('AY', 'aɪ'),\n",
    "('OW', 'oʊ'),\n",
    "('AW', 'aʊ'),\n",
    "('OY', 'ɔɪ'),\n",
    "('ER', 'ɚ'),\n",
    "('P', 'p'),\n",
    "('B', 'b'),\n",
    "('T', 't'),\n",
    "('D', 'd'),\n",
    "('K', 'k'),\n",
    "('G', 'g'),\n",
    "('CH', 'tʃ'),\n",
    "('JH', 'dʒ'),\n",
    "('F', 'f'),\n",
    "('V', 'v'),\n",
    "('TH', 'θ'),\n",
    "('DH', 'ð'),\n",
    "('S', 's'),\n",
    "('Z', 'z'),\n",
    "('SH', 'ʃ'),\n",
    "('ZH', 'ʒ'),\n",
    "('HH', 'h'),\n",
    "('M', 'm'),\n",
    "('EM', 'm̩'),\n",
    "('N', 'n'),\n",
    "('EN', 'n̩'),\n",
    "('NG', 'ŋ'),\n",
    "('ENG', 'ŋ̩'),\n",
    "('L', 'l'),\n",
    "('EL', 'l̩'),\n",
    "('R', 'ɹ'),\n",
    "('DX', 'ɾ'),\n",
    "('NX', 'ɾ̃'),\n",
    "('Y', 'j'),\n",
    "('W', 'w'),\n",
    "('Q', 'ʔ')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.721853Z",
     "start_time": "2019-04-15T16:11:08.716525Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AA',\n",
       " 'AE',\n",
       " 'AH',\n",
       " 'AO',\n",
       " 'AW',\n",
       " 'AX',\n",
       " 'AY',\n",
       " 'B',\n",
       " 'CH',\n",
       " 'D',\n",
       " 'DH',\n",
       " 'DX',\n",
       " 'EH',\n",
       " 'EL',\n",
       " 'EM',\n",
       " 'EN',\n",
       " 'ENG',\n",
       " 'ER',\n",
       " 'EY',\n",
       " 'F',\n",
       " 'G',\n",
       " 'HH',\n",
       " 'IH',\n",
       " 'IY',\n",
       " 'JH',\n",
       " 'K',\n",
       " 'L',\n",
       " 'M',\n",
       " 'N',\n",
       " 'NG',\n",
       " 'NX',\n",
       " 'OW',\n",
       " 'OY',\n",
       " 'P',\n",
       " 'Q',\n",
       " 'R',\n",
       " 'S',\n",
       " 'SH',\n",
       " 'T',\n",
       " 'TH',\n",
       " 'UH',\n",
       " 'UW',\n",
       " 'V',\n",
       " 'W',\n",
       " 'Y',\n",
       " 'Z',\n",
       " 'ZH'}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ARPABET_in_relation = set(map(lambda r: r[0],\n",
    "                              arpabetUnicodeIPArelation))\n",
    "ARPABET_in_relation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.727482Z",
     "start_time": "2019-04-15T16:11:08.722775Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{symbol for symbol in ARPABET_in_cmudict_nostress if symbol not in ARPABET_in_relation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The codebox above means every ARPABET symbol in the CMU dictionary is in the ARPABET-IPA relation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.735574Z",
     "start_time": "2019-04-15T16:11:08.728436Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'θ'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'TH'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def arpabetToUnicodeIPA(arpabetSymbol):\n",
    "    mapping = dict(arpabetUnicodeIPArelation)\n",
    "    #print(mapping)\n",
    "    return mapping[arpabetSymbol]\n",
    "arpabetToUnicodeIPA('TH')\n",
    "\n",
    "def invertMapping(mydict): \n",
    "    return dict([[val, key] for key,val in mydict.items()])\n",
    "def unicodeIPAToArpabet(unicodeIPAsymbol):\n",
    "    return invertMapping( dict(arpabetUnicodeIPArelation) )[unicodeIPAsymbol]\n",
    "unicodeIPAToArpabet('θ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.741326Z",
     "start_time": "2019-04-15T16:11:08.736511Z"
    }
   },
   "outputs": [],
   "source": [
    "def IPAifyUnstressedRep(unstressedRep):\n",
    "    arpabetTuple = dottedStringToTuple( unstressedRep ) \n",
    "    \n",
    "    IPAtuple = tuple(list(map(arpabetToUnicodeIPA, arpabetTuple)))\n",
    "    \n",
    "    dottedIPAsymbols = tupleToDottedString( IPAtuple )\n",
    "    return dottedIPAsymbols\n",
    "\n",
    "def IPAifyStressedRep(stressedRep):\n",
    "    stressedArpabetTuple = dottedStringToTuple( stressedRep )\n",
    "\n",
    "    arpabetSymbols = [symb[:-1] if symb[-1] == '0' or symb[-1] == '1' or symb[-1] == '2' else symb[:] for symb in stressedArpabetTuple]\n",
    "    stresses = [symb[-1] if symb[-1] == '0' or symb[-1] == '1' or symb[-1] == '2' else '' for symb in stressedArpabetTuple]\n",
    "#     print('arpabetSymbols:')\n",
    "#     print(arpabetSymbols)\n",
    "#     print('stresses:')\n",
    "#     print(stresses)\n",
    "#     print('together:')\n",
    "    \n",
    "    IPAsymbols = list(map(arpabetToUnicodeIPA, arpabetSymbols))\n",
    "    stressedIPAtuple = tuple(map(lambda s01: s01[0] + s01[1], zip(IPAsymbols,stresses)))\n",
    "    \n",
    "    dottedStressedIPAsymbols = tupleToDottedString( stressedIPAtuple )\n",
    "    return dottedStressedIPAsymbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.747400Z",
     "start_time": "2019-04-15T16:11:08.742275Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ɛ2.k.s.k.l.ʌ0.m.eɪ1.ʃ.ʌ0.n.p.ɔɪ2.n.t'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IPAifyStressedRep('EH2.K.S.K.L.AH0.M.EY1.SH.AH0.N.P.OY2.N.T')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply mapping to CMU lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:08.753012Z",
     "start_time": "2019-04-15T16:11:08.748342Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Orthography': '!EXCLAMATION-POINT',\n",
       "  'Transcription': 'EH2.K.S.K.L.AH0.M.EY1.SH.AH0.N.P.OY2.N.T'},\n",
       " {'Orthography': '\"CLOSE-QUOTE', 'Transcription': 'K.L.OW1.Z.K.W.OW1.T'},\n",
       " {'Orthography': '\"DOUBLE-QUOTE', 'Transcription': 'D.AH1.B.AH0.L.K.W.OW1.T'},\n",
       " {'Orthography': '\"END-OF-QUOTE', 'Transcription': 'EH1.N.D.AH0.V.K.W.OW1.T'},\n",
       " {'Orthography': '\"END-QUOTE', 'Transcription': 'EH1.N.D.K.W.OW1.T'}]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_lexicon[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:11.521230Z",
     "start_time": "2019-04-15T16:11:08.753971Z"
    }
   },
   "outputs": [],
   "source": [
    "lexicon_IPA_stress = list(map(lambda entry: modify_dict(entry, 'Transcription', IPAifyStressedRep(entry['Transcription'])),\n",
    "                              ds_lexicon))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:11.526062Z",
     "start_time": "2019-04-15T16:11:11.522425Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Orthography': '!EXCLAMATION-POINT',\n",
       "  'Transcription': 'ɛ2.k.s.k.l.ʌ0.m.eɪ1.ʃ.ʌ0.n.p.ɔɪ2.n.t'},\n",
       " {'Orthography': '\"CLOSE-QUOTE', 'Transcription': 'k.l.oʊ1.z.k.w.oʊ1.t'},\n",
       " {'Orthography': '\"DOUBLE-QUOTE', 'Transcription': 'd.ʌ1.b.ʌ0.l.k.w.oʊ1.t'},\n",
       " {'Orthography': '\"END-OF-QUOTE', 'Transcription': 'ɛ1.n.d.ʌ0.v.k.w.oʊ1.t'},\n",
       " {'Orthography': '\"END-QUOTE', 'Transcription': 'ɛ1.n.d.k.w.oʊ1.t'}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lexicon_IPA_stress[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:11.533017Z",
     "start_time": "2019-04-15T16:11:11.527770Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ɛ.k.s.k.l.ʌ.m.eɪ.ʃ.ʌ.n.p.ɔɪ.n.t'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def destress(symbol):\n",
    "    return symbol[:-1] if symbol[-1] == '0' or symbol[-1] == '1' or symbol[-1] == '2' else symbol[:]\n",
    "\n",
    "def removeStress(stressedRep):\n",
    "    stressedIPATuple = ds2t(stressedRep)\n",
    "    IPAtuple = tuple(map(destress,\n",
    "                         stressedIPATuple))\n",
    "    return t2ds(IPAtuple)\n",
    "\n",
    "removeStress('ɛ2.k.s.k.l.ʌ0.m.eɪ1.ʃ.ʌ0.n.p.ɔɪ2.n.t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:12.021853Z",
     "start_time": "2019-04-15T16:11:11.534432Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Orthography': '!EXCLAMATION-POINT',\n",
       "  'Transcription': 'ɛ.k.s.k.l.ʌ.m.eɪ.ʃ.ʌ.n.p.ɔɪ.n.t'},\n",
       " {'Orthography': '\"CLOSE-QUOTE', 'Transcription': 'k.l.oʊ.z.k.w.oʊ.t'},\n",
       " {'Orthography': '\"DOUBLE-QUOTE', 'Transcription': 'd.ʌ.b.ʌ.l.k.w.oʊ.t'},\n",
       " {'Orthography': '\"END-OF-QUOTE', 'Transcription': 'ɛ.n.d.ʌ.v.k.w.oʊ.t'},\n",
       " {'Orthography': '\"END-QUOTE', 'Transcription': 'ɛ.n.d.k.w.oʊ.t'}]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lexicon_IPA_nostress = list(map(lambda entry: modify_dict(entry, 'Transcription', removeStress(entry['Transcription'])),\n",
    "                                lexicon_IPA_stress))\n",
    "lexicon_IPA_nostress[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:12.144104Z",
     "start_time": "2019-04-15T16:11:12.023445Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " cmudict-0.7b\r\n",
      " cmudict-0.7b_IPA_destressed.tsv\r\n",
      " cmudict-0.7b_IPA_stressed.tsv\r\n",
      " cmudict-0.7b.tsv\r\n",
      "'Converting the CMU dictionary to use IPA representations.ipynb'\r\n"
     ]
    }
   ],
   "source": [
    "%ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:12.149184Z",
     "start_time": "2019-04-15T16:11:12.146434Z"
    }
   },
   "outputs": [],
   "source": [
    "stressed_fn = 'cmudict-0.7b_IPA_stressed.tsv'\n",
    "destressed_fn = 'cmudict-0.7b_IPA_destressed.tsv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:12.467900Z",
     "start_time": "2019-04-15T16:11:12.151177Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(stressed_fn, 'w', newline='', encoding='utf-8') as tsvfile:\n",
    "    writer = csv.DictWriter(tsvfile, delimiter='\\t', fieldnames=['Orthography', 'Transcription'], quoting=csv.QUOTE_NONE, quotechar='@')\n",
    "    writer.writeheader()\n",
    "    writer.writerows(lexicon_IPA_stress)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:13.102164Z",
     "start_time": "2019-04-15T16:11:12.470020Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(destressed_fn, 'w', newline='', encoding='utf-8') as tsvfile:\n",
    "    writer = csv.DictWriter(tsvfile, delimiter='\\t', fieldnames=['Orthography', 'Transcription'], quoting=csv.QUOTE_NONE, quotechar='@')\n",
    "    writer.writeheader()\n",
    "    writer.writerows(lexicon_IPA_nostress)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:13.222764Z",
     "start_time": "2019-04-15T16:11:13.103379Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     1\tOrthography\tTranscription\r",
      "\r\n",
      "     2\t!EXCLAMATION-POINT\tɛ2.k.s.k.l.ʌ0.m.eɪ1.ʃ.ʌ0.n.p.ɔɪ2.n.t\r",
      "\r\n",
      "     3\t\"CLOSE-QUOTE\tk.l.oʊ1.z.k.w.oʊ1.t\r",
      "\r\n",
      "     4\t\"DOUBLE-QUOTE\td.ʌ1.b.ʌ0.l.k.w.oʊ1.t\r",
      "\r\n",
      "     5\t\"END-OF-QUOTE\tɛ1.n.d.ʌ0.v.k.w.oʊ1.t\r",
      "\r\n",
      "     6\t\"END-QUOTE\tɛ1.n.d.k.w.oʊ1.t\r",
      "\r\n",
      "     7\t\"IN-QUOTES\tɪ1.n.k.w.oʊ1.t.s\r",
      "\r\n",
      "     8\t\"QUOTE\tk.w.oʊ1.t\r",
      "\r\n",
      "     9\t\"UNQUOTE\tʌ1.n.k.w.oʊ1.t\r",
      "\r\n",
      "    10\t#HASH-MARK\th.æ1.m.ɑ2.ɹ.k\r",
      "\r\n",
      "    11\t#POUND-SIGN\tp.aʊ1.n.d.s.aɪ2.n\r",
      "\r\n",
      "    12\t#SHARP-SIGN\tʃ.ɑ1.ɹ.p.s.aɪ2.n\r",
      "\r\n",
      "    13\t%PERCENT\tp.ɚ0.s.ɛ1.n.t\r",
      "\r\n",
      "    14\t&AMPERSAND\tæ1.m.p.ɚ0.s.æ2.n.d\r",
      "\r\n",
      "    15\t'ALLO\tɑ2.l.oʊ1\r",
      "\r\n",
      "    16\t'APOSTROPHE\tʌ0.p.ɑ1.s.t.ɹ.ʌ0.f.i0\r",
      "\r\n",
      "    17\t'BOUT\tb.aʊ1.t\r",
      "\r\n",
      "    18\t'CAUSE\tk.ʌ0.z\r",
      "\r\n",
      "    19\t'COURSE\tk.ɔ1.ɹ.s\r",
      "\r\n",
      "    20\t'CUSE\tk.j.u1.z\r",
      "\r\n",
      "    21\t'EM\tʌ0.m\r",
      "\r\n",
      "    22\t'END-INNER-QUOTE\tɛ1.n.d.ɪ1.n.ɚ0.k.w.oʊ1.t\r",
      "\r\n",
      "    23\t'END-QUOTE\tɛ1.n.d.k.w.oʊ1.t\r",
      "\r\n",
      "    24\t'FRISCO\tf.ɹ.ɪ1.s.k.oʊ0\r",
      "\r\n",
      "    25\t'GAIN\tg.ɛ1.n\r",
      "\r\n",
      "    26\t'INNER-QUOTE\tɪ1.n.ɚ0.k.w.oʊ1.t\r",
      "\r\n",
      "    27\t'KAY\tk.eɪ1\r",
      "\r\n",
      "    28\t'M\tʌ0.m\r",
      "\r\n",
      "    29\t'N\tʌ0.n\r",
      "\r\n",
      "    30\t'QUOTE\tk.w.oʊ1.t\r",
      "\r\n",
      "    31\t'ROUND\tɹ.aʊ1.n.d\r",
      "\r\n",
      "    32\t'S\tɛ1.s\r",
      "\r\n",
      "    33\t'SINGLE-QUOTE\ts.ɪ1.ŋ.g.ʌ0.l.k.w.oʊ1.t\r",
      "\r\n",
      "    34\t'TIL\tt.ɪ1.l\r",
      "\r\n",
      "    35\t'TIS\tt.ɪ1.z\r",
      "\r\n",
      "    36\t'TWAS\tt.w.ʌ1.z\r",
      "\r\n",
      "    37\t(BEGIN-PARENS\tb.ɪ0.g.ɪ1.n.p.ɚ0.ɛ1.n.z\r",
      "\r\n",
      "    38\t(IN-PARENTHESES\tɪ1.n.p.ɚ0.ɛ1.n.θ.ʌ0.s.i2.z\r",
      "\r\n",
      "    39\t(LEFT-PAREN\tl.ɛ1.f.t.p.ɚ0.ɛ1.n\r",
      "\r\n",
      "    40\t(OPEN-PARENTHESES\toʊ1.p.ʌ0.n.p.ɚ0.ɛ1.n.θ.ʌ0.s.i2.z\r",
      "\r\n",
      "    41\t(PAREN\tp.ɚ0.ɛ1.n\r",
      "\r\n",
      "    42\t(PARENS\tp.ɚ0.ɛ1.n.z\r",
      "\r\n",
      "    43\t(PARENTHESES\tp.ɚ0.ɛ1.n.θ.ʌ0.s.i2.z\r",
      "\r\n",
      "    44\t)CLOSE-PAREN\tk.l.oʊ1.z.p.ɚ0.ɛ1.n\r",
      "\r\n",
      "    45\t)CLOSE-PARENTHESES\tk.l.oʊ1.z.p.ɚ0.ɛ1.n.θ.ʌ0.s.i2.z\r",
      "\r\n",
      "    46\t)END-PAREN\tɛ1.n.d.p.ɚ0.ɛ1.n\r",
      "\r\n",
      "    47\t)END-PARENS\tɛ1.n.d.p.ɚ0.ɛ1.n.z\r",
      "\r\n",
      "    48\t)END-PARENTHESES\tɛ1.n.d.p.ɚ0.ɛ1.n.θ.ʌ0.s.i2.z\r",
      "\r\n",
      "    49\t)END-THE-PAREN\tɛ1.n.d.ð.ʌ0.p.ɚ0.ɛ1.n\r",
      "\r\n",
      "    50\t)PAREN\tp.ɚ0.ɛ1.n\r",
      "\r\n",
      "    51\t)PARENS\tp.ɚ0.ɛ1.n.z\r",
      "\r\n",
      "    52\t)RIGHT-PAREN\tɹ.aɪ1.t.p.ɛ2.ɹ.ʌ0.n\r",
      "\r\n",
      "    53\t)UN-PARENTHESES\tʌ1.n.p.ɚ0.ɛ1.n.θ.ʌ0.s.i1.z\r",
      "\r\n",
      "    54\t+PLUS\tp.l.ʊ1.s\r",
      "\r\n",
      "    55\t,COMMA\tk.ɑ1.m.ʌ0\r",
      "\r\n",
      "    56\t--DASH\td.æ1.ʃ\r",
      "\r\n",
      "    57\t-DASH\td.æ1.ʃ\r",
      "\r\n",
      "    58\t-HYPHEN\th.aɪ1.f.ʌ0.n\r",
      "\r\n",
      "    59\t...ELLIPSIS\tɪ2.l.ɪ1.p.s.ɪ0.s\r",
      "\r\n",
      "    60\t.DECIMAL\td.ɛ1.s.ʌ0.m.ʌ0.l\r",
      "\r\n",
      "cat: write error: Broken pipe\r\n"
     ]
    }
   ],
   "source": [
    "%cat -n cmudict-0.7b_IPA_stressed.tsv | head -60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-15T16:11:13.347638Z",
     "start_time": "2019-04-15T16:11:13.225036Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     1\tOrthography\tTranscription\r",
      "\r\n",
      "     2\t!EXCLAMATION-POINT\tɛ.k.s.k.l.ʌ.m.eɪ.ʃ.ʌ.n.p.ɔɪ.n.t\r",
      "\r\n",
      "     3\t\"CLOSE-QUOTE\tk.l.oʊ.z.k.w.oʊ.t\r",
      "\r\n",
      "     4\t\"DOUBLE-QUOTE\td.ʌ.b.ʌ.l.k.w.oʊ.t\r",
      "\r\n",
      "     5\t\"END-OF-QUOTE\tɛ.n.d.ʌ.v.k.w.oʊ.t\r",
      "\r\n",
      "     6\t\"END-QUOTE\tɛ.n.d.k.w.oʊ.t\r",
      "\r\n",
      "     7\t\"IN-QUOTES\tɪ.n.k.w.oʊ.t.s\r",
      "\r\n",
      "     8\t\"QUOTE\tk.w.oʊ.t\r",
      "\r\n",
      "     9\t\"UNQUOTE\tʌ.n.k.w.oʊ.t\r",
      "\r\n",
      "    10\t#HASH-MARK\th.æ.m.ɑ.ɹ.k\r",
      "\r\n",
      "    11\t#POUND-SIGN\tp.aʊ.n.d.s.aɪ.n\r",
      "\r\n",
      "    12\t#SHARP-SIGN\tʃ.ɑ.ɹ.p.s.aɪ.n\r",
      "\r\n",
      "    13\t%PERCENT\tp.ɚ.s.ɛ.n.t\r",
      "\r\n",
      "    14\t&AMPERSAND\tæ.m.p.ɚ.s.æ.n.d\r",
      "\r\n",
      "    15\t'ALLO\tɑ.l.oʊ\r",
      "\r\n",
      "    16\t'APOSTROPHE\tʌ.p.ɑ.s.t.ɹ.ʌ.f.i\r",
      "\r\n",
      "    17\t'BOUT\tb.aʊ.t\r",
      "\r\n",
      "    18\t'CAUSE\tk.ʌ.z\r",
      "\r\n",
      "    19\t'COURSE\tk.ɔ.ɹ.s\r",
      "\r\n",
      "    20\t'CUSE\tk.j.u.z\r",
      "\r\n",
      "    21\t'EM\tʌ.m\r",
      "\r\n",
      "    22\t'END-INNER-QUOTE\tɛ.n.d.ɪ.n.ɚ.k.w.oʊ.t\r",
      "\r\n",
      "    23\t'END-QUOTE\tɛ.n.d.k.w.oʊ.t\r",
      "\r\n",
      "    24\t'FRISCO\tf.ɹ.ɪ.s.k.oʊ\r",
      "\r\n",
      "    25\t'GAIN\tg.ɛ.n\r",
      "\r\n",
      "    26\t'INNER-QUOTE\tɪ.n.ɚ.k.w.oʊ.t\r",
      "\r\n",
      "    27\t'KAY\tk.eɪ\r",
      "\r\n",
      "    28\t'M\tʌ.m\r",
      "\r\n",
      "    29\t'N\tʌ.n\r",
      "\r\n",
      "    30\t'QUOTE\tk.w.oʊ.t\r",
      "\r\n",
      "    31\t'ROUND\tɹ.aʊ.n.d\r",
      "\r\n",
      "    32\t'S\tɛ.s\r",
      "\r\n",
      "    33\t'SINGLE-QUOTE\ts.ɪ.ŋ.g.ʌ.l.k.w.oʊ.t\r",
      "\r\n",
      "    34\t'TIL\tt.ɪ.l\r",
      "\r\n",
      "    35\t'TIS\tt.ɪ.z\r",
      "\r\n",
      "    36\t'TWAS\tt.w.ʌ.z\r",
      "\r\n",
      "    37\t(BEGIN-PARENS\tb.ɪ.g.ɪ.n.p.ɚ.ɛ.n.z\r",
      "\r\n",
      "    38\t(IN-PARENTHESES\tɪ.n.p.ɚ.ɛ.n.θ.ʌ.s.i.z\r",
      "\r\n",
      "    39\t(LEFT-PAREN\tl.ɛ.f.t.p.ɚ.ɛ.n\r",
      "\r\n",
      "    40\t(OPEN-PARENTHESES\toʊ.p.ʌ.n.p.ɚ.ɛ.n.θ.ʌ.s.i.z\r",
      "\r\n",
      "    41\t(PAREN\tp.ɚ.ɛ.n\r",
      "\r\n",
      "    42\t(PARENS\tp.ɚ.ɛ.n.z\r",
      "\r\n",
      "    43\t(PARENTHESES\tp.ɚ.ɛ.n.θ.ʌ.s.i.z\r",
      "\r\n",
      "    44\t)CLOSE-PAREN\tk.l.oʊ.z.p.ɚ.ɛ.n\r",
      "\r\n",
      "    45\t)CLOSE-PARENTHESES\tk.l.oʊ.z.p.ɚ.ɛ.n.θ.ʌ.s.i.z\r",
      "\r\n",
      "    46\t)END-PAREN\tɛ.n.d.p.ɚ.ɛ.n\r",
      "\r\n",
      "    47\t)END-PARENS\tɛ.n.d.p.ɚ.ɛ.n.z\r",
      "\r\n",
      "    48\t)END-PARENTHESES\tɛ.n.d.p.ɚ.ɛ.n.θ.ʌ.s.i.z\r",
      "\r\n",
      "    49\t)END-THE-PAREN\tɛ.n.d.ð.ʌ.p.ɚ.ɛ.n\r",
      "\r\n",
      "    50\t)PAREN\tp.ɚ.ɛ.n\r",
      "\r\n",
      "    51\t)PARENS\tp.ɚ.ɛ.n.z\r",
      "\r\n",
      "    52\t)RIGHT-PAREN\tɹ.aɪ.t.p.ɛ.ɹ.ʌ.n\r",
      "\r\n",
      "    53\t)UN-PARENTHESES\tʌ.n.p.ɚ.ɛ.n.θ.ʌ.s.i.z\r",
      "\r\n",
      "    54\t+PLUS\tp.l.ʊ.s\r",
      "\r\n",
      "    55\t,COMMA\tk.ɑ.m.ʌ\r",
      "\r\n",
      "    56\t--DASH\td.æ.ʃ\r",
      "\r\n",
      "    57\t-DASH\td.æ.ʃ\r",
      "\r\n",
      "    58\t-HYPHEN\th.aɪ.f.ʌ.n\r",
      "\r\n",
      "    59\t...ELLIPSIS\tɪ.l.ɪ.p.s.ɪ.s\r",
      "\r\n",
      "    60\t.DECIMAL\td.ɛ.s.ʌ.m.ʌ.l\r",
      "\r\n",
      "cat: write error: Broken pipe\r\n"
     ]
    }
   ],
   "source": [
    "%cat -n cmudict-0.7b_IPA_destressed.tsv | head -60"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}