pii_scrubber.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "class Scrubber:\n",
    "    def __init__(self):\n",
    "        # Borrowed from Gattaca which borrowed from textacy package, thanks!\n",
    "        self.EMAIL_REGEX = re.compile(r\"(?:^|(?<=[^\\w@.)]))([\\w+-](\\.(?!\\.))?)*?[\\w+-]@(?:\\w-?)*?\\w+(\\.([a-z]{2,})){1,3}(?:$|(?=\\b))\", flags=re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "        # I revised the phone_regex, as the old one captured credit card numbers as phone numbers\n",
    "        # self.PHONE_REGEX = re.compile(r'(?:^|(?<=[^\\w)]))(\\+?1[ .-]?)?(\\(?\\d{3}\\)?[ .-]?)?\\d{3}[ .-]?\\d{4}(\\s?(?:ext\\.?|[#x-])\\s?\\d{2,6})?(?:$|(?=\\W))')\n",
    "        self.PHONE_REGEX = re.compile(r\"((?:(?:(?:\\+\\d)?\\d\\d\\d[\\s-]?\\d\\d\\d[\\s-]?\\d\\d\\d\\d)|(?:(?:\\+\\d)?\\(\\d\\d\\d\\)[\\s-]?\\d\\d\\d[\\s-]?\\d\\d\\d\\d))(?:(?:ext\\.?|[#x-])\\s?\\d{2,6})?)\")\n",
    "        \n",
    "        # Borrowed from scrubadub, thanks!\n",
    "        self.SSN_REGEX   = re.compile((\"[0-9][0-9][0-9][\\S ][0-9][0-9][\\S ][0-9][0-9][0-9][0-9]\"), re.VERBOSE)\n",
    "\n",
    "        # This matches credit card numbers. Figured if we have SSN matched, might as well have this\n",
    "        # Captures: 1-Credit Card Number 2-3rd set of 4 digits (this can almost certainly be ignored)\n",
    "        self.CREDIT_REGEX = re.compile (r\"((?:\\d{4}[ -]){3}\\d{4})\")\n",
    "\n",
    "        \n",
    "        # This matches latitudes and longitudes of any of the major formats (degrees mins seconds, degrees mins, or just degrees)\n",
    "        # Assumes degree symbol not used\n",
    "        # Captures: 1-Latitude 2-Longitude 3-Longitude 4-Latitude\n",
    "        self.LATLONG_REGEX = re.compile(r\"(\\d+.{1,2}\\d+.{1,2}\\d+.{1,2}[NS])\\s*(\\d+.{1,2}\\d+.{1,2}\\d+.{1,2}[EW])|(\\d+\\.\\d+[NS])(\\d+\\.\\d+[EW])\") \n",
    "\n",
    "        # This matches addresses following pretty specific format. I'm not sure how many people will post their whole address in the proper\n",
    "        # formatting, but it might catch a couple here and there. \n",
    "        # Captures: 1-Street address, 2-city, 3-state, 4-zip code\n",
    "        # Accepted Format:  Street Address followed by comma or new line\n",
    "        #          City name (comma, space) State(space) Zip Code\n",
    "        # \n",
    "        self.ADDRESS_REGEX = re.compile(r\"([\\w ]*)[,|\\n]([a-zA-Z ]+), ([a-zA-Z]+) (\\d+\\-?\\d*)\")\n",
    " \n",
    "    def email(self, text): \n",
    "        return re.sub(self.EMAIL_REGEX, '[SCRUBBED_EMAIL]', text)\n",
    "    \n",
    "    def phone(self, text): \n",
    "        return re.sub(self.PHONE_REGEX, '[SCRUBBED_PHONE]', text)\n",
    "\n",
    "    def ssn(self, text): \n",
    "        return re.sub(self.SSN_REGEX, '[SCRUBBED_SSN]', text)\n",
    "\n",
    "    def latlong(self, text): \n",
    "        return re.sub(self.LATLONG_REGEX, '[SCRUBBED_LATLONG]', text)\n",
    "\n",
    "    def credit(self, text): \n",
    "        return re.sub(self.CREDIT_REGEX, '[SCRUBBED_CREDIT]', text)\n",
    "    \n",
    "    def address(self, text):\n",
    "        return re.sub(self.ADDRESS_REGEX, '[SCRUBBED_ADDRESS]', text)\n",
    "\n",
    "    def all_pii(self, text):\n",
    "        text = self.email(text)\n",
    "        text = self.phone(text)\n",
    "        text = self.ssn(text)\n",
    "        text = self.credit(text)  \n",
    "        text = self.latlong(text)\n",
    "        text = self.address(text)\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Scrubber' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-61e25b24d32f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# example\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Hi this is Sam at 800-213-9888. my credit card is very truly 5000-4400-9897-9999 and my social security number is 000=22 2222. Thx'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mscrub\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mScrubber\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mscrub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall_pii\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Scrubber' is not defined"
     ]
    }
   ],
   "source": [
    "# example\n",
    "text = 'Hi this is Sam at 800-213-9888. my credit card is very truly 5000-4400-9897-9999 and my social security number is 000=22 2222. Thx'\n",
    "scrub = Scrubber()\n",
    "scrub.all_pii(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": false,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}