Enrich Names CSV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e76a4264",
   "metadata": {},
   "source": [
    "# Process Names Spreadsheet\n",
    "\n",
    "This notebook walk you through the process of parsing a CSV containing names and appending demographic facts.\n",
    "\n",
    "This notebook runs on Python3 and has no external dependencies.\n",
    "\n",
    "Before you get started, you will need to prepare to make API calls to the HumanGraphics API. If you need help, this [article](https://www.humangraphics.io/help-center-articles/making-your-first-api-call) walks you through the preparation process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "7d41c760",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up some context\n",
    "\n",
    "# First, read our home directory from the environment\n",
    "from os import getenv\n",
    "HOME = getenv(\"HOME\")\n",
    "\n",
    "# Next, load our .humangraphics file.\n",
    "#\n",
    "# It should be stored at: $HOME/.humangraphics\n",
    "#\n",
    "# It should look like this (unindented), where $PLAN is the plan you're subscribed to:\n",
    "#\n",
    "#    [api]\n",
    "#    key = XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n",
    "#    base_url = https://api.humangraphics.io/$PLAN\n",
    "#\n",
    "from configparser import ConfigParser\n",
    "config = ConfigParser()\n",
    "config.read(f\"{HOME}/.humangraphics\")\n",
    "\n",
    "HUMANGRAPHICS_API_KEY = config[\"api\"][\"key\"]\n",
    "HUMANGRAPHICS_API_BASE_URL = config[\"api\"][\"base_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "cad5ac34",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import Request, urlopen\n",
    "from urllib.error import HTTPError\n",
    "from json import dumps, loads\n",
    "from time import sleep\n",
    "\n",
    "# Here are some functions to help along the way\n",
    "def parse_human_name(name):\n",
    "    attempt = 1\n",
    "    while True:\n",
    "        # Prepare our entity content\n",
    "        entity = dumps({\"text\": name}).encode(\"utf-8\")\n",
    "\n",
    "        # Prepare our request\n",
    "        rin = Request(f\"{HUMANGRAPHICS_API_BASE_URL}/v1/humans/names/parse\")\n",
    "        rin.add_header(\"x-blobr-key\", HUMANGRAPHICS_API_KEY)\n",
    "        rin.add_header(\"content-type\", \"application/json; charset=utf-8\")\n",
    "        rin.add_header(\"content-length\", len(entity))\n",
    "        rin.add_header(\"accept\", \"application/json\")\n",
    "\n",
    "        # Make our request\n",
    "        try:\n",
    "            rout = urlopen(rin, entity)\n",
    "        except HTTPError as e:\n",
    "            if e.code == 429:\n",
    "                # Let's use an exponential backoff strategy\n",
    "                sleep(min(64, 1 << (attempt - 1)))\n",
    "                attempt = attempt + 1\n",
    "                continue\n",
    "            raise e;\n",
    "            \n",
    "        break\n",
    "\n",
    "        # Process our successful response\n",
    "        \n",
    "    # Decoding UTF-8 is safe here because HumanGraphics always uses UTF-8 for JSON.\n",
    "    return loads(rout.read().decode(\"utf-8\"))\n",
    "\n",
    "def dict_multi_get(d, *args):\n",
    "    if d is None:\n",
    "        return None\n",
    "\n",
    "    result = d\n",
    "    for arg in args:\n",
    "        result = result.get(arg)\n",
    "        if result is None:\n",
    "            return None\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "0c0bb18c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up some contants\n",
    "NAME_FIELD_NAME = \"name\"\n",
    "\n",
    "HUMAN_NAME_CONFIDENCE = \"human_name_confidence\"\n",
    "GIVEN_NAME_FIELD_NAME = \"given_name\"\n",
    "MIDDLE_NAME_FIELD_NAME = \"middle_name\"\n",
    "SECOND_MIDDLE_NAME_FIELD_NAME = \"second_middle_name\"\n",
    "NICK_NAME_FIELD_NAME = \"nick_name\"\n",
    "FAMILY_NAME_FIELD_NAME = \"family_name\"\n",
    "SECOND_FAMILY_NAME_FIELD_NAME = \"second_family_name\"\n",
    "\n",
    "AGE_BEST_ESTIMATE_VALUE_FIELD_NAME = \"age_best_estimate_value\"\n",
    "AGE_BEST_ESTIMATE_LIKELIHOOD_FIELD_NAME = \"age_best_estimate_likelihood\"\n",
    "AGE_UNDER_18_LIKELIHOOD_FIELD_NAME = \"age_under_18_likelihood\"\n",
    "AGE_FROM_18_TO_24_LIKELIHOOD_FIELD_NAME = \"age_from_18_to_24_likelihood\"\n",
    "AGE_FROM_25_TO_34_LIKELIHOOD_FIELD_NAME = \"age_from_25_to_34_likelihood\"\n",
    "AGE_FROM_35_TO_44_LIKELIHOOD_FIELD_NAME = \"age_from_35_to_44_likelihood\"\n",
    "AGE_FROM_45_TO_54_LIKELIHOOD_FIELD_NAME = \"age_from_45_to_54_likelihood\"\n",
    "AGE_FROM_55_to_64_LIKELIHOOD_FIELD_NAME = \"age_from_55_to_64_likelihood\"\n",
    "AGE_OVER_65_LIKELIHOOD_FIELD_NAME = \"age_over_65_likelihood\"\n",
    "\n",
    "GENDER_BEST_ESTIMATE_VALUE_FIELD_NAME = \"gender_best_estimate_value\"\n",
    "GENDER_BEST_ESTIMATE_LIKELIHOOD_FIELD_NAME = \"gender_best_etimate_likelihood\"\n",
    "GENDER_MALE_LIKELIHOOD_FIELD_NAME = \"gender_male_likelihood\"\n",
    "GENDER_FEMALE_LIKELIHOOD_FIELD_NAME = \"gender_female_likelihood\"\n",
    "\n",
    "RACE_BEST_ESTIMATE_VALUE_FIELD_NAME = \"race_best_estimate_value\"\n",
    "RACE_BEST_ETIMATE_LIKELIHOOD_FIELD_NAME = \"race_best_estimate_likelihood\"\n",
    "RACE_WHITE_LIKELIHOOD_FIELD_NAME = \"race_white_likelihood\"\n",
    "RACE_BLACK_LIKELIHOOD_FIELD_NAME = \"race_black_likelihood\"\n",
    "RACE_ASIAN_PACIFIC_ISLANDER_LIKELIHOOD_FIELD_NAME = \"race_asian_pacific_islander_likelihood\"\n",
    "RACE_AMERICAN_INDIAN_LIKELIHOOD_FIELD_NAME = \"race_american_indian_likelihood\"\n",
    "RACE_HISPANIC_LIKELIHOOD_FIELD_NAME = \"race_hispanic_likelihood\"\n",
    "RACE_TWO_PLUS_RACES_LIKELIHOOD_FIELD_NAME = \"race_two_plus_races_likelihood\"\n",
    "\n",
    "def get_age_best_estimate_likelihood(d):\n",
    "    age_best_estimate = dict_multi_get(d, \"age\")\n",
    "    return None if age_best_estimate is None else dict_multi_get(d, \"ageEstimate\", age_best_estimate)\n",
    "\n",
    "def get_gender_best_estimate_likelihood(d):\n",
    "    gender_best_estimate = dict_multi_get(d, \"gender\")\n",
    "    return None if gender_best_estimate is None else dict_multi_get(d, \"genderEstimate\", gender_best_estimate)\n",
    "\n",
    "def get_race_best_estimate_likelihood(d):\n",
    "    race_best_estimate = dict_multi_get(d, \"race\")\n",
    "    return None if race_best_estimate is None else dict_multi_get(d, \"raceEstimate\", race_best_estimate)\n",
    "                \n",
    "APPENDED_FIELDS = [\n",
    "    (HUMAN_NAME_CONFIDENCE, lambda d: dict_multi_get(d, \"confidence\")),\n",
    "    (GIVEN_NAME_FIELD_NAME, lambda d: dict_multi_get(d, \"match\", \"givenName\", \"matchedText\")),\n",
    "    (MIDDLE_NAME_FIELD_NAME, lambda d: dict_multi_get(d, \"match\", \"middleName\", \"matchedText\")),\n",
    "    (SECOND_MIDDLE_NAME_FIELD_NAME, lambda d: dict_multi_get(d, \"match\", \"secondMiddleName\", \"matchedText\")),\n",
    "    (NICK_NAME_FIELD_NAME, lambda d: dict_multi_get(d, \"match\", \"nickName\", \"matchedText\")),\n",
    "    (FAMILY_NAME_FIELD_NAME, lambda d: dict_multi_get(d, \"match\", \"familyName\", \"matchedText\")),\n",
    "    (SECOND_FAMILY_NAME_FIELD_NAME, lambda d: dict_multi_get(d, \"match\", \"secondFamilyName\", \"matchedText\")),\n",
    "    (AGE_BEST_ESTIMATE_VALUE_FIELD_NAME, lambda d: dict_multi_get(d, \"age\")),\n",
    "    (AGE_BEST_ESTIMATE_LIKELIHOOD_FIELD_NAME, get_age_best_estimate_likelihood),\n",
    "    (AGE_UNDER_18_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"under18\")),\n",
    "    (AGE_FROM_18_TO_24_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"from18To24\")),\n",
    "    (AGE_FROM_25_TO_34_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"from25To34\")),\n",
    "    (AGE_FROM_35_TO_44_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"from35To44\")),\n",
    "    (AGE_FROM_45_TO_54_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"from45To54\")),\n",
    "    (AGE_FROM_55_to_64_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"from55To64\")),\n",
    "    (AGE_OVER_65_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"ageEstimate\", \"over65\")),\n",
    "    (GENDER_BEST_ESTIMATE_VALUE_FIELD_NAME, lambda d: dict_multi_get(d, \"gender\")),\n",
    "    (GENDER_BEST_ESTIMATE_LIKELIHOOD_FIELD_NAME, get_gender_best_estimate_likelihood),\n",
    "    (GENDER_MALE_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"genderEstimate\", \"male\")),\n",
    "    (GENDER_FEMALE_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"genderEstimate\", \"female\")),\n",
    "    (RACE_BEST_ESTIMATE_VALUE_FIELD_NAME, lambda d: dict_multi_get(d, \"race\")),\n",
    "    (RACE_BEST_ETIMATE_LIKELIHOOD_FIELD_NAME, get_race_best_estimate_likelihood),\n",
    "    (RACE_WHITE_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"raceEstimate\", \"white\")),\n",
    "    (RACE_BLACK_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"raceEstimate\", \"black\")),\n",
    "    (RACE_ASIAN_PACIFIC_ISLANDER_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"raceEstimate\", \"asianPacificIslander\")),\n",
    "    (RACE_AMERICAN_INDIAN_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"raceEstimate\", \"americanIndian\")),\n",
    "    (RACE_HISPANIC_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"raceEstimate\", \"hispanic\")),\n",
    "    (RACE_TWO_PLUS_RACES_LIKELIHOOD_FIELD_NAME, lambda d: dict_multi_get(d, \"raceEstimate\", \"twoPlusRaces\")),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "161b794d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from csv import DictReader, DictWriter\n",
    "\n",
    "INPUT_FILENAME = f\"{HOME}/records.csv\"\n",
    "OUTPUT_FILENAME = f\"{HOME}/enriched-records.csv\"\n",
    "\n",
    "# The input file should contain a header row with field names.\n",
    "# It must define a field called \"name\" which contains the name to parse.\n",
    "# Any additional fields are ignored and passed along to the output.\n",
    "with open(INPUT_FILENAME, \"r\", newline=\"\") as input_file:\n",
    "    input_rows = DictReader(input_file)\n",
    "    \n",
    "    if NAME_FIELD_NAME not in input_rows.fieldnames:\n",
    "        raise ValueError(f\"Input file {INPUT_FILENAME} does not contain a field named {NAME_FIELD_NAME}\")\n",
    "        \n",
    "    with open(OUTPUT_FILENAME, \"w\", newline=\"\") as output_file:\n",
    "        output_rows = DictWriter(output_file, fieldnames=input_rows.fieldnames + [ fi[0] for fi in APPENDED_FIELDS ])\n",
    "        \n",
    "        output_rows.writeheader()\n",
    "        \n",
    "        for input_row in input_rows:\n",
    "            name = input_row[NAME_FIELD_NAME].strip()\n",
    "            \n",
    "            if name == \"\":\n",
    "                output_rows.writerow(input_row | { fi[0]: \"\" for fi in APPENDED_FIELDS })\n",
    "                continue\n",
    "                \n",
    "            parsed_human_name = None\n",
    "            try:\n",
    "                parsed_human_name = parse_human_name(name)\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                output_rows.writerow(input_row | { fi[0]: \"\" for fi in APPENDED_FIELDS })\n",
    "                continue\n",
    "                \n",
    "            output_row = {**input_row}\n",
    "            for fi in APPENDED_FIELDS:\n",
    "                name = fi[0]\n",
    "                value = fi[1](parsed_human_name)\n",
    "                output_row[name] = str(value) if value else \"\"\n",
    "                \n",
    "            # print(output_row)\n",
    "                \n",
    "            output_rows.writerow(output_row)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}