01_generate_pseudopeople_simulated_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6f802687-416a-4b38-82d3-ab387dc16c38",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Generate pseudopeople simulated datasets\n",
    "\n",
    "The very first step is generating pseudopeople data that will be used to create the files for the case study."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a821e3e-9ae1-466d-b24f-66d1b0703a92",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "import pseudopeople as psp\n",
    "import os, time\n",
    "import logging\n",
    "from pathlib import Path\n",
    "\n",
    "# Importing pandas for access, regardless of whether we are using it as the compute engine\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57d8bb5e-1f72-4db1-9d64-a7758e7ba087",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf1e371-a255-4ed0-86d0-fde6cabbc49f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from person_linkage_case_study_utils import distributed_compute, utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc4546dc-f329-48b7-9988-5d25991939f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.simplefilter(action=\"ignore\", category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ac996f3-1270-4211-a829-728e3b2ba6f7",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# DO NOT EDIT if this notebook is not called 01_generate_pseudopeople_simulated_datasets.ipynb!\n",
    "# This notebook is designed to be run with papermill; this cell is tagged 'parameters'\n",
    "# When you run this, save it to another filename.\n",
    "data_to_use = \"small_sample\"\n",
    "output_dir = \"output/01_generate_pseudopeople_simulated_datasets\"\n",
    "compute_engine = \"pandas\"\n",
    "# Only matter if distributing\n",
    "compute_engine_num_workers = 5\n",
    "compute_engine_cpus_per_worker = 2\n",
    "compute_engine_threads_per_worker = 1\n",
    "compute_engine_memory_per_worker = \"10GB\"\n",
    "queue = None\n",
    "account = None\n",
    "# NOTE: This is, as Dask requests, a directory local to the compute node.\n",
    "# But IHME's cluster doesn't support this very well -- it can be small-ish,\n",
    "# full of stuff from other users, etc.\n",
    "compute_engine_local_directory = (\n",
    "    f\"/tmp/{os.environ['USER']}_{int(time.time())}_person_linkage_case_study\"\n",
    ")\n",
    "walltime = None\n",
    "compute_engine_memory_constrained = True\n",
    "scheduler = \"slurm\"\n",
    "\n",
    "very_noisy = True\n",
    "pseudopeople_seed = 0\n",
    "\n",
    "ri_simulated_population = None\n",
    "usa_simulated_population = None\n",
    "\n",
    "log_directory = f\"{output_dir}/{data_to_use}/logs\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73c05dea-d01e-4ca6-bb3c-b3ddea03eb68",
   "metadata": {},
   "outputs": [],
   "source": [
    "if compute_engine != \"pandas\":\n",
    "    utils.ensure_empty(compute_engine_local_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d23e90be-2f7f-46db-ae17-a3dae23b7a4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = str(Path(output_dir) / data_to_use)\n",
    "utils.ensure_empty(output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c907a06-99bc-4d30-a46c-bd25e3318c01",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ops, pd = distributed_compute.start_compute_engine(\n",
    "    compute_engine,\n",
    "    num_workers=compute_engine_num_workers,\n",
    "    cpus_per_worker=compute_engine_cpus_per_worker,\n",
    "    threads_per_worker=compute_engine_threads_per_worker,\n",
    "    memory_per_worker=compute_engine_memory_per_worker,\n",
    "    worker_walltime=walltime,\n",
    "    local_directory=compute_engine_local_directory,\n",
    "    log_directory=log_directory,\n",
    "    memory_constrained=compute_engine_memory_constrained,\n",
    "    scheduler=scheduler,\n",
    "    queue=queue,\n",
    "    account=account,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79ff29f5-473d-4dab-9e7f-453f284ad8d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "! date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41e6be4e-a9b4-47c3-ae1a-f6e7d2c9dc6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "psp.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a08bfa0-f644-4faa-b303-ef24d9a32b86",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Load simulated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8abfcc66-502c-45f1-a269-d0c60fa5f0c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "if data_to_use == \"small_sample\":\n",
    "    pseudopeople_input_dir = None\n",
    "elif data_to_use == \"ri\":\n",
    "    assert ri_simulated_population is not None\n",
    "    pseudopeople_input_dir = ri_simulated_population\n",
    "elif data_to_use == \"usa\":\n",
    "    assert usa_simulated_population is not None\n",
    "    pseudopeople_input_dir = usa_simulated_population\n",
    "else:\n",
    "    raise ValueError()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc05826f-98d7-4dac-adf9-8d9f80809811",
   "metadata": {},
   "outputs": [],
   "source": [
    "psp_kwargs = {\n",
    "    \"source\": pseudopeople_input_dir,\n",
    "    \"seed\": pseudopeople_seed,\n",
    "}\n",
    "if compute_engine.startswith(\"dask\"):\n",
    "    psp_kwargs[\"engine\"] = \"dask\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02bd7b4d-645d-4dca-94ae-ee1780048aa0",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Noise configuration\n",
    "\n",
    "In order to give ourselves more of a challenge, we significantly increase the amount of noise\n",
    "from the pseudopeople defaults."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dbb002e-8468-4885-a4b5-52fdb75ce7c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "default_configuration = psp.get_config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a42207b2-5f2b-40c1-82d8-4d6ca36fe4c4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Helper functions for changing the default configuration according to a pattern\n",
    "def column_noise_value(dataset, column, noise_type, default_value):\n",
    "    if very_noisy and dataset in (\n",
    "        \"decennial_census\",\n",
    "        \"taxes_w2_and_1099\",\n",
    "        \"social_security\",\n",
    "    ):\n",
    "        if noise_type == \"make_typos\":\n",
    "            if column == \"middle_initial\":\n",
    "                # 5% of middle initials (which are all a single token anyway) are wrong.\n",
    "                return {\"cell_probability\": 0.05, \"token_probability\": 1}\n",
    "            elif column in (\"first_name\", \"last_name\", \"street_name\"):\n",
    "                # 10% of these text columns were entered carelessly, at a rate of 1 error\n",
    "                # per 10 characters.\n",
    "                # The pseudopeople default is 1% careless.\n",
    "                return {\"cell_probability\": 0.1, \"token_probability\": 0.1}\n",
    "        elif noise_type == \"write_wrong_digits\" and (\n",
    "            dataset != \"social_security\" or column != \"ssn\"\n",
    "        ):\n",
    "            # 10% of number columns were written carelessly, at a rate of 1 error\n",
    "            # per 10 characters.\n",
    "            # The pseudopeople default is 1% careless.\n",
    "            # Note that this is applied on top of (the default lower levels of) typos,\n",
    "            # since typos also apply to numeric characters.\n",
    "            # We never introduce error on the SSN in the SSA dataset\n",
    "            return {\"cell_probability\": 0.1, \"token_probability\": 0.1}\n",
    "\n",
    "    return default_value\n",
    "\n",
    "\n",
    "def row_noise_value(dataset, noise_type, default_value):\n",
    "    return default_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7c9dd19-eac0-455f-9af3-7bc7bc3029e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "custom_configuration = {\n",
    "    dataset: {\n",
    "        noise_category: (\n",
    "            (\n",
    "                {\n",
    "                    column: {\n",
    "                        noise_type: column_noise_value(\n",
    "                            dataset, column, noise_type, noise_type_config\n",
    "                        )\n",
    "                        for noise_type, noise_type_config in column_config.items()\n",
    "                    }\n",
    "                    for column, column_config in noise_category_config.items()\n",
    "                }\n",
    "                if noise_category == \"column_noise\"\n",
    "                else {\n",
    "                    noise_type: row_noise_value(dataset, noise_type, noise_type_config)\n",
    "                    for noise_type, noise_type_config in noise_category_config.items()\n",
    "                }\n",
    "            )\n",
    "        )\n",
    "        for noise_category, noise_category_config in dataset_config.items()\n",
    "    }\n",
    "    for dataset, dataset_config in default_configuration.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8056884-bb09-475b-9499-56ba79b96c6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "psp_kwargs[\"config\"] = custom_configuration"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fe8e46b-f936-4704-b120-9bbcbce6c3e9",
   "metadata": {},
   "source": [
    "### Simulated 1040 tax filings\n",
    "\n",
    "We assume that the last 5 years of taxes would be available and used in the construction of the reference files -- see section about reference files below.\n",
    "\n",
    "Note that these are retrieved by *tax* year, so the 2029 taxes would be available in early 2030\n",
    "(around when our hypothetical case study is taking place)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8157a22b-45f3-4fc1-803e-4670a554b23f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tax_years = list(range(2025, 2030))\n",
    "tax_years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b600fee-3aeb-474a-b4e2-9099aec164be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "psp_kwargs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b9a92b7-bdaf-454d-8ad4-97db95d6b461",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "for year in tax_years:\n",
    "    print(year)\n",
    "    df = psp.generate_taxes_1040(\n",
    "        year=year,\n",
    "        **psp_kwargs,\n",
    "    )\n",
    "    df_ops.to_parquet(\n",
    "        df, str(Path(output_dir) / f\"simulated_taxes_1040_{year}.parquet\")\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8bad94f-c803-41e0-bf17-9ecfdb56cdbd",
   "metadata": {},
   "source": [
    "### Simulated W2/1099 tax filings\n",
    "\n",
    "We assume that the last 5 years of taxes would be available and used in the construction of the reference files.\n",
    "\n",
    "Note that these are retrieved by *tax* year, so the 2029 taxes would be available in early 2030\n",
    "(around when our hypothetical case study is taking place)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c072c69-f7c7-4bf1-b99e-48c3257063ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "for year in tax_years:\n",
    "    print(year)\n",
    "    df = psp.generate_taxes_w2_and_1099(\n",
    "        year=year,\n",
    "        **psp_kwargs,\n",
    "    )\n",
    "    df_ops.to_parquet(\n",
    "        df, str(Path(output_dir) / f\"simulated_taxes_w2_and_1099_{year}.parquet\")\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed6967cb-dfa6-4b43-98b9-e590d266f55f",
   "metadata": {},
   "source": [
    "### Simulated 2030 Census Unedited File (CUF)\n",
    "\n",
    "For now, we gloss over the data schema for addresses.\n",
    "We don't know how addresses would be formatted in the CUF (and it's hard to guess, because\n",
    "address is not part of the Census form), but it likely would have some of these fields\n",
    "(street number, street name, etc) combined.\n",
    "\n",
    "While PVS input files do not in general have names split into first, middle, and last,\n",
    "I am guessing the CUF **would** have first name, middle initial, last name (which is how pseudopeople\n",
    "generates it), because that [matches the Census questionnaire](https://www2.census.gov/programs-surveys/decennial/2020/technical-documentation/questionnaires-and-instructions/questionnaires/2020-informational-questionnaire-english_DI-Q1.pdf)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "670c2201-c8a3-43d8-8f3a-f98bf576666a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "simulated_census_2030 = psp.generate_decennial_census(\n",
    "    year=2030,\n",
    "    **psp_kwargs,\n",
    ")\n",
    "df_ops.to_parquet(\n",
    "    simulated_census_2030, str(Path(output_dir) / f\"simulated_census_2030.parquet\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "480d3767-1895-4c4f-91d8-72d13a19b5ec",
   "metadata": {},
   "source": [
    "### Simulated SSA Numident\n",
    "\n",
    "Wagner and Layne, p.4:\n",
    "\n",
    "> The reference files are derived from the Social Security Administration\n",
    "    (SSA) Numerical Identification file (SSA Numident). The Numident contains all\n",
    "    transactions recorded against one Social Security Number (SSN)...\n",
    "\n",
    "Based on the [SSA Numident through 2007 which is publicly available from NARA](https://aad.archives.gov/aad/series-description.jsp?s=5057),\n",
    "we know there are three kinds of transactions: SSN applications, deaths, and claiming benefits.\n",
    "SSN holders may change their information (e.g. changing name or sex) by submitting another application,\n",
    "which generates an additional application transaction.\n",
    "(The policies about this are found [on the SSA website](https://secure.ssa.gov/poms.nsf/lnx/0110212200).)\n",
    "\n",
    "The paper [\"Likely Transgender Individuals in U.S. Federal Administrative Records and the 2010 Census\" by Benjamin Cerf Harris](https://www.census.gov/content/dam/Census/library/working-papers/2015/adrm/carra-wp-2015-03.pdf)\n",
    "includes some helpful statistics (Table 2).\n",
    "The average person in the SSA Numident has 2.2 transactions (called \"claims\" in that paper, but with the same definition\n",
    "as our term \"transaction\": \"Any time an SSN is created or information associated with an existing SSN is changed, that event is registered\n",
    "as a claim.\").\n",
    "\n",
    "pseudopeople does not currently include correction, name change, or benefits claim transactions.\n",
    "It only includes SSN creation and death of the SSN holder.\n",
    "\n",
    "I've figured that there would be some delay in getting the Numident -- so by Census processing time\n",
    "for the 2030 Census, only the SSA transactions by the end of 2029 would be available.\n",
    "Note that with pseudopeople's current design it is only possible to set this cutoff at the end of a calendar year.\n",
    "The NORC report says that \"the Census NUMIDENT is recreated each year, to reflect\n",
    "Social Security transaction records through **March** of each year\" (p. 105),\n",
    "though it isn't clear when in the year the Census Numident is actually re-created."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3da0de7b-2e91-4d23-96bd-d331a820d7e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "simulated_ssa_numident = psp.generate_social_security(\n",
    "    year=2029,\n",
    "    **psp_kwargs,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "122a7f79-8054-4664-b916-f023585bf398",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "df_ops.to_parquet(\n",
    "    simulated_ssa_numident, str(Path(output_dir) / \"simulated_ssa_numident.parquet\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac19c70b-36c9-486f-9e65-6a32ca876ad7",
   "metadata": {},
   "outputs": [],
   "source": [
    "! date"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "person_linkage_case_study_20240423",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}